textmachine/backend/internal/pipeline/live_conformance_test.go

//go:build live

package pipeline

// live_conformance_test.go: per-adapter LIVE conformance (§C, D12 Q3). This file is
// OUTSIDE the default test run — it is build-tagged `live`, so `go test ./...` never
// compiles or touches it and CI stays key-free and $0. Each call is REAL MONEY
// (pennies, but real). Run it manually on backend/.env keys:
//
//	set -a; . ./.env; set +a
//	TM_LIVE=1 go test -tags live -run TestLive -v ./internal/pipeline/
//
// It hits a live provider with a tiny dense-CJK→ru translation and asserts the
// ADAPTER produced a VALID translation — not an echo (the DeepSeek echo-mine, still
// disarmed on the real wire), not empty (reasoning ate the budget → read `content`,
// raise max_tokens), not a truncation — and logs the parsed usage + the response
// slug (canonicalization). It reuses the runner's own verdict machinery (classify),
// so "valid" here means exactly what the pipeline means by ok.
//
// Division of labour (D12 Q3): the Polygon Python oracle is the CROSS-provider
// acceptance GATE ("every adapter hits a live provider and gets a valid translation");
// these Go tests are the per-adapter wire+parse checks. BOTH — the oracle is the gate.

import (
	"context"
	"os"
	"strings"
	"testing"
	"time"

	"textmachine/backend/internal/config"
	"textmachine/backend/internal/lang"
	"textmachine/backend/internal/llm"
	"textmachine/backend/internal/obs"
)

// liveModels are the boevoy cloud models to probe, each with a dense-CJK fragment
// that stresses exactly the silent-refusal failure modes (echo on dense CJK, empty
// on a starved reasoning budget). grok is commented out by default: it bills the
// clean-prod xAI account (the hygiene contract in models.yaml) — enable deliberately.
// srcLang is what the echo share is measured AGAINST: D39.64 replaced the Han+kana-hardcoded cjkShare with
// sourceScriptShare over the SOURCE language's declared scripts, so the probe must now say which language
// its fragment is in (a ja fragment measured against zh scripts under-reports its own kana echo).
var liveModels = []struct {
	model   string
	srcLang string
	source  string
}{
	{"deepseek-v4-flash", "zh", "祝福是鲁迅创作的短篇小说，收录于《彷徨》。故事以第一人称叙述。"}, // the echo-mine repro family (Lu Xun 祝福)
	{"glm-5", "ja", "静かな図書館の朝、彼女は古い本を開いて、ゆっくりと読み始めた。"},
	{"kimi-k2.6", "zh", "这是一个用于验证适配器的测试句子，内容需要被完整翻译成俄语。"},
	// {"grok-4.20-0309-non-reasoning", "zh", "这是一个测试句子。"}, // clean-prod xAI only
}

func TestLiveAdapterConformance(t *testing.T) {
	if os.Getenv("TM_LIVE") != "1" {
		t.Skip("live conformance is opt-in and spends real money: set TM_LIVE=1 with backend/.env loaded")
	}
	models, err := config.LoadModels("../../configs/models.yaml")
	if err != nil {
		t.Fatalf("load boevoy models.yaml: %v", err)
	}

	for _, tc := range liveModels {
		t.Run(tc.model, func(t *testing.T) {
			mod, ok := models.Models[tc.model]
			if !ok {
				t.Skipf("%s not defined in configs/models.yaml", tc.model)
			}
			if prov := models.Providers[mod.Provider]; prov.Kind != "local" && prov.APIKey() == "" {
				t.Skipf("%s: provider key %s not set — skipping", tc.model, prov.APIKeyEnv)
			}

			client, err := BuildClient(models, tc.model, obs.NewLogger())
			if err != nil {
				t.Fatalf("build client: %v", err)
			}
			// The raw-wire channel is opt-in through the SAME switch tmctl uses (main.go: LOG_LLM_BODIES=1
			// plus LOG_LEVEL=debug); without it nothing about this run changes. It exists here because
			// the neutral LLMResponse carries no thinking field at all, and every provider we probe
			// accounts reasoning as a SUBSET of completion_tokens — so "the model actually thought"
			// cannot be asserted from usage, only from reasoning_content on the raw body (quirks 00 §9).
			ctx, cancel := context.WithTimeout(obs.WithReqInfo(context.Background(), obs.ReqInfo{
				TraceID:   obs.NewTraceID(),
				LogBodies: os.Getenv("LOG_LLM_BODIES") == "1",
			}), 5*time.Minute)
			defer cancel()

			resp, err := client.Complete(ctx, llm.LLMRequest{
				Model: tc.model,
				Messages: []llm.Message{
					{Role: "system", Content: "Ты профессиональный литературный переводчик. Переведи фрагмент на русский язык. Выведи ТОЛЬКО перевод, без комментариев.", CacheBoundary: true},
					{Role: "user", Content: tc.source},
				},
				// ≥8000 so a hybrid-reasoning model's thinking does not empty `content`
				// (the DeepSeek/Kimi finish=length-with-empty-content quirk).
				MaxTokens:       8000,
				Temperature:     0.3,
				ReasoningEffort: "off",
			})
			if err != nil {
				t.Fatalf("live call failed (adapter/transport): %v", err)
			}

			// Validate with the SAME verdict machine the runner uses: a valid
			// translation is ok; an echo → cjk_artifact, empty → empty, refusal →
			// soft_refusal. The response slug is logged for canonicalization review
			// (Gemini returns models/…, xAI redirects retired slugs, etc.).
			scripts := lang.LangScripts(tc.srcLang)
			cls := classify(classifyInput{Source: tc.source, Output: resp.Text, Finish: resp.FinishReason,
				TargetLang: "ru", SourceScripts: scripts})
			t.Logf("model=%q finish=%q usage=%+v source_script_share=%.2f verdict=%q text=%.120q",
				resp.Model, resp.FinishReason, resp.Usage, sourceScriptShare(resp.Text, scripts), cls.Reason, resp.Text)

			if strings.TrimSpace(resp.Text) == "" {
				t.Fatalf("%s returned EMPTY content (read `content`, raise max_tokens — reasoning likely ate the budget)", tc.model)
			}
			if !cls.ok() {
				t.Fatalf("%s did not produce a valid translation: verdict=%s detail=%q", tc.model, cls.Reason, cls.Detail)
			}
			if resp.Model == "" {
				t.Errorf("%s: empty response model slug — cannot canonicalize price (PriceForResponse would fall back)", tc.model)
			}
		})
	}
}