textmachine/platform/internal/runner/bankapply_live_test.go

package runner

import (
	"os"
	"path/filepath"
	"slices"
	"strings"
	"testing"

	"gopkg.in/yaml.v3"

	"textmachine/platform/internal/ingest"
)

// `--dry-run` is the whole of what separates a preview from a write, so its presence is pinned at
// the argv — the one place a dropped flag is a one-token diff.
//
// Mutation caught: passing preview through without the flag (every call becomes a write), or
// passing the flag unconditionally (every write becomes a no-op).
func TestAPreviewIsTheDryRunFlagAndNothingElse(t *testing.T) {
	if !slices.Contains(BankApplyArgs("/w", "/d.json", true), "--dry-run") {
		t.Error("a preview call carries no --dry-run: it would WRITE")
	}
	if slices.Contains(BankApplyArgs("/w", "/d.json", false), "--dry-run") {
		t.Error("an apply call carries --dry-run: it would never write")
	}
}

// The canon's central promise about the preview, proven against the REAL engine: «"preview": true
// answers the same receipt without changing anything» (§applyBankCorrections). A regress of one
// constant turns the safe look into an irreversible write, and no unit test can make this claim —
// what writes is the engine, so the engine is what must be seen not writing.
//
// Gated like every live-engine test: a bare clone stays green and says why.
func TestALivePreviewWritesNothingAndALiveApplyWrites(t *testing.T) {
	bin := os.Getenv("TM_PLATFORM_TEST_ENGINE_BIN")
	tpl := os.Getenv("TM_PLATFORM_TEST_BOOK_TEMPLATE")
	if bin == "" || tpl == "" {
		t.Skip("TM_PLATFORM_TEST_ENGINE_BIN and TM_PLATFORM_TEST_BOOK_TEMPLATE not set: " +
			"the preview's no-write promise is not checked against a real engine")
	}
	dir := t.TempDir()
	writeProbeBook(t, tpl, dir, "bk_PREVIEWPROBE")
	doc, err := ingest.EncodeDecisions("bk_PREVIEWPROBE", []ingest.BankDecision{
		{Action: "approve", Src: "测试", Sense: "", Dst: "проба", Note: "preview pin"},
	})
	if err != nil {
		t.Fatal(err)
	}
	decisions := filepath.Join(t.TempDir(), "decisions.json")
	if err := os.WriteFile(decisions, doc, 0o600); err != nil {
		t.Fatal(err)
	}
	before := dirListing(t, dir)
	rn := New(nil)
	out, err := rn.BankApply(t.Context(), bin, dir, decisions, true)
	if err != nil || !out.Exited || out.ExitCode != 0 || !out.Decoded {
		t.Fatalf("the live preview did not answer: %+v (%v; stderr %q)", out, err, out.Stderr)
	}
	if out.Report.Mode != "projection" || !out.Report.Changed {
		t.Fatalf("the preview's report: mode %q changed %v, want a projection that WOULD change",
			out.Report.Mode, out.Report.Changed)
	}
	// Nothing appeared and nothing moved — except the lock file, which the verb's own contract
	// names: a projection computed without the arbiter is a projection of nothing.
	after := dirListing(t, dir)
	for name, sum := range after {
		if filepath.Ext(name) == ".lock" {
			continue
		}
		if prev, ok := before[name]; !ok {
			t.Errorf("the preview CREATED %s", name)
		} else if prev != sum {
			t.Errorf("the preview CHANGED %s", name)
		}
	}
	// The same document applied for real is the counter-proof: the engine both can and does write
	// here, so the silence above was the preview's doing and not a fixture that cannot write.
	out, err = rn.BankApply(t.Context(), bin, dir, decisions, false)
	if err != nil || out.ExitCode != 0 || !out.Decoded || out.Report.Mode != "apply" || !out.Report.Changed {
		t.Fatalf("the live apply did not land: %+v (%v; stderr %q)", out, err, out.Stderr)
	}
	applied := dirListing(t, dir)
	grew := false
	for name := range applied {
		if _, ok := after[name]; !ok && filepath.Ext(name) != ".lock" {
			grew = true
		}
	}
	if !grew {
		t.Error("the apply wrote no decision file: this fixture proves nothing about the preview")
	}
}

// str reads one string key of a decoded book configuration, and says which key is missing rather
// than panicking on the type assertion — a template without a `pipeline:` is an operator's mistake a
// test should name.
func str(t *testing.T, cfg map[string]any, key string) string {
	t.Helper()
	v, ok := cfg[key].(string)
	if !ok || v == "" {
		t.Fatalf("the deployment template carries no %q: this probe cannot render a book from it", key)
	}
	return v
}

// zeroCostPipeline writes the template's pipeline with every stage pointed at the deployment's LOCAL
// model, and returns the path. The escalation budget is zeroed and the per-stage hops are dropped
// with it: an escalation target is a reachable model too, and one paid name is all it takes for the
// engine to demand a key.
//
// The local model is FOUND, not named here: models.yaml declares providers with a `kind`, and the
// one whose kind is `local` is the pair a stand serves itself (the in-test stub listens where that
// provider points). A deployment that declares no local provider cannot run this probe for free, and
// the skip says exactly that instead of failing on somebody's missing key.
func zeroCostPipeline(t *testing.T, pipelinePath, modelsPath string) string {
	t.Helper()
	var models struct {
		Providers map[string]struct {
			Kind string `yaml:"kind"`
		} `yaml:"providers"`
		Models map[string]struct {
			Provider string `yaml:"provider"`
		} `yaml:"models"`
	}
	raw, err := os.ReadFile(modelsPath)
	if err != nil {
		t.Fatalf("the template's models file cannot be read (%s): %v", modelsPath, err)
	}
	if err := yaml.Unmarshal(raw, &models); err != nil {
		t.Fatal(err)
	}
	free := ""
	for name, m := range models.Models {
		if models.Providers[m.Provider].Kind == "local" && (free == "" || name < free) {
			free = name
		}
	}
	if free == "" {
		t.Skipf("this deployment's %s declares no provider of kind `local`: the probe would have to buy its calls", modelsPath)
	}
	raw, err = os.ReadFile(pipelinePath)
	if err != nil {
		t.Fatalf("the template's pipeline cannot be read (%s): %v", pipelinePath, err)
	}
	var pipe map[string]any
	if err := yaml.Unmarshal(raw, &pipe); err != nil {
		t.Fatal(err)
	}
	stages, ok := pipe["stages"].([]any)
	if !ok || len(stages) == 0 {
		t.Fatalf("the template's pipeline declares no stages: %s", pipelinePath)
	}
	for _, s := range stages {
		stage, ok := s.(map[string]any)
		if !ok {
			t.Fatalf("a stage of %s is not a mapping", pipelinePath)
		}
		stage["model"] = free
		delete(stage, "escalate_to")
		delete(stage, "label_models")
	}
	// ⚠ THE GATES CARRY MODELS TOO, and the helper's own promise — «the probe runs on the deployment's
	// ZERO-COST pair» — is not kept without them. It went unnoticed for the same reason the contrast
	// path did: until the bank contour was turned on in the shipping pipeline (unified backlog row
	// 140) a deployment template named no gate models at all, and the stage loop above was the whole
	// of the deployment's spending. On a `pipeline-c1` template it is not: the terminologist and the
	// classifier resolve to a paid provider, and the engine refuses the run for a missing key long
	// before the guard this probe is about could fire.
	if gates, ok := pipe["gates"].(map[string]any); ok {
		for _, g := range gates {
			gate, ok := g.(map[string]any)
			if !ok {
				continue
			}
			if _, has := gate["model"]; has {
				gate["model"] = free
			}
			// ⚠ AND THE SECOND MODEL IN THE SAME GATE. `classify_model` is the classifier phase's own
			// model and falls back to `model` only when it is EMPTY (backend/internal/config/pipeline.go,
			// `ClassifyModel`) — so a deployment whose pipeline sets it would resolve a paid model here
			// with `model` already zeroed, which is the same failure this helper was just fixed for, one
			// key over. Today's shipping config does not set it; a deployment's may.
			if _, has := gate["classify_model"]; has {
				gate["classify_model"] = free
			}
			delete(gate, "escalate_to")
		}
	}
	if esc, ok := pipe["escalation"].(map[string]any); ok {
		esc["budget_usd"] = 0
		delete(esc, "chains")
	}
	// ⚠ THE BANK CONTOUR'S ONE DEPLOYMENT-PROVIDED INPUT has to be re-pointed, because this rendered
	// pipeline is about to MOVE. `mining.contrast_path` is resolved against the directory of the
	// pipeline file (backend/internal/config/pipeline.go, resolvePrompt), the artefact is a
	// multi-megabyte word-frequency list that is deliberately NOT in git, and the mirror below copies
	// neither it nor anything else of that size — so a relative path silently becomes a path into an
	// empty temp directory, and every live test on such a template dies at the engine's write-path
	// guard with a message that reads like a defect of this zone.
	//
	// It appeared the day the contour was turned on in the SHIPPING pipeline (unified backlog row
	// 140): before that a deployment template pointing at `pipeline-c1.yaml` named no contrast at all,
	// which is why this harness had never had to think about it.
	if mining, ok := pipe["mining"].(map[string]any); ok {
		if raw, ok := mining["contrast_path"].(string); ok && raw != "" {
			abs := raw
			if !filepath.IsAbs(abs) {
				abs = filepath.Join(filepath.Dir(pipelinePath), abs)
			}
			if _, err := os.Stat(abs); err != nil {
				// A HOST CONDITION and not a failure: the artefact is deployment-provided, and a clone
				// that has not been given one cannot run a write-path probe. Loud, and it names the
				// file, because the alternative is a red test whose message is about somebody else.
				t.Skipf("this deployment's pipeline enables the bank contour and names %s, which is not on this host: "+
					"a live write-path probe cannot run (put the jieba-style contrast list there — deploy/README.md, "+
					"«Артефакт контраста»)", abs)
			}
			mining["contrast_path"] = abs
		}
	}
	out, err := yaml.Marshal(pipe)
	if err != nil {
		t.Fatal(err)
	}
	// The rendered pipeline lives in a MIRROR of the deployment's configuration directory, not next
	// to the book: everything else a pipeline pulls in is resolved relative to it — the prompt pack
	// by convention (`<config dir>/pairs/<pair>.yaml`, whose own default root is `../../prompts`) and
	// the pair's chunking calibration by the same path. Writing the file anywhere else silently
	// changes those two, and the engine then refuses to load at all («no prompt for pair zh-ru»).
	// Linking rather than copying keeps the probe honest about WHOSE prompts it ran on.
	// A directory of its OWN, never the book's: what lands beside book.yaml is the book, and one
	// probe here inventories that directory file by file to prove a preview wrote nothing.
	root := t.TempDir()
	cfgDir := filepath.Join(root, "cfg")
	if err := os.MkdirAll(cfgDir, 0o755); err != nil {
		t.Fatal(err)
	}
	origCfg := filepath.Dir(pipelinePath)
	for target, link := range map[string]string{
		filepath.Join(origCfg, "..", "prompts"): filepath.Join(root, "prompts"),
		filepath.Join(origCfg, "pairs"):         filepath.Join(cfgDir, "pairs"),
		filepath.Join(origCfg, "langpacks"):     filepath.Join(cfgDir, "langpacks"),
	} {
		if _, err := os.Stat(target); err != nil {
			continue // this deployment does not carry that layer; the engine's own fallback applies
		}
		if err := os.Symlink(target, link); err != nil {
			t.Skipf("the probe cannot mirror the deployment's %s (%v): it would run on different conventions than the deployment does", target, err)
		}
	}
	path := filepath.Join(cfgDir, "pipeline-zero-cost.yaml")
	if err := os.WriteFile(path, out, 0o600); err != nil {
		t.Fatal(err)
	}
	return path
}

// writeProbeBook renders a minimal live book: the deployment template with the identity, languages
// and source the intake would have filled in (the four keys books.Service.provision sets).
func writeProbeBook(t *testing.T, tpl, dir, bookID string) {
	t.Helper()
	raw, err := os.ReadFile(tpl)
	if err != nil {
		t.Fatal(err)
	}
	var cfg map[string]any
	if err := yaml.Unmarshal(raw, &cfg); err != nil {
		t.Fatal(err)
	}
	cfg["book_id"] = bookID
	cfg["source_lang"] = "zh"
	cfg["target_lang"] = "ru"
	cfg["source_file"] = "source.txt"
	// The probe runs on the deployment's ZERO-COST pair, and it is rendered here rather than assumed
	// of the template: a deployment template is an OPERATOR's artefact whose pipeline points at
	// whatever that deployment translates with — a paid model, on every stand built by the recipe in
	// STACK_DECISIONS — and the engine refuses to load a configuration whose reachable models have no
	// key long before any guard these tests are about could fire (config.checkKeysFor, which skips a
	// provider of kind `local`). So the probe keeps the template's every other key and swaps the
	// stage models for the local one, derived from the same models.yaml the book points at.
	cfg["pipeline"] = zeroCostPipeline(t, str(t, cfg, "pipeline"), str(t, cfg, "models"))
	out, err := yaml.Marshal(cfg)
	if err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(dir, ConfigFile), out, 0o600); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(dir, "source.txt"), []byte("第1章 测试\n这是一个测试文本。\n"), 0o600); err != nil {
		t.Fatal(err)
	}
}

// dirListing is the book directory as facts: every file name with a digest of its bytes.
func dirListing(t *testing.T, dir string) map[string]string {
	t.Helper()
	out := map[string]string{}
	entries, err := os.ReadDir(dir)
	if err != nil {
		t.Fatal(err)
	}
	for _, e := range entries {
		if e.IsDir() {
			continue
		}
		b, err := os.ReadFile(filepath.Join(dir, e.Name()))
		if err != nil {
			t.Fatal(err)
		}
		out[e.Name()] = string(rune(len(b))) + "-" + string(b[:min(len(b), 64)])
	}
	return out
}

// The rendering itself, on a configuration written for the purpose and with no engine involved: what
// makes the live probes free is that NO paid model stays reachable, and "reachable" includes the
// escalation hop of a stage. On the stand's own template that hop costs nothing to leave in — its
// escalation budget is zero, so the engine does not ask for its key — but a deployment configured for
// real work sets that budget, and then one leftover hop buys the probe a bill.
//
// Mutation caught: keeping `escalate_to` or `label_models` on a stage; leaving the escalation budget
// as the template set it; choosing a model that is not the local provider's.
func TestTheProbePipelineLeavesNoPaidModelReachable(t *testing.T) {
	dir := t.TempDir()
	models := filepath.Join(dir, "models.yaml")
	if err := os.WriteFile(models, []byte(`
providers:
  paid: { kind: openai_compatible, api_key_env: SOME_KEY }
  bench: { kind: local, base_url: http://127.0.0.1:11434/v1 }
models:
  expensive: { provider: paid }
  free-one: { provider: bench }
`), 0o600); err != nil {
		t.Fatal(err)
	}
	pipeline := filepath.Join(dir, "pipeline.yaml")
	if err := os.WriteFile(pipeline, []byte(`
core: T
version: 1
stages:
  - name: draft
    role: translator
    model: expensive
    escalate_to: expensive
  - name: edit
    role: editor
    model: expensive
    label_models: { violence: expensive }
escalation:
  budget_usd: 5
  chains: { a: [expensive] }
`), 0o600); err != nil {
		t.Fatal(err)
	}
	var rendered map[string]any
	raw, err := os.ReadFile(zeroCostPipeline(t, pipeline, models))
	if err != nil {
		t.Fatal(err)
	}
	if err := yaml.Unmarshal(raw, &rendered); err != nil {
		t.Fatal(err)
	}
	if strings.Contains(string(raw), "expensive") {
		t.Fatalf("a paid model survived the rendering:\n%s", raw)
	}
	stages, _ := rendered["stages"].([]any)
	if len(stages) != 2 {
		t.Fatalf("the rendering lost the stages: %v", rendered["stages"])
	}
	for _, s := range stages {
		stage := s.(map[string]any)
		if stage["model"] != "free-one" {
			t.Errorf("stage %v runs on %v, want the local provider's model", stage["name"], stage["model"])
		}
		if _, ok := stage["escalate_to"]; ok {
			t.Errorf("stage %v kept an escalation hop", stage["name"])
		}
	}
	esc, _ := rendered["escalation"].(map[string]any)
	if esc == nil || esc["budget_usd"] != 0 {
		t.Errorf("the escalation budget is %v, want it spent down to nothing", esc["budget_usd"])
	}
	// The template's own shape is otherwise untouched — the probe runs the deployment's pipeline,
	// not a pipeline of the test's invention.
	if rendered["core"] != "T" || rendered["version"] != 1 {
		t.Errorf("the rendering changed more than the models: %v", rendered)
	}
}