package runner import ( "os" "path/filepath" "slices" "strings" "testing" "gopkg.in/yaml.v3" "textmachine/platform/internal/ingest" ) // `--dry-run` is the whole of what separates a preview from a write, so its presence is pinned at // the argv — the one place a dropped flag is a one-token diff. // // Mutation caught: passing preview through without the flag (every call becomes a write), or // passing the flag unconditionally (every write becomes a no-op). func TestAPreviewIsTheDryRunFlagAndNothingElse(t *testing.T) { if !slices.Contains(BankApplyArgs("/w", "/d.json", true), "--dry-run") { t.Error("a preview call carries no --dry-run: it would WRITE") } if slices.Contains(BankApplyArgs("/w", "/d.json", false), "--dry-run") { t.Error("an apply call carries --dry-run: it would never write") } } // The canon's central promise about the preview, proven against the REAL engine: «"preview": true // answers the same receipt without changing anything» (§applyBankCorrections). A regress of one // constant turns the safe look into an irreversible write, and no unit test can make this claim — // what writes is the engine, so the engine is what must be seen not writing. // // Gated like every live-engine test: a bare clone stays green and says why. func TestALivePreviewWritesNothingAndALiveApplyWrites(t *testing.T) { bin := os.Getenv("TM_PLATFORM_TEST_ENGINE_BIN") tpl := os.Getenv("TM_PLATFORM_TEST_BOOK_TEMPLATE") if bin == "" || tpl == "" { t.Skip("TM_PLATFORM_TEST_ENGINE_BIN and TM_PLATFORM_TEST_BOOK_TEMPLATE not set: " + "the preview's no-write promise is not checked against a real engine") } dir := t.TempDir() writeProbeBook(t, tpl, dir, "bk_PREVIEWPROBE") doc, err := ingest.EncodeDecisions("bk_PREVIEWPROBE", []ingest.BankDecision{ {Action: "approve", Src: "测试", Sense: "", Dst: "проба", Note: "preview pin"}, }) if err != nil { t.Fatal(err) } decisions := filepath.Join(t.TempDir(), "decisions.json") if err := os.WriteFile(decisions, doc, 0o600); err != nil { t.Fatal(err) } before := dirListing(t, dir) rn := New(nil) out, err := rn.BankApply(t.Context(), bin, dir, decisions, true) if err != nil || !out.Exited || out.ExitCode != 0 || !out.Decoded { t.Fatalf("the live preview did not answer: %+v (%v; stderr %q)", out, err, out.Stderr) } if out.Report.Mode != "projection" || !out.Report.Changed { t.Fatalf("the preview's report: mode %q changed %v, want a projection that WOULD change", out.Report.Mode, out.Report.Changed) } // Nothing appeared and nothing moved — except the lock file, which the verb's own contract // names: a projection computed without the arbiter is a projection of nothing. after := dirListing(t, dir) for name, sum := range after { if filepath.Ext(name) == ".lock" { continue } if prev, ok := before[name]; !ok { t.Errorf("the preview CREATED %s", name) } else if prev != sum { t.Errorf("the preview CHANGED %s", name) } } // The same document applied for real is the counter-proof: the engine both can and does write // here, so the silence above was the preview's doing and not a fixture that cannot write. out, err = rn.BankApply(t.Context(), bin, dir, decisions, false) if err != nil || out.ExitCode != 0 || !out.Decoded || out.Report.Mode != "apply" || !out.Report.Changed { t.Fatalf("the live apply did not land: %+v (%v; stderr %q)", out, err, out.Stderr) } applied := dirListing(t, dir) grew := false for name := range applied { if _, ok := after[name]; !ok && filepath.Ext(name) != ".lock" { grew = true } } if !grew { t.Error("the apply wrote no decision file: this fixture proves nothing about the preview") } } // str reads one string key of a decoded book configuration, and says which key is missing rather // than panicking on the type assertion — a template without a `pipeline:` is an operator's mistake a // test should name. func str(t *testing.T, cfg map[string]any, key string) string { t.Helper() v, ok := cfg[key].(string) if !ok || v == "" { t.Fatalf("the deployment template carries no %q: this probe cannot render a book from it", key) } return v } // zeroCostPipeline writes the template's pipeline with every stage pointed at the deployment's LOCAL // model, and returns the path. The escalation budget is zeroed and the per-stage hops are dropped // with it: an escalation target is a reachable model too, and one paid name is all it takes for the // engine to demand a key. // // The local model is FOUND, not named here: models.yaml declares providers with a `kind`, and the // one whose kind is `local` is the pair a stand serves itself (the in-test stub listens where that // provider points). A deployment that declares no local provider cannot run this probe for free, and // the skip says exactly that instead of failing on somebody's missing key. func zeroCostPipeline(t *testing.T, pipelinePath, modelsPath string) string { t.Helper() var models struct { Providers map[string]struct { Kind string `yaml:"kind"` } `yaml:"providers"` Models map[string]struct { Provider string `yaml:"provider"` } `yaml:"models"` } raw, err := os.ReadFile(modelsPath) if err != nil { t.Fatalf("the template's models file cannot be read (%s): %v", modelsPath, err) } if err := yaml.Unmarshal(raw, &models); err != nil { t.Fatal(err) } free := "" for name, m := range models.Models { if models.Providers[m.Provider].Kind == "local" && (free == "" || name < free) { free = name } } if free == "" { t.Skipf("this deployment's %s declares no provider of kind `local`: the probe would have to buy its calls", modelsPath) } raw, err = os.ReadFile(pipelinePath) if err != nil { t.Fatalf("the template's pipeline cannot be read (%s): %v", pipelinePath, err) } var pipe map[string]any if err := yaml.Unmarshal(raw, &pipe); err != nil { t.Fatal(err) } stages, ok := pipe["stages"].([]any) if !ok || len(stages) == 0 { t.Fatalf("the template's pipeline declares no stages: %s", pipelinePath) } for _, s := range stages { stage, ok := s.(map[string]any) if !ok { t.Fatalf("a stage of %s is not a mapping", pipelinePath) } stage["model"] = free delete(stage, "escalate_to") delete(stage, "label_models") } // ⚠ THE GATES CARRY MODELS TOO, and the helper's own promise — «the probe runs on the deployment's // ZERO-COST pair» — is not kept without them. It went unnoticed for the same reason the contrast // path did: until the bank contour was turned on in the shipping pipeline (unified backlog row // 140) a deployment template named no gate models at all, and the stage loop above was the whole // of the deployment's spending. On a `pipeline-c1` template it is not: the terminologist and the // classifier resolve to a paid provider, and the engine refuses the run for a missing key long // before the guard this probe is about could fire. if gates, ok := pipe["gates"].(map[string]any); ok { for _, g := range gates { gate, ok := g.(map[string]any) if !ok { continue } if _, has := gate["model"]; has { gate["model"] = free } // ⚠ AND THE SECOND MODEL IN THE SAME GATE. `classify_model` is the classifier phase's own // model and falls back to `model` only when it is EMPTY (backend/internal/config/pipeline.go, // `ClassifyModel`) — so a deployment whose pipeline sets it would resolve a paid model here // with `model` already zeroed, which is the same failure this helper was just fixed for, one // key over. Today's shipping config does not set it; a deployment's may. if _, has := gate["classify_model"]; has { gate["classify_model"] = free } delete(gate, "escalate_to") } } if esc, ok := pipe["escalation"].(map[string]any); ok { esc["budget_usd"] = 0 delete(esc, "chains") } // ⚠ THE BANK CONTOUR'S ONE DEPLOYMENT-PROVIDED INPUT has to be re-pointed, because this rendered // pipeline is about to MOVE. `mining.contrast_path` is resolved against the directory of the // pipeline file (backend/internal/config/pipeline.go, resolvePrompt), the artefact is a // multi-megabyte word-frequency list that is deliberately NOT in git, and the mirror below copies // neither it nor anything else of that size — so a relative path silently becomes a path into an // empty temp directory, and every live test on such a template dies at the engine's write-path // guard with a message that reads like a defect of this zone. // // It appeared the day the contour was turned on in the SHIPPING pipeline (unified backlog row // 140): before that a deployment template pointing at `pipeline-c1.yaml` named no contrast at all, // which is why this harness had never had to think about it. if mining, ok := pipe["mining"].(map[string]any); ok { if raw, ok := mining["contrast_path"].(string); ok && raw != "" { abs := raw if !filepath.IsAbs(abs) { abs = filepath.Join(filepath.Dir(pipelinePath), abs) } if _, err := os.Stat(abs); err != nil { // A HOST CONDITION and not a failure: the artefact is deployment-provided, and a clone // that has not been given one cannot run a write-path probe. Loud, and it names the // file, because the alternative is a red test whose message is about somebody else. t.Skipf("this deployment's pipeline enables the bank contour and names %s, which is not on this host: "+ "a live write-path probe cannot run (put the jieba-style contrast list there — deploy/README.md, "+ "«Артефакт контраста»)", abs) } mining["contrast_path"] = abs } } out, err := yaml.Marshal(pipe) if err != nil { t.Fatal(err) } // The rendered pipeline lives in a MIRROR of the deployment's configuration directory, not next // to the book: everything else a pipeline pulls in is resolved relative to it — the prompt pack // by convention (`/pairs/.yaml`, whose own default root is `../../prompts`) and // the pair's chunking calibration by the same path. Writing the file anywhere else silently // changes those two, and the engine then refuses to load at all («no prompt for pair zh-ru»). // Linking rather than copying keeps the probe honest about WHOSE prompts it ran on. // A directory of its OWN, never the book's: what lands beside book.yaml is the book, and one // probe here inventories that directory file by file to prove a preview wrote nothing. root := t.TempDir() cfgDir := filepath.Join(root, "cfg") if err := os.MkdirAll(cfgDir, 0o755); err != nil { t.Fatal(err) } origCfg := filepath.Dir(pipelinePath) for target, link := range map[string]string{ filepath.Join(origCfg, "..", "prompts"): filepath.Join(root, "prompts"), filepath.Join(origCfg, "pairs"): filepath.Join(cfgDir, "pairs"), filepath.Join(origCfg, "langpacks"): filepath.Join(cfgDir, "langpacks"), } { if _, err := os.Stat(target); err != nil { continue // this deployment does not carry that layer; the engine's own fallback applies } if err := os.Symlink(target, link); err != nil { t.Skipf("the probe cannot mirror the deployment's %s (%v): it would run on different conventions than the deployment does", target, err) } } path := filepath.Join(cfgDir, "pipeline-zero-cost.yaml") if err := os.WriteFile(path, out, 0o600); err != nil { t.Fatal(err) } return path } // writeProbeBook renders a minimal live book: the deployment template with the identity, languages // and source the intake would have filled in (the four keys books.Service.provision sets). func writeProbeBook(t *testing.T, tpl, dir, bookID string) { t.Helper() raw, err := os.ReadFile(tpl) if err != nil { t.Fatal(err) } var cfg map[string]any if err := yaml.Unmarshal(raw, &cfg); err != nil { t.Fatal(err) } cfg["book_id"] = bookID cfg["source_lang"] = "zh" cfg["target_lang"] = "ru" cfg["source_file"] = "source.txt" // The probe runs on the deployment's ZERO-COST pair, and it is rendered here rather than assumed // of the template: a deployment template is an OPERATOR's artefact whose pipeline points at // whatever that deployment translates with — a paid model, on every stand built by the recipe in // STACK_DECISIONS — and the engine refuses to load a configuration whose reachable models have no // key long before any guard these tests are about could fire (config.checkKeysFor, which skips a // provider of kind `local`). So the probe keeps the template's every other key and swaps the // stage models for the local one, derived from the same models.yaml the book points at. cfg["pipeline"] = zeroCostPipeline(t, str(t, cfg, "pipeline"), str(t, cfg, "models")) out, err := yaml.Marshal(cfg) if err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(dir, ConfigFile), out, 0o600); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(dir, "source.txt"), []byte("第1章 测试\n这是一个测试文本。\n"), 0o600); err != nil { t.Fatal(err) } } // dirListing is the book directory as facts: every file name with a digest of its bytes. func dirListing(t *testing.T, dir string) map[string]string { t.Helper() out := map[string]string{} entries, err := os.ReadDir(dir) if err != nil { t.Fatal(err) } for _, e := range entries { if e.IsDir() { continue } b, err := os.ReadFile(filepath.Join(dir, e.Name())) if err != nil { t.Fatal(err) } out[e.Name()] = string(rune(len(b))) + "-" + string(b[:min(len(b), 64)]) } return out } // The rendering itself, on a configuration written for the purpose and with no engine involved: what // makes the live probes free is that NO paid model stays reachable, and "reachable" includes the // escalation hop of a stage. On the stand's own template that hop costs nothing to leave in — its // escalation budget is zero, so the engine does not ask for its key — but a deployment configured for // real work sets that budget, and then one leftover hop buys the probe a bill. // // Mutation caught: keeping `escalate_to` or `label_models` on a stage; leaving the escalation budget // as the template set it; choosing a model that is not the local provider's. func TestTheProbePipelineLeavesNoPaidModelReachable(t *testing.T) { dir := t.TempDir() models := filepath.Join(dir, "models.yaml") if err := os.WriteFile(models, []byte(` providers: paid: { kind: openai_compatible, api_key_env: SOME_KEY } bench: { kind: local, base_url: http://127.0.0.1:11434/v1 } models: expensive: { provider: paid } free-one: { provider: bench } `), 0o600); err != nil { t.Fatal(err) } pipeline := filepath.Join(dir, "pipeline.yaml") if err := os.WriteFile(pipeline, []byte(` core: T version: 1 stages: - name: draft role: translator model: expensive escalate_to: expensive - name: edit role: editor model: expensive label_models: { violence: expensive } escalation: budget_usd: 5 chains: { a: [expensive] } `), 0o600); err != nil { t.Fatal(err) } var rendered map[string]any raw, err := os.ReadFile(zeroCostPipeline(t, pipeline, models)) if err != nil { t.Fatal(err) } if err := yaml.Unmarshal(raw, &rendered); err != nil { t.Fatal(err) } if strings.Contains(string(raw), "expensive") { t.Fatalf("a paid model survived the rendering:\n%s", raw) } stages, _ := rendered["stages"].([]any) if len(stages) != 2 { t.Fatalf("the rendering lost the stages: %v", rendered["stages"]) } for _, s := range stages { stage := s.(map[string]any) if stage["model"] != "free-one" { t.Errorf("stage %v runs on %v, want the local provider's model", stage["name"], stage["model"]) } if _, ok := stage["escalate_to"]; ok { t.Errorf("stage %v kept an escalation hop", stage["name"]) } } esc, _ := rendered["escalation"].(map[string]any) if esc == nil || esc["budget_usd"] != 0 { t.Errorf("the escalation budget is %v, want it spent down to nothing", esc["budget_usd"]) } // The template's own shape is otherwise untouched — the probe runs the deployment's pipeline, // not a pipeline of the test's invention. if rendered["core"] != "T" || rendered["version"] != 1 { t.Errorf("the rendering changed more than the models: %v", rendered) } }