textmachine/platform/internal/runner/liveconsent_test.go

488 lines
21 KiB
Go

package runner
import (
"fmt"
"net"
"net/url"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"testing"
"gopkg.in/yaml.v3"
)
// ⛔ CONSENT, NOT CIRCUMSTANCE — the predicate in front of every live translation in this package.
//
// The live probe next door runs the REAL engine through the pipeline the deployment's book template
// names, and where those calls land is decided by that template's `models.yaml`, not by the stub the
// probe puts on `127.0.0.1:11434`: the `local` provider points AT that stub, and `deepseek` points at
// `https://api.deepseek.com/v1`. So a template carrying a vendor model sends the probe to that vendor
// and spends a reader's credit.
//
// ⛔ WHAT STOOD HERE BEFORE ASKED THE WRONG QUESTION. The only thing between that probe and a real
// invoice was a BUSY PORT — «another stand?» — which answers "is this address free", never "may this
// run spend money". A well-provisioned host answers the first one yes.
//
// ⛔ AND THE FIRST VERSION OF THIS GUARD ASKED THE RIGHT QUESTION OF THE WRONG TEXT: it walked the
// pipeline for the key `model` and cleared everything else. The engine's escalation carries models
// under OTHER keys — `escalate_to: deepseek-v4-pro`, `default: [deepseek-v4-pro, …]`, `adult: [grok-4.3]`
// — and the zone's own recipe for a $0 pipeline rewrites only `model:` lines, so the file it produces
// keeps three vendor models and the guard called it free (acceptance of 11.09, finding F1). A guard
// that can be blinded by a KEY NAME is a guard against yesterday's configuration.
//
// So it does not read keys at all. It asks the catalogue which models are not on this host and then
// looks for their NAMES in the pipeline's own bytes. A new key cannot hide a name; a renamed stage
// cannot; a structure this parser has never seen cannot. It errs towards refusing — a vendor slug in a
// COMMENT stops the run — and refusing is the safe direction.
//
// The shape is the engine zone's own and is not invented here: its live adapter probes are `TM_LIVE`
// plus a build tag, «вне дефолтного go test/CI, на живых ключах backend/.env» — a deliberate act,
// never a side effect of a battery.
const liveConsentEnv = "TM_PLATFORM_LIVE"
// paidModelIn is a PURE question about a book template: which model it could reach that is not on
// this host, "" when every model it names is, and `decided=false` when the guard cannot tell.
//
// ⛔ IT NEITHER SKIPS NOR FAILS, and that is the point of its shape (acceptance F2). The first version
// called `t.Skipf` from inside, so its own blindness came out GREEN: a mutation that stopped it seeing
// models made every fixture skip and the suite pass. An assertion and a skip condition living in one
// function cannot tell "this is free" from "I could not look". Here the caller decides: the live probe
// treats `decided=false` as a refusal to run, and the pin below asserts it, so blindness is RED where
// it is measured and SAFE where it is acted on.
func paidModelIn(template string) (paid string, decided bool, why string) {
var book struct {
Pipeline string `yaml:"pipeline"`
Models string `yaml:"models"`
}
if err := readYAMLInto(template, &book); err != nil {
return "", false, err.Error()
}
if !filepath.IsAbs(book.Pipeline) || !filepath.IsAbs(book.Models) {
// A relative path here is relative to a book directory this question does not have, and
// guessing it would answer "no vendor" about a file the guard never read.
return "", false, fmt.Sprintf("the template names pipeline %q and models %q, and only absolute paths can be resolved from here", book.Pipeline, book.Models)
}
var catalogue struct {
Providers map[string]struct {
BaseURL string `yaml:"base_url"`
} `yaml:"providers"`
Models map[string]struct {
Provider string `yaml:"provider"`
} `yaml:"models"`
}
if err := readYAMLInto(book.Models, &catalogue); err != nil {
return "", false, err.Error()
}
if len(catalogue.Models) == 0 {
return "", false, "the model catalogue names no models at all, so nothing about this pipeline can be resolved"
}
pipeline, err := os.ReadFile(book.Pipeline)
if err != nil {
return "", false, err.Error()
}
// ⚠ `default_model` IS NOT CONSULTED, and the first version of this guard was wrong to. It reads as
// a fallback and is not one: the catalogue's own words call it a «fallback-якорь ЦЕНЫ … неизвестная
// модель … книжится по этой цене», i.e. it decides how an unknown model is PRICED, never which model
// is CALLED. Counting it made the guard refuse every $0 pipeline on a catalogue whose price anchor is
// a vendor — which is the shipped one — and that is a guard nobody can satisfy.
//
// What covers the case it was reaching for is the ENGINE: a pipeline naming a model the catalogue
// does not carry does not run at all. Measured against the real binary rather than assumed —
// `tmctl manifest` over a pipeline whose every `model:` is `no-such-model-anywhere` exits 10 with
// «model "no-such-model-anywhere" is not defined in models.yaml», before any call. So a name this
// scan cannot resolve is a name the engine will not dial.
// ⛔ SORTED, because the answer is a SENTENCE AN OPERATOR READS and a map's order is not one. A
// pipeline naming two vendors would otherwise name a different one each run, and the pin that
// expects a particular name would flake — which it did, caught by this pack's own mutation round.
names := make([]string, 0, len(catalogue.Models))
for name := range catalogue.Models {
names = append(names, name)
}
sort.Strings(names)
var onHost int
for _, name := range names {
if !namedIn(pipeline, name) {
continue
}
if !onThisHost(catalogue.Providers[catalogue.Models[name].Provider].BaseURL) {
return name, true, ""
}
onHost++
}
if onHost == 0 {
// Not "free": UNREAD. A pipeline naming no model this catalogue knows is one whose calls this
// guard cannot place, and saying "free" about it is the failure this whole file exists to stop.
return "", false, "the pipeline names no model the catalogue carries, so where its calls would go cannot be established"
}
return "", true, ""
}
// namedIn reports whether a model slug appears in the pipeline's bytes as a whole name. Slugs carry
// dots and dashes, so the boundary is "not a character a slug is made of" rather than \b.
func namedIn(pipeline []byte, slug string) bool {
if slug == "" {
return false
}
return regexp.MustCompile(`(^|[^A-Za-z0-9._-])` + regexp.QuoteMeta(slug) + `([^A-Za-z0-9._-]|$)`).Match(pipeline)
}
// onThisHost reports whether a provider's address is a loopback one — the only kind whose calls
// cannot leave this machine and therefore cannot be billed.
func onThisHost(baseURL string) bool {
if baseURL == "" {
return false // an address nobody declared is not one this guard may clear
}
u, err := url.Parse(baseURL)
if err != nil {
return false
}
host := u.Hostname()
if strings.EqualFold(host, "localhost") {
return true
}
ip := net.ParseIP(host)
return ip != nil && ip.IsLoopback()
}
func readYAMLInto(path string, into any) error {
b, err := os.ReadFile(path)
if err != nil {
return fmt.Errorf("the live guard could not read %s: %w", path, err)
}
if err := yaml.Unmarshal(b, into); err != nil {
return fmt.Errorf("the live guard could not parse %s: %w", path, err)
}
return nil
}
// EVERY ANSWER THE GUARD CAN GIVE, on documents written for the question — a host's live templates
// cannot show them all at once, and the two that matter most are the ones no real template produces:
// a vendor hidden under a key nobody listed, and the guard's own blindness.
func TestTheLiveGuardTellsAPaidPipelineFromAFreeOne(t *testing.T) {
dir := t.TempDir()
catalogue := filepath.Join(dir, "models.yaml")
writeFile(t, catalogue, `
default_model: local-qwen3-8b
providers:
deepseek:
base_url: https://api.deepseek.com/v1
xai:
base_url: https://api.x.ai/v1
local:
base_url: http://127.0.0.1:11434/v1
loopback-by-name:
base_url: http://localhost:11434/v1
models:
deepseek-v4-flash:
provider: deepseek
deepseek-v4-pro:
provider: deepseek
grok-4.3:
provider: xai
local-qwen3-8b:
provider: local
local-by-name:
provider: loopback-by-name
`)
for _, tc := range []struct {
name, pipeline string
wantPaid string
wantDecided bool
}{
{"every model is on this host", "stages:\n - id: draft\n model: local-qwen3-8b\n", "", true},
{"a vendor under model", "stages:\n - id: draft\n model: local-qwen3-8b\n - id: edit\n model: deepseek-v4-flash\n", "deepseek-v4-flash", true},
// ⛔ THE ONE THAT WAS MISSED, and the shape the zone's own $0 recipe leaves behind: every
// `model:` line is local and the escalation is not.
{"a vendor under escalate_to", "stages:\n - id: draft\n model: local-qwen3-8b\n escalate_to: deepseek-v4-pro\n", "deepseek-v4-pro", true},
{"a vendor inside an escalation chain", "stages:\n - id: draft\n model: local-qwen3-8b\nescalation:\n chains:\n default: [deepseek-v4-pro, grok-4.3]\n", "deepseek-v4-pro", true},
{"a vendor under a key this guard has never heard of", "stages:\n - id: draft\n model: local-qwen3-8b\nsome_future_knob:\n whatever: grok-4.3\n", "grok-4.3", true},
{"a vendor named only in a comment still refuses", "stages:\n - id: draft\n model: local-qwen3-8b\n# was deepseek-v4-pro before the rewrite\n", "deepseek-v4-pro", true},
// A loopback address WRITTEN AS A NAME is still this host — and the branch that says so had no
// fixture until a mutation walked through it.
{"a provider on localhost by name", "stages:\n - id: draft\n model: local-by-name\n", "", true},
// ⛔ BLINDNESS IS ITS OWN ANSWER AND IT IS NOT «FREE».
{"a pipeline naming no model at all", "stages:\n - id: draft\n role: translator\n", "", false},
} {
t.Run(tc.name, func(t *testing.T) {
slug := strings.ReplaceAll(tc.name, " ", "-")
pipe := filepath.Join(dir, slug+".yaml")
writeFile(t, pipe, tc.pipeline)
tpl := filepath.Join(dir, slug+"-book.yaml")
writeFile(t, tpl, "pipeline: "+pipe+"\nmodels: "+catalogue+"\n")
paid, decided, why := paidModelIn(tpl)
if paid != tc.wantPaid || decided != tc.wantDecided {
t.Fatalf("the guard answered (paid %q, decided %v, why %q), want (%q, %v): a wrong answer "+
"here either spends a reader's credit inside a battery or stops the $0 probe from running",
paid, decided, why, tc.wantPaid, tc.wantDecided)
}
if !decided && why == "" {
t.Error("the guard could not decide and said nothing about why: a caller cannot act on that")
}
})
}
}
// THE ANSWER IS THE SAME EVERY TIME, and that is a property in its own right: it is a sentence an
// operator reads and a value a pin expects. Over a pipeline naming TWO vendors the first version of
// this guard answered whichever the map handed it first, so the fixture next door flaked — caught by
// this pack's own mutation round rather than by a red run.
//
// Twenty repetitions: Go randomises map order per range, so an unsorted answer over two candidates
// surviving twenty identical draws is about one chance in half a million.
func TestTheLiveGuardNamesTheSameModelEveryTime(t *testing.T) {
dir := t.TempDir()
catalogue := filepath.Join(dir, "models.yaml")
writeFile(t, catalogue, `
providers:
deepseek:
base_url: https://api.deepseek.com/v1
xai:
base_url: https://api.x.ai/v1
local:
base_url: http://127.0.0.1:11434/v1
models:
deepseek-v4-pro:
provider: deepseek
grok-4.3:
provider: xai
local-qwen3-8b:
provider: local
`)
pipe := filepath.Join(dir, "p.yaml")
writeFile(t, pipe, "stages:\n - id: draft\n model: local-qwen3-8b\nescalation:\n chains:\n default: [deepseek-v4-pro, grok-4.3]\n")
tpl := filepath.Join(dir, "book.yaml")
writeFile(t, tpl, "pipeline: "+pipe+"\nmodels: "+catalogue+"\n")
first, _, _ := paidModelIn(tpl)
if first == "" {
t.Fatalf("the guard cleared a pipeline naming two vendors: this fixture is not about ordering at all")
}
for i := 2; i <= 20; i++ {
got, _, _ := paidModelIn(tpl)
if got != first {
t.Fatalf("draw %d named %q where draw 1 named %q: the answer depends on map order, so the "+
"sentence an operator reads changes between runs and any pin over it flakes", i, got, first)
}
}
}
// ⛔ `default_model` IS NOT A REASON TO REFUSE, and this pin holds the line against reading it as one.
//
// It looks like a fallback and is not: the catalogue calls it a price anchor — what an UNKNOWN model
// is booked at, never what gets called. A guard that counted it refused every free pipeline on the
// shipped catalogue, whose anchor is a vendor; that is a guard nobody can satisfy, and a guard nobody
// can satisfy gets switched off.
//
// What makes the omission safe is the ENGINE, measured rather than assumed: a pipeline naming a model
// the catalogue does not carry exits 10 at config load — «model "…" is not defined in models.yaml» —
// before any call. A name this scan cannot resolve is a name that will not be dialled.
func TestTheLiveGuardDoesNotReadThePriceAnchorAsAFallback(t *testing.T) {
dir := t.TempDir()
catalogue := filepath.Join(dir, "models.yaml")
writeFile(t, catalogue, `
default_model: deepseek-v4-flash
providers:
deepseek:
base_url: https://api.deepseek.com/v1
local:
base_url: http://127.0.0.1:11434/v1
models:
deepseek-v4-flash:
provider: deepseek
local-qwen3-8b:
provider: local
`)
pipe := filepath.Join(dir, "p.yaml")
writeFile(t, pipe, "stages:\n - id: draft\n model: local-qwen3-8b\n")
tpl := filepath.Join(dir, "book.yaml")
writeFile(t, tpl, "pipeline: "+pipe+"\nmodels: "+catalogue+"\n")
if paid, decided, why := paidModelIn(tpl); paid != "" || !decided {
t.Fatalf("every model this pipeline names is on this host and the guard answered (%q, %v, %q): "+
"the catalogue's PRICE anchor is not a model anyone calls, and refusing over it makes the $0 "+
"path unreachable", paid, decided, why)
}
}
// A template the guard cannot read is not a free one.
func TestTheLiveGuardRefusesWhatItCannotResolve(t *testing.T) {
dir := t.TempDir()
// Every way the question can fail to have an answer. Each one used to be an unexercised branch,
// and an unexercised branch of a money guard is a branch nobody has read (acceptance 11.09, 7).
absent := filepath.Join(dir, "absent.yaml")
emptyCatalogue := filepath.Join(dir, "empty-models.yaml")
writeFile(t, emptyCatalogue, "providers:\n local:\n base_url: http://127.0.0.1:11434/v1\n")
brokenCatalogue := filepath.Join(dir, "broken-models.yaml")
writeFile(t, brokenCatalogue, "providers: [this is not a mapping\n")
somePipeline := filepath.Join(dir, "some-pipeline.yaml")
writeFile(t, somePipeline, "stages:\n - id: draft\n model: whatever\n")
for _, tc := range []struct{ name, body string }{
{"relative paths", "pipeline: ../p.yaml\nmodels: ../m.yaml\n"},
{"a pipeline that is not there", "pipeline: " + absent + "\nmodels: " + absent + "\n"},
{"a template that is not there at all", ""},
{"a catalogue that is not there", "pipeline: " + somePipeline + "\nmodels: " + absent + "\n"},
{"a catalogue that does not parse", "pipeline: " + somePipeline + "\nmodels: " + brokenCatalogue + "\n"},
{"a catalogue naming no models", "pipeline: " + somePipeline + "\nmodels: " + emptyCatalogue + "\n"},
} {
t.Run(tc.name, func(t *testing.T) {
tpl := filepath.Join(dir, strings.ReplaceAll(tc.name, " ", "-")+".yaml")
if tc.body != "" {
writeFile(t, tpl, tc.body)
}
paid, decided, why := paidModelIn(tpl)
if decided || paid != "" {
t.Fatalf("the guard decided (%q, %v) about a template it could not resolve", paid, decided)
}
if why == "" {
t.Error("it refused without saying why")
}
})
}
}
// ⛔ THE GUARD IS ASKED OF THE RENDER, AND THIS IS WHERE THAT EARNS ITS KEEP.
//
// The probe never hands the engine the deployment's own pipeline: `writeProbeBook` derives a
// zero-cost one from it and points the book at that. The derivation is a rewriter BY KEY — stages,
// gates, the escalation block — and it is complete today. A guard on the template could neither see
// that (it judged a document nothing loads) nor catch the day it stops being complete.
//
// So the fixture hides a vendor under a key the derivation does not know, renders the book the way
// the probe does, and asks about the RESULT. Before 11.09 this answered "free" (acceptance, finding 1).
func TestTheGuardSeesAVendorThatSurvivedTheZeroCostDerivation(t *testing.T) {
dir := t.TempDir()
catalogue := filepath.Join(dir, "models.yaml")
writeFile(t, catalogue, `
providers:
deepseek:
kind: openai
base_url: https://api.deepseek.com/v1
local:
kind: local
base_url: http://127.0.0.1:11434/v1
models:
deepseek-v4-pro:
provider: deepseek
local-qwen3-8b:
provider: local
`)
// Every key the derivation KNOWS carries a vendor here, and one key it does not.
pipe := filepath.Join(dir, "pipeline.yaml")
writeFile(t, pipe, `
stages:
- name: draft
role: translator
model: deepseek-v4-pro
escalate_to: deepseek-v4-pro
escalation:
budget_usd: 5
chains:
default: [deepseek-v4-pro]
some_knob_added_next_quarter:
model_for_something: deepseek-v4-pro
`)
tpl := filepath.Join(dir, "template.yaml")
writeFile(t, tpl, "pipeline: "+pipe+"\nmodels: "+catalogue+"\nsource_lang: zh\ntarget_lang: ru\n")
// The template itself is obviously paid, and that is NOT what is being measured…
if paid, _, _ := paidModelIn(tpl); paid == "" {
t.Fatal("the template reads as free, so this fixture is not about a survivor at all")
}
// …what is, is what the probe would actually run.
book := t.TempDir()
writeProbeBook(t, tpl, book, "bk_SURVIVORPROBE")
rendered := filepath.Join(book, ConfigFile)
paid, decided, why := paidModelIn(rendered)
if !decided {
t.Fatalf("the rendered configuration could not be resolved (%s): the guard cannot clear or refuse "+
"what it cannot read, and this is the artefact the engine loads", why)
}
if paid != "deepseek-v4-pro" {
body, _ := os.ReadFile(rendered)
t.Fatalf("the guard answered %q about the configuration the engine would load: a vendor that "+
"survived the zero-cost derivation is the only way a model gets dialled here, and it is "+
"exactly what a guard on the TEMPLATE could never see\n--- rendered ---\n%s", paid, body)
}
}
// The other half: a template whose every vendor the derivation DOES know comes out free, so the guard
// is not a blanket refusal that would take the $0 path away again.
func TestTheRenderOfAnOrdinaryPaidTemplateComesOutFree(t *testing.T) {
dir := t.TempDir()
catalogue := filepath.Join(dir, "models.yaml")
writeFile(t, catalogue, `
providers:
deepseek:
kind: openai
base_url: https://api.deepseek.com/v1
local:
kind: local
base_url: http://127.0.0.1:11434/v1
models:
deepseek-v4-pro:
provider: deepseek
local-qwen3-8b:
provider: local
`)
pipe := filepath.Join(dir, "pipeline.yaml")
writeFile(t, pipe, `
stages:
- name: draft
role: translator
model: deepseek-v4-pro
escalate_to: deepseek-v4-pro
gates:
terminology:
model: deepseek-v4-pro
escalation:
budget_usd: 5
chains:
default: [deepseek-v4-pro]
`)
tpl := filepath.Join(dir, "template.yaml")
writeFile(t, tpl, "pipeline: "+pipe+"\nmodels: "+catalogue+"\nsource_lang: zh\ntarget_lang: ru\n")
book := t.TempDir()
writeProbeBook(t, tpl, book, "bk_ORDINARYPROBE")
if paid, decided, why := paidModelIn(filepath.Join(book, ConfigFile)); paid != "" || !decided {
body, _ := os.ReadFile(filepath.Join(book, ConfigFile))
t.Fatalf("the render of an ordinary paid template answered (%q, %v, %q): the derivation covers "+
"every key here, so refusing it would take the free path away from every stand\n%s",
paid, decided, why, body)
}
}
// ⛔ AN ADDRESS IS THIS HOST'S ONLY WHEN IT SAYS SO, and every other answer is «not on this host» —
// the safe side. The branches below were reachable and unexercised, and one of them is not academic:
// the engine's own catalogue permits a provider with NO `base_url` (its requirement carries an
// exception for `kind: anthropic`), and a provider whose address nobody declared must not be cleared
// by a guard whose whole job is to know where the calls go.
func TestAnAddressIsThisHostsOnlyWhenItSaysSo(t *testing.T) {
for _, tc := range []struct {
name, baseURL string
want bool
}{
{"a loopback address", "http://127.0.0.1:11434/v1", true},
{"loopback by name", "http://localhost:11434/v1", true},
{"IPv6 loopback", "http://[::1]:11434/v1", true},
{"a vendor", "https://api.deepseek.com/v1", false},
{"a private address that is not this host", "http://10.0.0.7:11434/v1", false},
{"no address at all", "", false},
{"an address that does not parse", "http://[::1", false},
{"a host that is neither name nor address", "http://%zz/v1", false},
} {
t.Run(tc.name, func(t *testing.T) {
if got := onThisHost(tc.baseURL); got != tc.want {
t.Fatalf("onThisHost(%q) = %v, want %v: clearing an address that is not this host is how "+
"a battery comes to spend a reader's credit", tc.baseURL, got, tc.want)
}
})
}
}
func writeFile(t *testing.T, path, body string) {
t.Helper()
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
t.Fatal(err)
}
}