857 lines
37 KiB
Go
857 lines
37 KiB
Go
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// content_labels_test.go covers the generic content-label mechanism (D39.25/D39.26): a book declares
|
|
// content_labels (DATA), a provider/model declares accepts_labels (DATA), the run config declares an
|
|
// ORDERED content_policy registry, and the loader RESOLVES routing before validating it.
|
|
//
|
|
// Every case here uses SYNTHETIC label values ("restricted-synth", "second-synth"). That is the
|
|
// mechanical generality proof the pack owes: if any Go identifier or branch knew the value "adult",
|
|
// these tests could not pass — the engine only ever compares sets it was handed.
|
|
//
|
|
// Reverting resolveContentRouting/checkContentRouting makes the routing cases below stop resolving and
|
|
// the refusal cases stop reporting.
|
|
|
|
// labelFixture writes a models.yaml + pipeline.yaml pair and returns a loader for the pipeline under an
|
|
// arbitrary label set. The models catalog is deliberately small: one endpoint that accepts a synthetic
|
|
// label, one that accepts nothing, one that accepts both synthetic labels.
|
|
func labelFixture(t *testing.T, pipelineBody string) (*Models, func(labels []string) (*Pipeline, error)) {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
promptPath := filepath.Join(dir, "p.md")
|
|
if err := os.WriteFile(promptPath, []byte("sys\n---USER---\n{{text}}"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
modelsPath := filepath.Join(dir, "models.yaml")
|
|
body := fmt.Sprintf(`
|
|
prices_checked: %q
|
|
default_model: plain-model
|
|
providers:
|
|
plain:
|
|
kind: openai
|
|
base_url: http://plain
|
|
capable:
|
|
kind: openai
|
|
base_url: http://capable
|
|
accepts_labels: [restricted-synth, second-synth]
|
|
narrowed:
|
|
kind: openai
|
|
base_url: http://narrowed
|
|
accepts_labels: [restricted-synth]
|
|
models:
|
|
plain-model: { provider: plain, price: { input_per_m: 1, output_per_m: 2 } }
|
|
capable-model: { provider: capable, price: { input_per_m: 1, output_per_m: 2 } }
|
|
capable-hop: { provider: capable, price: { input_per_m: 1, output_per_m: 2 } }
|
|
second-model: { provider: capable, price: { input_per_m: 1, output_per_m: 2 } }
|
|
narrowed-model: { provider: narrowed, price: { input_per_m: 1, output_per_m: 2 } }
|
|
no-labels-model:
|
|
provider: capable
|
|
price: { input_per_m: 1, output_per_m: 2 }
|
|
accepts_labels: []
|
|
`, time.Now().UTC().Format("2006-01-02"))
|
|
if err := os.WriteFile(modelsPath, []byte(body), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
models, err := LoadModels(modelsPath)
|
|
if err != nil {
|
|
t.Fatalf("models load: %v", err)
|
|
}
|
|
pipePath := filepath.Join(dir, "pipe.yaml")
|
|
if err := os.WriteFile(pipePath, []byte(strings.ReplaceAll(pipelineBody, "@PROMPT@", promptPath)), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return models, func(labels []string) (*Pipeline, error) { return LoadPipeline(pipePath, models, "zh-ru", labels) }
|
|
}
|
|
|
|
// basePipeline is a two-stage C1 config whose draft stage carries a label route and a hop.
|
|
const basePipeline = `core: C1
|
|
version: 1
|
|
defaults: { max_output_ratio: 2, min_max_tokens: 512 }
|
|
content_policy:
|
|
- { label: restricted-synth, action: route, chain: restricted }
|
|
stages:
|
|
- name: draft
|
|
role: translator
|
|
model: plain-model
|
|
prompt_override: @PROMPT@
|
|
prompt_version: v
|
|
escalate_to: capable-hop
|
|
label_models: { restricted-synth: capable-model }
|
|
- name: edit
|
|
role: editor
|
|
model: plain-model
|
|
prompt_override: @PROMPT@
|
|
prompt_version: v
|
|
label_models: { restricted-synth: second-model }
|
|
escalation:
|
|
chains:
|
|
# The head DIFFERS from the draft stage's escalate_to on purpose: otherwise "the label's chain
|
|
# substitutes escalate_to" is an identity and a revert of that line would pass unnoticed.
|
|
restricted: [second-model]
|
|
default: [plain-model]
|
|
budget_usd: 1.0
|
|
`
|
|
|
|
// TestContentRoutingResolvesPerStage pins the resolve half: with no labels the resolved models are the
|
|
// configured ones byte for byte (the unlabelled path must not move), and with the label active each
|
|
// stage resolves to its label model while the hop comes from the label's chain HEAD.
|
|
func TestContentRoutingResolvesPerStage(t *testing.T) {
|
|
_, load := labelFixture(t, basePipeline)
|
|
|
|
p, err := load(nil)
|
|
if err != nil {
|
|
t.Fatalf("unlabelled load: %v", err)
|
|
}
|
|
for _, st := range p.Stages {
|
|
if st.ResolvedModel != st.Model || st.ResolvedHop != st.EscalateTo {
|
|
t.Errorf("unlabelled stage %q resolved to (%q,%q), want the configured (%q,%q) — the no-label path must be identical",
|
|
st.Name, st.ResolvedModel, st.ResolvedHop, st.Model, st.EscalateTo)
|
|
}
|
|
}
|
|
if err := p.ContentRoutingError(); err != nil {
|
|
t.Errorf("unlabelled book must have no routing problems, got %v", err)
|
|
}
|
|
|
|
p, err = load([]string{"restricted-synth"})
|
|
if err != nil {
|
|
t.Fatalf("labelled load: %v", err)
|
|
}
|
|
if err := p.ContentRoutingError(); err != nil {
|
|
t.Fatalf("labelled book must load cleanly (every reachable model accepts the label), got %v", err)
|
|
}
|
|
if got := p.Stages[0].ResolvedModel; got != "capable-model" {
|
|
t.Errorf("draft resolved model = %q, want capable-model (label_models)", got)
|
|
}
|
|
if got := p.Stages[0].ResolvedHop; got != "second-model" {
|
|
t.Errorf("draft resolved hop = %q, want second-model — the chain head SUBSTITUTES the configured escalate_to (capable-hop)", got)
|
|
}
|
|
if got := p.Stages[1].ResolvedModel; got != "second-model" {
|
|
t.Errorf("edit resolved model = %q, want second-model", got)
|
|
}
|
|
// D12 editor-pinned: a stage with no escalate_to gets no hop even under a label.
|
|
if got := p.Stages[1].ResolvedHop; got != "" {
|
|
t.Errorf("edit resolved hop = %q, want empty (editor is pinned — a refusal there is terminal)", got)
|
|
}
|
|
// Reachability is label-dependent: the replaced escalate_to and the unreferenced default chain are
|
|
// NOT reachable for this book.
|
|
reach := strings.Join(p.ReachableModels(), ",")
|
|
for _, want := range []string{"capable-model", "second-model"} {
|
|
if !strings.Contains(reach, want) {
|
|
t.Errorf("reachable set %q must contain %q", reach, want)
|
|
}
|
|
}
|
|
if strings.Contains(reach, "capable-hop") {
|
|
t.Errorf("reachable set %q must NOT contain capable-hop — the label's chain substituted it", reach)
|
|
}
|
|
if strings.Contains(reach, "plain-model") {
|
|
t.Errorf("reachable set %q must NOT contain plain-model — the label replaced it (a capability demand on a model the run never calls would refuse a valid book)", reach)
|
|
}
|
|
}
|
|
|
|
// TestContentRoutingRefusals covers the book-DEPENDENT refusals: they are reported through
|
|
// ContentProblems (fatal for the money path, a warning for the $0 read paths) rather than failing the
|
|
// load outright.
|
|
func TestContentRoutingRefusals(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
pipeline string
|
|
labels []string
|
|
want string
|
|
}{
|
|
{
|
|
name: "label with no policy entry",
|
|
pipeline: basePipeline,
|
|
labels: []string{"unregistered-synth"},
|
|
want: "no content_policy entry",
|
|
},
|
|
{
|
|
name: "model that may not receive the label",
|
|
pipeline: strings.Replace(basePipeline,
|
|
"label_models: { restricted-synth: capable-model }",
|
|
"label_models: { restricted-synth: no-labels-model }", 1),
|
|
labels: []string{"restricted-synth"},
|
|
want: "may not receive content label",
|
|
},
|
|
{
|
|
name: "route label with no escalation budget",
|
|
pipeline: strings.Replace(basePipeline,
|
|
" budget_usd: 1.0", " budget_usd: 0", 1),
|
|
labels: []string{"restricted-synth"},
|
|
want: "no hop can ever fire",
|
|
},
|
|
{
|
|
// A terminal policy takes no per-stage routes either (they would be meaningless), so the fixture
|
|
// drops them together with the chain — that pairing is itself checked by the shape test below.
|
|
name: "terminal policy refuses the book",
|
|
pipeline: strings.NewReplacer(
|
|
" - { label: restricted-synth, action: route, chain: restricted }",
|
|
" - { label: restricted-synth, action: terminal }",
|
|
" label_models: { restricted-synth: capable-model }\n", "",
|
|
" label_models: { restricted-synth: second-model }\n", "",
|
|
).Replace(basePipeline),
|
|
labels: []string{"restricted-synth"},
|
|
want: "action=terminal",
|
|
},
|
|
{
|
|
// A route policy that re-routes NOTHING is inert: the labelled book would quietly run on the
|
|
// ordinary models with every check green.
|
|
name: "route policy that routes nothing",
|
|
pipeline: strings.NewReplacer(
|
|
" label_models: { restricted-synth: capable-model }\n", "",
|
|
" label_models: { restricted-synth: second-model }\n", "",
|
|
).Replace(basePipeline),
|
|
labels: []string{"restricted-synth"},
|
|
want: "routes nothing",
|
|
},
|
|
{
|
|
// Two active route labels with DIFFERENT heads: only one hop can execute, so the second remedy
|
|
// could never fire — a hop set silently truncated, refused with the multi-hop wording.
|
|
name: "two active labels select two different fallback heads",
|
|
pipeline: strings.NewReplacer(
|
|
" - { label: restricted-synth, action: route, chain: restricted }",
|
|
" - { label: restricted-synth, action: route, chain: restricted }\n - { label: second-synth, action: route, chain: other }",
|
|
" default: [plain-model]", " default: [plain-model]\n other: [capable-hop]",
|
|
).Replace(basePipeline),
|
|
labels: []string{"restricted-synth", "second-synth"},
|
|
want: "MULTI-HOP IS NOT BUILT",
|
|
},
|
|
{
|
|
name: "label hop equals the label primary",
|
|
pipeline: strings.Replace(basePipeline,
|
|
"label_models: { restricted-synth: capable-model }",
|
|
"label_models: { restricted-synth: second-model }", 1),
|
|
labels: []string{"restricted-synth"},
|
|
want: "same fallback hop",
|
|
},
|
|
}
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
_, load := labelFixture(t, c.pipeline)
|
|
p, err := load(c.labels)
|
|
if err != nil {
|
|
t.Fatalf("load must SUCCEED (book-dependent refusals are collected, not returned): %v", err)
|
|
}
|
|
rerr := p.ContentRoutingError()
|
|
if rerr == nil || !strings.Contains(rerr.Error(), c.want) {
|
|
t.Fatalf("want a routing problem mentioning %q, got %v", c.want, rerr)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestContentPolicyShapeIsFatal covers the config-SHAPE errors, which fail the load on EVERY path (a
|
|
// malformed registry is broken regardless of which book reads it) — including the ratified refusal to
|
|
// silently truncate a multi-member chain.
|
|
func TestContentPolicyShapeIsFatal(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
pipeline string
|
|
want string
|
|
}{
|
|
{"multi-member chain", strings.Replace(basePipeline, "restricted: [second-model]", "restricted: [second-model, capable-hop]", 1), "MULTI-HOP IS NOT BUILT"},
|
|
{"unknown action", strings.Replace(basePipeline, "action: route, chain: restricted", "action: quarantine, chain: restricted", 1), "action must be route|allow|terminal"},
|
|
{"route without chain", strings.Replace(basePipeline, "action: route, chain: restricted", "action: route", 1), "requires `chain:`"},
|
|
{"chain not defined", strings.Replace(basePipeline, "action: route, chain: restricted", "action: route, chain: nosuch", 1), "not defined in escalation.chains"},
|
|
{"duplicate registry label", strings.Replace(basePipeline,
|
|
" - { label: restricted-synth, action: route, chain: restricted }",
|
|
" - { label: restricted-synth, action: route, chain: restricted }\n - { label: restricted-synth, action: terminal }", 1), "duplicate label"},
|
|
{"label_models model undefined", strings.Replace(basePipeline, "restricted-synth: capable-model", "restricted-synth: ghost-model", 1), "is not defined in models.yaml"},
|
|
{"label_models label unregistered", strings.Replace(basePipeline, "restricted-synth: second-model", "ghost-synth: second-model", 1), "no content_policy entry"},
|
|
{"retired channel key", strings.Replace(basePipeline, " escalate_to: capable-hop", " escalate_to: capable-hop\n channel: adult", 1), "`channel:` is retired"},
|
|
}
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
_, load := labelFixture(t, c.pipeline)
|
|
// Fatal on every path: asserted WITHOUT labels, so it cannot be mistaken for a book-dependent refusal.
|
|
if _, err := load(nil); err == nil || !strings.Contains(err.Error(), c.want) {
|
|
t.Fatalf("want a load error mentioning %q, got %v", c.want, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestContentPolicyOrderDecidesPrecedence pins the reason the registry is an ordered LIST and not a map:
|
|
// when two of the book's labels claim one stage, the FIRST entry wins — deterministically, written down
|
|
// in the data file (a map would resolve by Go's randomised iteration order).
|
|
func TestContentPolicyOrderDecidesPrecedence(t *testing.T) {
|
|
two := strings.Replace(basePipeline,
|
|
" - { label: restricted-synth, action: route, chain: restricted }",
|
|
" - { label: restricted-synth, action: route, chain: restricted }\n - { label: second-synth, action: route, chain: restricted }", 1)
|
|
two = strings.Replace(two,
|
|
"label_models: { restricted-synth: capable-model }",
|
|
"label_models: { restricted-synth: capable-model, second-synth: second-model }", 1)
|
|
_, load := labelFixture(t, two)
|
|
|
|
p, err := load([]string{"restricted-synth", "second-synth"})
|
|
if err != nil {
|
|
t.Fatalf("load: %v", err)
|
|
}
|
|
if err := p.ContentRoutingError(); err != nil {
|
|
t.Fatalf("both labels are accepted by the capable provider, so this must load: %v", err)
|
|
}
|
|
if got := p.Stages[0].ResolvedModel; got != "capable-model" {
|
|
t.Errorf("draft resolved to %q, want capable-model — the FIRST registry entry claiming the stage wins", got)
|
|
}
|
|
// Reversing the registry order must reverse the winner: precedence lives in the data, not in Go.
|
|
reversed := strings.Replace(two,
|
|
" - { label: restricted-synth, action: route, chain: restricted }\n - { label: second-synth, action: route, chain: restricted }",
|
|
" - { label: second-synth, action: route, chain: restricted }\n - { label: restricted-synth, action: route, chain: restricted }", 1)
|
|
_, loadRev := labelFixture(t, reversed)
|
|
pr, err := loadRev([]string{"restricted-synth", "second-synth"})
|
|
if err != nil {
|
|
t.Fatalf("reversed load: %v", err)
|
|
}
|
|
if got := pr.Stages[0].ResolvedModel; got != "second-model" {
|
|
t.Errorf("with the reversed registry the draft resolved to %q, want second-model", got)
|
|
}
|
|
}
|
|
|
|
// TestSecondLabelIsDataOnly is the generality test D39.25 demands in mechanical form: adding a SECOND
|
|
// label needs no Go change — one registry entry, one accepts_labels value, one label_models entry, all
|
|
// data. If any of that had to be taught to the engine, this test could not pass.
|
|
func TestSecondLabelIsDataOnly(t *testing.T) {
|
|
body := strings.Replace(basePipeline,
|
|
" - { label: restricted-synth, action: route, chain: restricted }",
|
|
" - { label: second-synth, action: route, chain: restricted }", 1)
|
|
body = strings.Replace(body,
|
|
"label_models: { restricted-synth: capable-model }",
|
|
"label_models: { second-synth: capable-model }", 1)
|
|
body = strings.Replace(body,
|
|
"label_models: { restricted-synth: second-model }",
|
|
"label_models: { second-synth: second-model }", 1)
|
|
_, load := labelFixture(t, body)
|
|
p, err := load([]string{"second-synth"})
|
|
if err != nil {
|
|
t.Fatalf("second label load: %v", err)
|
|
}
|
|
if err := p.ContentRoutingError(); err != nil {
|
|
t.Fatalf("second label must route on data alone: %v", err)
|
|
}
|
|
if p.Stages[0].ResolvedModel != "capable-model" || p.Stages[0].ResolvedHop != "second-model" {
|
|
t.Errorf("second label resolved (%q,%q), want (capable-model,second-model)", p.Stages[0].ResolvedModel, p.Stages[0].ResolvedHop)
|
|
}
|
|
}
|
|
|
|
// TestAcceptsLabelsNarrowingAndWidening pins the pointer semantics that keep the permission fail-closed:
|
|
// nil inherits the provider's set, a pointer to an EMPTY list subtracts everything, and a model may
|
|
// never widen beyond its provider (a permission is a statement about the endpoint receiving the bytes).
|
|
func TestAcceptsLabelsNarrowingAndWidening(t *testing.T) {
|
|
models, _ := labelFixture(t, basePipeline)
|
|
if got := models.AcceptsLabels("capable-model"); len(got) != 2 {
|
|
t.Errorf("capable-model must INHERIT its provider's two labels, got %v", got)
|
|
}
|
|
if got := models.AcceptsLabels("no-labels-model"); len(got) != 0 {
|
|
t.Errorf("an explicit empty accepts_labels must SUBTRACT the provider's set, got %v", got)
|
|
}
|
|
if got := models.MissingLabels("no-labels-model", []string{"restricted-synth"}); len(got) != 1 {
|
|
t.Errorf("MissingLabels on a subtracted model = %v, want the label reported missing", got)
|
|
}
|
|
if got := models.MissingLabels("ghost", []string{"restricted-synth"}); len(got) != 1 {
|
|
t.Errorf("an UNKNOWN model must accept nothing (fail closed), got %v", got)
|
|
}
|
|
if got := models.MissingLabels("capable-model", nil); got != nil {
|
|
t.Errorf("no labels wanted ⇒ nothing missing, got %v", got)
|
|
}
|
|
|
|
// Widening past a SILENT provider: every provider row in the shipping catalog declares nothing today,
|
|
// so this is the natural shape of the mistake — one line on the model row (the translator/editor choice
|
|
// IS per model) granting a label the endpoint never got. A nil provider set means "accepts nothing".
|
|
silent := filepath.Join(t.TempDir(), "models.yaml")
|
|
if err := os.WriteFile(silent, []byte(fmt.Sprintf(`
|
|
prices_checked: %q
|
|
default_model: m
|
|
providers:
|
|
p: { kind: openai, base_url: http://p }
|
|
models:
|
|
m:
|
|
provider: p
|
|
price: { input_per_m: 1, output_per_m: 2 }
|
|
accepts_labels: [restricted-synth]
|
|
`, time.Now().UTC().Format("2006-01-02"))), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := LoadModels(silent); err == nil || !strings.Contains(err.Error(), "may only NARROW") {
|
|
t.Fatalf("a model may not grant a label its (silent) provider does not accept, got %v", err)
|
|
}
|
|
|
|
// Widening: a model naming a label its provider does not accept is a load error.
|
|
dir := t.TempDir()
|
|
path := filepath.Join(dir, "models.yaml")
|
|
body := fmt.Sprintf(`
|
|
prices_checked: %q
|
|
default_model: m
|
|
providers:
|
|
p:
|
|
kind: openai
|
|
base_url: http://p
|
|
accepts_labels: [restricted-synth]
|
|
models:
|
|
m:
|
|
provider: p
|
|
price: { input_per_m: 1, output_per_m: 2 }
|
|
accepts_labels: [restricted-synth, wider-synth]
|
|
`, time.Now().UTC().Format("2006-01-02"))
|
|
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := LoadModels(path); err == nil || !strings.Contains(err.Error(), "may only NARROW") {
|
|
t.Fatalf("a model widening its provider's permissions must fail loud, got %v", err)
|
|
}
|
|
}
|
|
|
|
// TestRetiredPermissiveFlagIsLoud pins the retirement of the boolean the label set replaced. Presence at
|
|
// ANY value is the error: `permissive: false` was also a statement, and silently accepting it would let
|
|
// an author believe the old gate still guards.
|
|
func TestRetiredPermissiveFlagIsLoud(t *testing.T) {
|
|
for _, val := range []string{"true", "false"} {
|
|
dir := t.TempDir()
|
|
path := filepath.Join(dir, "models.yaml")
|
|
body := fmt.Sprintf(`
|
|
prices_checked: %q
|
|
default_model: m
|
|
providers:
|
|
p:
|
|
kind: openai
|
|
base_url: http://p
|
|
permissive: %s
|
|
models:
|
|
m: { provider: p, price: { input_per_m: 1, output_per_m: 2 } }
|
|
`, time.Now().UTC().Format("2006-01-02"), val)
|
|
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := LoadModels(path); err == nil || !strings.Contains(err.Error(), "`permissive:` is retired") {
|
|
t.Errorf("permissive: %s must fail loud naming accepts_labels, got %v", val, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestCheckKeysIsLabelAware pins that a labelled book demands the keys of the models it actually calls,
|
|
// not of a chain its labels never select (which would block a valid labelled run on an unrelated key).
|
|
func TestCheckKeysIsLabelAware(t *testing.T) {
|
|
dir := t.TempDir()
|
|
promptPath := filepath.Join(dir, "p.md")
|
|
if err := os.WriteFile(promptPath, []byte("sys\n---USER---\n{{text}}"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
modelsPath := filepath.Join(dir, "models.yaml")
|
|
body := fmt.Sprintf(`
|
|
prices_checked: %q
|
|
default_model: keyed-model
|
|
providers:
|
|
keyed:
|
|
kind: openai
|
|
base_url: http://keyed
|
|
api_key_env: TM_TEST_LABEL_KEY_PRESENT
|
|
accepts_labels: [restricted-synth]
|
|
unkeyed:
|
|
kind: openai
|
|
base_url: http://unkeyed
|
|
api_key_env: TM_TEST_LABEL_KEY_ABSENT
|
|
models:
|
|
keyed-model: { provider: keyed, price: { input_per_m: 1, output_per_m: 2 } }
|
|
keyed-hop: { provider: keyed, price: { input_per_m: 1, output_per_m: 2 } }
|
|
unkeyed-model: { provider: unkeyed, price: { input_per_m: 1, output_per_m: 2 } }
|
|
`, time.Now().UTC().Format("2006-01-02"))
|
|
if err := os.WriteFile(modelsPath, []byte(body), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
models, err := LoadModels(modelsPath)
|
|
if err != nil {
|
|
t.Fatalf("models: %v", err)
|
|
}
|
|
pipeBody := fmt.Sprintf(`core: C1
|
|
version: 1
|
|
defaults: { max_output_ratio: 2, min_max_tokens: 512 }
|
|
content_policy:
|
|
- { label: restricted-synth, action: route, chain: restricted }
|
|
stages:
|
|
- name: draft
|
|
role: translator
|
|
model: keyed-model
|
|
prompt_override: %s
|
|
prompt_version: v
|
|
escalate_to: unkeyed-model
|
|
label_models: { restricted-synth: keyed-model }
|
|
escalation:
|
|
chains:
|
|
restricted: [keyed-hop]
|
|
default: [unkeyed-model]
|
|
budget_usd: 1.0
|
|
`, promptPath)
|
|
pipePath := filepath.Join(dir, "pipe.yaml")
|
|
if err := os.WriteFile(pipePath, []byte(pipeBody), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Setenv("TM_TEST_LABEL_KEY_PRESENT", "x")
|
|
t.Setenv("TM_TEST_LABEL_KEY_ABSENT", "") // t.Setenv restores it; os.Unsetenv would leak into later tests
|
|
|
|
// Unlabelled: the historical set — every named chain + escalate_to — so the missing key IS demanded.
|
|
unlabelled, err := LoadPipeline(pipePath, models, "zh-ru", nil)
|
|
if err != nil {
|
|
t.Fatalf("unlabelled load: %v", err)
|
|
}
|
|
if err := models.CheckKeys(unlabelled); err == nil || !strings.Contains(err.Error(), "TM_TEST_LABEL_KEY_ABSENT") {
|
|
t.Fatalf("unlabelled preflight must keep demanding the unreferenced chain's key (historical behaviour), got %v", err)
|
|
}
|
|
// Labelled: the unkeyed model is unreachable (its chain is not selected, its escalate_to is replaced).
|
|
labelled, err := LoadPipeline(pipePath, models, "zh-ru", []string{"restricted-synth"})
|
|
if err != nil {
|
|
t.Fatalf("labelled load: %v", err)
|
|
}
|
|
if err := labelled.ContentRoutingError(); err != nil {
|
|
t.Fatalf("labelled routing must be clean: %v", err)
|
|
}
|
|
if err := models.CheckKeys(labelled); err != nil {
|
|
t.Fatalf("labelled preflight must only demand the keys of reachable models, got %v", err)
|
|
}
|
|
}
|
|
|
|
// TestShippingConfigsUnderLabels loads every SHIPPING pipeline against the real models catalog with a
|
|
// label set, which is the case no prior test covered: the shipping configs declare an `adult` chain but
|
|
// no content_policy, so a labelled book must be REFUSED for the money path with a message an operator
|
|
// can act on — and, critically, the refusal must be the missing POLICY, never a spurious capability
|
|
// demand on a model the labelled run would never call (the self-inconsistency the design panel caught).
|
|
func TestShippingConfigsUnderLabels(t *testing.T) {
|
|
models, err := LoadModels(filepath.Join("..", "..", "configs", "models.yaml"))
|
|
if err != nil {
|
|
t.Fatalf("load shipping models.yaml: %v", err)
|
|
}
|
|
for _, name := range []string{
|
|
"pipeline-c1.yaml", "pipeline-arm-glm.yaml", "pipeline-arm-mistral.yaml",
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
path := filepath.Join("..", "..", "configs", name)
|
|
// Unlabelled: loads clean and resolves to the configured models (the byte-identical path).
|
|
p, err := LoadPipeline(path, models, "zh-ru", nil)
|
|
if err != nil {
|
|
t.Fatalf("unlabelled load must stay clean: %v", err)
|
|
}
|
|
for _, st := range p.Stages {
|
|
if st.ResolvedModel != st.Model || st.ResolvedHop != st.EscalateTo {
|
|
t.Fatalf("stage %q must resolve to its configured models without labels", st.Name)
|
|
}
|
|
}
|
|
if err := p.ContentRoutingError(); err != nil {
|
|
t.Fatalf("unlabelled shipping config must have no routing problems: %v", err)
|
|
}
|
|
// Labelled: the shipping configs carry no registry, so the refusal NAMES the missing policy (the
|
|
// actionable line). Capability complaints about the configured models appear alongside it — they
|
|
// are honest, since with no policy nothing is re-routed — but the chains no policy selects must
|
|
// NOT be dragged in.
|
|
lp, err := LoadPipeline(path, models, "zh-ru", []string{"restricted-synth"})
|
|
if err != nil {
|
|
t.Fatalf("a labelled load must not fail on config SHAPE (the refusal is book-dependent): %v", err)
|
|
}
|
|
rerr := lp.ContentRoutingError()
|
|
if rerr == nil || !strings.Contains(rerr.Error(), "no content_policy entry") {
|
|
t.Fatalf("want the missing-policy refusal, got %v", rerr)
|
|
}
|
|
// The unreferenced `adult` chain and the default chain must NOT be dragged into the invariant.
|
|
if strings.Contains(rerr.Error(), "grok-4.3") || strings.Contains(rerr.Error(), "gemini-3.1-pro-preview") {
|
|
t.Errorf("the refusal must not name models of chains no policy selects: %v", rerr)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestShippingChainsStayOneHop pins the data precondition of the single-hop rule for the chain the owner's
|
|
// vocabulary decision will point a policy at: `chains.adult` must stay one member long, or the policy that
|
|
// references it will not load ("multi-hop is not built"). C2 is excluded — CheckRunnable refuses that core.
|
|
func TestShippingChainsStayOneHop(t *testing.T) {
|
|
models, err := LoadModels(filepath.Join("..", "..", "configs", "models.yaml"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, name := range []string{
|
|
"pipeline-c1.yaml", "pipeline-arm-glm.yaml", "pipeline-arm-mistral.yaml",
|
|
} {
|
|
p, err := LoadPipeline(filepath.Join("..", "..", "configs", name), models, "zh-ru", nil)
|
|
if err != nil {
|
|
t.Fatalf("%s: %v", name, err)
|
|
}
|
|
// Every chain EXCEPT the historical multi-hop "default" (which no policy references and which the
|
|
// pack deliberately left dead). No content value appears here: naming one would make the owner's
|
|
// pending vocabulary answer — a pure data edit — turn this test red.
|
|
for chain, members := range p.Escal.Chains {
|
|
if chain == "default" {
|
|
continue
|
|
}
|
|
if len(members) != 1 {
|
|
t.Errorf("%s: chain %q has %d members — a route policy can execute exactly one hop (D39.26 point 3), so a policy pointed at it would not load", name, chain, len(members))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestNoPrivilegedLabelValue is the tripwire item 14 asks for in the direction the synthetic-value tests
|
|
// cannot cover: they prove the engine works for values it was never taught, but not that no value is
|
|
// PRIVILEGED. Here a book declares "adult" against a registry that only knows the synthetic label — the
|
|
// engine must treat it like any unknown label (refuse, route nothing). Teaching Go to recognise the value
|
|
// (a special case in the unregistered-label loop, a synthesised policy) makes this RED.
|
|
func TestNoPrivilegedLabelValue(t *testing.T) {
|
|
_, load := labelFixture(t, basePipeline)
|
|
p, err := load([]string{"adult"})
|
|
if err != nil {
|
|
t.Fatalf("load: %v", err)
|
|
}
|
|
rerr := p.ContentRoutingError()
|
|
if rerr == nil || !strings.Contains(rerr.Error(), "no content_policy entry") {
|
|
t.Fatalf("a value the registry never declared must be refused like any other, got %v", rerr)
|
|
}
|
|
for _, st := range p.Stages {
|
|
if st.ResolvedModel != st.Model || st.ResolvedHop != st.EscalateTo {
|
|
t.Errorf("stage %q must NOT route on an unregistered value: (%q,%q) vs configured (%q,%q)",
|
|
st.Name, st.ResolvedModel, st.ResolvedHop, st.Model, st.EscalateTo)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestLabelSpellingIsCanonicalEverywhere pins the contract that keeps the four data keys from drifting: a
|
|
// book's labels are NORMALISED (trim + lower + dedupe + sort), and every other place a label is AUTHORED
|
|
// must be written that way or the load says so. Without this a provider spelled "Restricted-Synth" would
|
|
// fail closed but tell the operator the book's label is not accepted while the row visibly contains it.
|
|
func TestLabelSpellingIsCanonicalEverywhere(t *testing.T) {
|
|
dir := t.TempDir()
|
|
write := func(name, body string) string {
|
|
p := filepath.Join(dir, name)
|
|
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return p
|
|
}
|
|
today := time.Now().UTC().Format("2006-01-02")
|
|
// accepts_labels must be canonical.
|
|
bad := write("models-bad.yaml", fmt.Sprintf(`
|
|
prices_checked: %q
|
|
default_model: m
|
|
providers:
|
|
p: { kind: openai, base_url: http://p, accepts_labels: ["Restricted-Synth"] }
|
|
models:
|
|
m: { provider: p, price: { input_per_m: 1, output_per_m: 2 } }
|
|
`, today))
|
|
if _, err := LoadModels(bad); err == nil || !strings.Contains(err.Error(), "lower-case") {
|
|
t.Errorf("a non-canonical accepts_labels entry must fail loud, got %v", err)
|
|
}
|
|
// The book side NORMALISES instead: three spellings of one label collapse to one entry.
|
|
bookPath := write("book.yaml", `
|
|
book_id: b
|
|
title: t
|
|
source_lang: zh
|
|
target_lang: ru
|
|
content_labels: [" Restricted-Synth ", restricted-synth, RESTRICTED-SYNTH]
|
|
pipeline: pipe.yaml
|
|
models: models.yaml
|
|
source_file: src.txt
|
|
ceilings: { book_usd: 1 }
|
|
`)
|
|
write("src.txt", "текст")
|
|
write("pipe.yaml", "core: C1\nversion: 1\n")
|
|
write("models.yaml", fmt.Sprintf("prices_checked: %q\ndefault_model: m\nproviders:\n p: { kind: openai, base_url: http://p }\nmodels:\n m: { provider: p, price: { input_per_m: 1, output_per_m: 2 } }\n", today))
|
|
b, err := LoadBook(bookPath)
|
|
if err != nil {
|
|
t.Fatalf("book load: %v", err)
|
|
}
|
|
if len(b.ContentLabels) != 1 || b.ContentLabels[0] != "restricted-synth" {
|
|
t.Errorf("content_labels must normalise to one canonical entry, got %v", b.ContentLabels)
|
|
}
|
|
}
|
|
|
|
// TestAcceptsLabelSetShape covers the entry-level validation of the capability list (empty string,
|
|
// duplicate) — small, but it is the only thing standing between a typo and a permission nobody meant.
|
|
func TestAcceptsLabelSetShape(t *testing.T) {
|
|
for _, c := range []struct{ name, list, want string }{
|
|
{"empty entry", `["", restricted-synth]`, "is empty"},
|
|
{"duplicate", `[restricted-synth, restricted-synth]`, "duplicate"},
|
|
} {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
dir := t.TempDir()
|
|
p := filepath.Join(dir, "models.yaml")
|
|
body := fmt.Sprintf(`
|
|
prices_checked: %q
|
|
default_model: m
|
|
providers:
|
|
p: { kind: openai, base_url: http://p, accepts_labels: %s }
|
|
models:
|
|
m: { provider: p, price: { input_per_m: 1, output_per_m: 2 } }
|
|
`, time.Now().UTC().Format("2006-01-02"), c.list)
|
|
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := LoadModels(p); err == nil || !strings.Contains(err.Error(), c.want) {
|
|
t.Fatalf("want an error mentioning %q, got %v", c.want, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestAllowActionAssertsWithoutRouting covers the third policy shape: `allow` applies the capability
|
|
// invariant and changes NO route. It is the shape a label needs when the providers are already correct —
|
|
// with only route/terminal such a label was inexpressible (route demands a chain, a budget and a
|
|
// per-stage model it has no use for; terminal refuses the book), which is why a two-label vocabulary
|
|
// where one label re-routes and the other only asserts could not be written as DATA at all.
|
|
func TestAllowActionAssertsWithoutRouting(t *testing.T) {
|
|
// An allow-only registry: no chain, no budget, no label_models anywhere.
|
|
allowOnly := `core: C1
|
|
version: 1
|
|
defaults: { max_output_ratio: 2, min_max_tokens: 512 }
|
|
content_policy:
|
|
- { label: restricted-synth, action: allow }
|
|
stages:
|
|
- name: draft
|
|
role: translator
|
|
model: capable-model
|
|
prompt_override: @PROMPT@
|
|
prompt_version: v
|
|
escalate_to: capable-hop
|
|
- name: edit
|
|
role: editor
|
|
model: second-model
|
|
prompt_override: @PROMPT@
|
|
prompt_version: v
|
|
`
|
|
_, load := labelFixture(t, allowOnly)
|
|
p, err := load([]string{"restricted-synth"})
|
|
if err != nil {
|
|
t.Fatalf("an allow-only policy must need no chain and no budget: %v", err)
|
|
}
|
|
if err := p.ContentRoutingError(); err != nil {
|
|
t.Fatalf("every reachable model accepts the label, so this must be runnable: %v", err)
|
|
}
|
|
// The route is untouched — that is the whole semantic.
|
|
for _, st := range p.Stages {
|
|
if st.ResolvedModel != st.Model || st.ResolvedHop != st.EscalateTo {
|
|
t.Errorf("stage %q must keep its configured route under action=allow: (%q,%q) vs (%q,%q)",
|
|
st.Name, st.ResolvedModel, st.ResolvedHop, st.Model, st.EscalateTo)
|
|
}
|
|
}
|
|
// …and the INVARIANT still bites: an endpoint that may not receive the label is refused, which is the
|
|
// only thing an allow policy does.
|
|
incapable := strings.Replace(allowOnly, "model: capable-model", "model: plain-model", 1)
|
|
_, loadBad := labelFixture(t, incapable)
|
|
pb, err := loadBad([]string{"restricted-synth"})
|
|
if err != nil {
|
|
t.Fatalf("load: %v", err)
|
|
}
|
|
rerr := pb.ContentRoutingError()
|
|
if rerr == nil || !strings.Contains(rerr.Error(), "may not receive content label") {
|
|
t.Fatalf("action=allow must still refuse an incapable endpoint — the invariant IS the policy, got %v", rerr)
|
|
}
|
|
}
|
|
|
|
// TestAllowActionRejectsRoutingKnobs pins that an allow policy refuses the knobs it cannot honour, rather
|
|
// than ignoring them: an author who wrote a chain or a per-stage model meant something the engine will
|
|
// not do, and silently dropping it is how a config comes to lie about its own routing.
|
|
func TestAllowActionRejectsRoutingKnobs(t *testing.T) {
|
|
base := `core: C1
|
|
version: 1
|
|
defaults: { max_output_ratio: 2, min_max_tokens: 512 }
|
|
content_policy:
|
|
- { label: restricted-synth, action: allow%s }
|
|
stages:
|
|
- name: draft
|
|
role: translator
|
|
model: capable-model
|
|
prompt_override: @PROMPT@
|
|
prompt_version: v
|
|
escalate_to: capable-hop%s
|
|
escalation:
|
|
chains: { restricted: [second-model] }
|
|
budget_usd: 1.0
|
|
`
|
|
cases := []struct{ name, policyTail, stageTail, want string }{
|
|
{"chain on an allow policy", ", chain: restricted", "", "takes no chain"},
|
|
{"label_models for an allow label", "", "\n label_models: { restricted-synth: second-model }", "is meaningless"},
|
|
}
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
_, load := labelFixture(t, fmt.Sprintf(base, c.policyTail, c.stageTail))
|
|
if _, err := load(nil); err == nil || !strings.Contains(err.Error(), c.want) {
|
|
t.Fatalf("want a load error mentioning %q, got %v", c.want, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestRatifiedLabelPairIsDataOnly is the delta's acceptance shape, written as the owner ratified it: TWO
|
|
// labels, one that only ASSERTS (its providers are already correct, so no model moves) and one that
|
|
// RE-ROUTES (its endpoints differ). Both values here are synthetic — the point is that the SHAPE needs no
|
|
// Go change, so whichever vocabulary the data ends up carrying, the mechanism already holds it.
|
|
func TestRatifiedLabelPairIsDataOnly(t *testing.T) {
|
|
pair := `core: C1
|
|
version: 1
|
|
defaults: { max_output_ratio: 2, min_max_tokens: 512 }
|
|
content_policy:
|
|
- { label: restricted-synth, action: allow }
|
|
- { label: second-synth, action: route, chain: rerouted }
|
|
stages:
|
|
- name: draft
|
|
role: translator
|
|
model: capable-model
|
|
prompt_override: @PROMPT@
|
|
prompt_version: v
|
|
escalate_to: capable-hop
|
|
label_models: { second-synth: narrowed-model }
|
|
- name: edit
|
|
role: editor
|
|
model: capable-model
|
|
prompt_override: @PROMPT@
|
|
prompt_version: v
|
|
escalation:
|
|
chains: { rerouted: [second-model] }
|
|
budget_usd: 1.0
|
|
`
|
|
_, load := labelFixture(t, pair)
|
|
|
|
// (a) the asserting label alone: nothing moves, no chain/budget needed by it.
|
|
only, err := load([]string{"restricted-synth"})
|
|
if err != nil {
|
|
t.Fatalf("load: %v", err)
|
|
}
|
|
if err := only.ContentRoutingError(); err != nil {
|
|
t.Fatalf("restricted-synth must be runnable on capable endpoints: %v", err)
|
|
}
|
|
if only.Stages[0].ResolvedModel != "capable-model" || only.Stages[0].ResolvedHop != "capable-hop" {
|
|
t.Errorf("the asserting label must not move the route, got (%q,%q)", only.Stages[0].ResolvedModel, only.Stages[0].ResolvedHop)
|
|
}
|
|
|
|
// (b) BOTH labels: the re-routing one moves the draft leg, and the invariant is applied to the union —
|
|
// narrowed-model accepts only one of the two synthetic labels, so it must be refused here.
|
|
both, err := load([]string{"restricted-synth", "second-synth"})
|
|
if err != nil {
|
|
t.Fatalf("load: %v", err)
|
|
}
|
|
rerr := both.ContentRoutingError()
|
|
if rerr == nil || !strings.Contains(rerr.Error(), "narrowed-model") {
|
|
t.Fatalf("the invariant must be applied to the UNION of the book's labels (narrowed-model accepts only one), got %v", rerr)
|
|
}
|
|
if both.Stages[0].ResolvedModel != "narrowed-model" {
|
|
t.Errorf("the re-routing label must still resolve its stage, got %q", both.Stages[0].ResolvedModel)
|
|
}
|
|
|
|
// (c) the pair on endpoints that accept BOTH: runnable, one leg re-routed, the other untouched.
|
|
ok := strings.Replace(pair, "label_models: { second-synth: narrowed-model }", "label_models: { second-synth: second-model }", 1)
|
|
ok = strings.Replace(ok, "chains: { rerouted: [second-model] }", "chains: { rerouted: [capable-hop] }", 1)
|
|
_, loadOK := labelFixture(t, ok)
|
|
p, err := loadOK([]string{"restricted-synth", "second-synth"})
|
|
if err != nil {
|
|
t.Fatalf("load: %v", err)
|
|
}
|
|
if err := p.ContentRoutingError(); err != nil {
|
|
t.Fatalf("the ratified PAIR shape must be runnable on capable endpoints with zero Go changes: %v", err)
|
|
}
|
|
if p.Stages[0].ResolvedModel != "second-model" || p.Stages[0].ResolvedHop != "capable-hop" {
|
|
t.Errorf("draft resolved (%q,%q), want (second-model,capable-hop)", p.Stages[0].ResolvedModel, p.Stages[0].ResolvedHop)
|
|
}
|
|
if p.Stages[1].ResolvedModel != "capable-model" {
|
|
t.Errorf("the edit stage carries no route for either label, so it stays configured, got %q", p.Stages[1].ResolvedModel)
|
|
}
|
|
}
|