351 lines
17 KiB
Go
351 lines
17 KiB
Go
package config
|
||
|
||
import (
|
||
"path/filepath"
|
||
"strings"
|
||
"testing"
|
||
)
|
||
|
||
// bank_contour_test.go: backlog row 140 — the shipping config translated «naked». The banknote channel,
|
||
// the terminologist and the classifier all existed and all lived only in a book's run-local config, so
|
||
// the config the platform actually deploys carried zero of them (D39.126 п.2а: `gates.terminology` /
|
||
// `gates.banknote` = 0 occurrences). This pins the contour as SHIPPED, and pins the one number in it that
|
||
// has a measurement behind it.
|
||
|
||
// TestShippingC1CarriesTheBankContour is the row itself: all three parts, in the config that deploys.
|
||
func TestShippingC1CarriesTheBankContour(t *testing.T) {
|
||
models, err := LoadModels(filepath.Join("..", "..", "configs", "models.yaml"))
|
||
if err != nil {
|
||
t.Fatalf("load models: %v", err)
|
||
}
|
||
p, err := LoadPipeline(filepath.Join("..", "..", "configs", "pipeline-c1.yaml"), models, "zh-ru", nil)
|
||
if err != nil {
|
||
t.Fatalf("load pipeline-c1: %v", err)
|
||
}
|
||
if !p.Gates.Banknote.Enabled {
|
||
t.Error("the banknote channel is the DIRECT dst delivery the co-occurrence miner cannot extract (蛊→гу); the shipping config must carry it")
|
||
}
|
||
// The banknote gate only SLICES the block; the instruction to write one lives in the prompt, and
|
||
// enabling one without the other is a channel that never fires (D39.10, two coordinated changes).
|
||
for _, st := range p.Stages {
|
||
if st.Role != "translator" {
|
||
continue
|
||
}
|
||
if !strings.HasSuffix(st.PromptPath, "translator-banknote.md") {
|
||
t.Errorf("stage %q resolved %q — with the banknote gate on, the translator must be given the prompt that instructs the block", st.Name, st.PromptPath)
|
||
}
|
||
}
|
||
if !p.Gates.Terminology.Enabled {
|
||
t.Error("the terminologist is what makes a book's bank say ONE thing per term; the shipping config must carry it")
|
||
}
|
||
if !p.Gates.Terminology.ClassifyTypes {
|
||
t.Error("the §2 classifier is what stops a realia surface being FORCED to a transliteration (元石 → «юаньши»); the shipping config must carry it")
|
||
}
|
||
// The contour's one deployment-provided input. The KEY must be declared here — the file itself is a
|
||
// deployment question, checked on the write path (CheckMiningContrast).
|
||
if p.Mining.ContrastPath == "" {
|
||
t.Error("both bank roles run inside the bank-mining stop, which returns early without a contrast artifact: the gates above would be configured and never execute")
|
||
}
|
||
}
|
||
|
||
// TestTheBankRoleBudgetsClearTheirMeasuredCost is the money half, and it is a floor rather than a value:
|
||
// the numbers may be raised freely, and lowering them below what the roles were MEASURED to cost is the
|
||
// defect this pin exists for — a budget that cuts work the buyer already paid for.
|
||
//
|
||
// THE MEASUREMENT (backlog row 16, the cold run's ten chapters, July money re-pinned ×4.47 per D39.179 п.1):
|
||
// terminologist $0.005750 × 4.47 = $0.0257, classifier $0.007706 × 4.47 = $0.0344 — for TEN chapters.
|
||
// Both budgets are BOOK-WIDE and accumulate across every run of the book (store.RoleSpentUSD sums the
|
||
// role's checkpoints over all its jobs), and the terminologist's candidate set depends on WHICH chunks
|
||
// have been drafted — so between purchases it changes, the request hash changes with it, and a book sold
|
||
// in ten purchases pays ten book-wide consolidations rather than replaying one for $0 (measured, the
|
||
// money-honesty pack). The floor asserted here is therefore ten times the ten-chapter figure.
|
||
//
|
||
// ⚠ The plain ×4.47 of the old numbers (0.2235 / 0.0894) does NOT clear that floor — repairing the price
|
||
// drift alone would have kept a sizing made for a ten-chapter probe. ⚠ And the CLASSIFIER is the more
|
||
// expensive of the two by measurement, which is why it no longer carries half the render's budget.
|
||
func TestTheBankRoleBudgetsClearTheirMeasuredCost(t *testing.T) {
|
||
const (
|
||
measuredTerminologistPerTenChapters = 0.0257
|
||
measuredClassifierPerTenChapters = 0.0344
|
||
purchasesABookMayBeSoldIn = 10
|
||
)
|
||
models, err := LoadModels(filepath.Join("..", "..", "configs", "models.yaml"))
|
||
if err != nil {
|
||
t.Fatalf("load models: %v", err)
|
||
}
|
||
p, err := LoadPipeline(filepath.Join("..", "..", "configs", "pipeline-c1.yaml"), models, "zh-ru", nil)
|
||
if err != nil {
|
||
t.Fatalf("load pipeline-c1: %v", err)
|
||
}
|
||
g := p.Gates.Terminology
|
||
if floor := measuredTerminologistPerTenChapters * purchasesABookMayBeSoldIn; g.BudgetUSD < floor {
|
||
t.Errorf("gates.terminology.budget_usd = %.4f, below the measured floor %.4f — a budget under the measured cost cuts PAID work, which is the incident this calibration answers", g.BudgetUSD, floor)
|
||
}
|
||
if floor := measuredClassifierPerTenChapters * purchasesABookMayBeSoldIn; g.ClassifyBudgetUSD < floor {
|
||
t.Errorf("gates.terminology.classify_budget_usd = %.4f, below the measured floor %.4f — on the cold run of 31.08 a $0.02 ceiling here cut the classifier twice", g.ClassifyBudgetUSD, floor)
|
||
}
|
||
}
|
||
|
||
// TestABankContourThatCanNeverFireIsRefused is the LOADER half of "configured and inert is impossible", and
|
||
// it was unpinned after being built — the acceptance's own planting removed the refusal and left the whole
|
||
// battery green. Both bank roles live inside the bank-mining stop, which returns before building a single
|
||
// candidate when `mining.contrast_path` is empty; a fully budgeted terminologist under an unset key is a
|
||
// gate that can never run, the class this loader refuses everywhere else.
|
||
func TestABankContourThatCanNeverFireIsRefused(t *testing.T) {
|
||
dir := termProject(t, true)
|
||
// The same gate, with and without the one key that decides whether it can execute at all.
|
||
const gate = `
|
||
gates:
|
||
terminology:
|
||
enabled: true
|
||
model: fake
|
||
budget_usd: 1
|
||
target_script: Cyrillic
|
||
`
|
||
_, err := LoadPipeline(draftOnlyConfig(t, dir, gate), miniModels(t), "zh-ru", nil)
|
||
if err == nil {
|
||
t.Fatal("a terminology gate with no mining contrast can never execute and must be refused")
|
||
}
|
||
for _, want := range []string{"mining.contrast_path", "bank-mining stop", "never execute"} {
|
||
if !strings.Contains(err.Error(), want) {
|
||
t.Errorf("the refusal must name the cure and the reason, missing %q: %v", want, err)
|
||
}
|
||
}
|
||
// And with the key declared it loads: the loader judges the KEY, never whether the FILE is on this
|
||
// machine — that is a deployment question, and refusing a $0 read on it is the D20.4 mistake.
|
||
if _, err := LoadPipeline(draftOnlyConfig(t, dir, gate+"mining:\n contrast_path: nowhere-on-this-disk.txt\n"), miniModels(t), "zh-ru", nil); err != nil {
|
||
t.Fatalf("the loader judges the key, not the file: %v", err)
|
||
}
|
||
}
|
||
|
||
// TestABankModelIsAReachableModel closes a MONEY hole found by planting. ReachableModels is what
|
||
// buildClients pre-builds and what CheckKeys demands keys for, both at OPEN time — and the bank roles were
|
||
// not in it. A terminologist or classifier model that is undefined, or whose provider has no key in the
|
||
// environment, therefore passed the entire preflight and killed the run at the bank-mining stop, which is
|
||
// after the whole draft wave has been bought. It was latent while no shipping config ran the contour;
|
||
// backlog row 140 makes it live, and pipeline-c1.yaml is safe only because its bank model happens to be the
|
||
// draft model — a coincidence, not a guarantee.
|
||
func TestABankModelIsAReachableModel(t *testing.T) {
|
||
models, err := LoadModels(filepath.Join("..", "..", "configs", "models.yaml"))
|
||
if err != nil {
|
||
t.Fatalf("load models: %v", err)
|
||
}
|
||
p, err := LoadPipeline(filepath.Join("..", "..", "configs", "pipeline-c1.yaml"), models, "zh-ru", nil)
|
||
if err != nil {
|
||
t.Fatalf("load pipeline-c1: %v", err)
|
||
}
|
||
// ⚠ ASSERTED ON A CONFIG WHERE THE BANK MODEL IS ITS OWN, not on c1 — and that is the whole design of
|
||
// this test. In pipeline-c1.yaml the terminologist's model IS the draft stage's model, so it is reachable
|
||
// through the stage whatever this function does: c1 is safe by coincidence, and a test that read it could
|
||
// not fail. The hole is only visible where the bank role runs a model nothing else does, which is exactly
|
||
// what an operator does when they point classification at something cheaper.
|
||
own := *p
|
||
own.Gates.Terminology.Model = "a-bank-model-no-stage-uses"
|
||
own.Gates.Terminology.ClassifyModel = "a-classify-model-no-stage-uses"
|
||
reachable := map[string]bool{}
|
||
for _, m := range own.ReachableModels() {
|
||
reachable[m] = true
|
||
}
|
||
for _, c := range []struct{ name, model string }{
|
||
{"gates.terminology.model", own.Gates.Terminology.Model},
|
||
{"the classifier's model", own.Gates.Terminology.ClassifierModel()},
|
||
} {
|
||
if !reachable[c.model] {
|
||
t.Errorf("%s (%q) is not among the reachable models — its client is never pre-built and its key is never demanded, so the run discovers the problem at the bank-mining stop, after the draft wave is paid for", c.name, c.model)
|
||
}
|
||
}
|
||
// And the shipping config's own bank models are reachable too, however they got there.
|
||
for _, m := range []string{p.Gates.Terminology.Model, p.Gates.Terminology.ClassifierModel()} {
|
||
if m == "" {
|
||
t.Fatal("premise: the shipping config must name its bank models")
|
||
}
|
||
}
|
||
// A DISABLED gate must add nothing: a book that does not run the contour must not be made to hold a key
|
||
// it never uses.
|
||
off := own
|
||
off.Gates.Terminology.Enabled = false
|
||
for _, m := range off.ReachableModels() {
|
||
if m == off.Gates.Terminology.Model || m == off.Gates.Terminology.ClassifierModel() {
|
||
t.Errorf("a disabled gate must demand nothing, yet %q is reachable", m)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestTheUnlabelledPreflightDemandsEveryCallersKey closes the SAME money hole one layer further in — and it
|
||
// exists because the fix that was supposed to close it did not.
|
||
//
|
||
// ⚠ THE PREVIOUS FIX PUT THE BANK ROLES INTO ReachableModels() AND STOPPED THERE, and the test above
|
||
// asserted exactly that: the LIST. But CheckKeys reads that list only when the book is LABELLED. An
|
||
// unlabelled book — the shipping default — takes a second branch that collects stage models by hand, and
|
||
// that branch never learned about the gates. So the money hole stayed open on the default path while a
|
||
// green test said it was closed. A pin on the list is not a pin on the preflight; this one calls CheckKeys.
|
||
func TestTheUnlabelledPreflightDemandsEveryCallersKey(t *testing.T) {
|
||
m := &Models{
|
||
Providers: map[string]Provider{
|
||
"stage-provider": {Kind: "openai", BaseURL: "http://x", APIKeyEnv: "TEST_STAGE_KEY"},
|
||
"other-provider": {Kind: "openai", BaseURL: "http://y", APIKeyEnv: "TEST_OTHER_KEY"},
|
||
},
|
||
Models: map[string]Model{
|
||
"draft-model": {Provider: "stage-provider"},
|
||
"bank-model": {Provider: "other-provider"},
|
||
"classify-model": {Provider: "other-provider"},
|
||
"repair-model": {Provider: "other-provider"},
|
||
},
|
||
}
|
||
// The stage's own key is present, so the ONLY thing that can fail is a gate's key. The gates run on a
|
||
// DIFFERENT provider on purpose: sharing the draft's provider is what makes pipeline-c1 safe today, and
|
||
// a fixture that shared it could not fail.
|
||
t.Setenv("TEST_STAGE_KEY", "sk-present")
|
||
t.Setenv("TEST_OTHER_KEY", "")
|
||
|
||
base := func() *Pipeline {
|
||
return &Pipeline{Stages: []Stage{{Name: "draft", Model: "draft-model"}}}
|
||
}
|
||
for _, c := range []struct {
|
||
name string
|
||
mut func(*Pipeline)
|
||
}{
|
||
{"the terminologist", func(p *Pipeline) {
|
||
p.Gates.Terminology = TerminologyGate{Enabled: true, Model: "bank-model"}
|
||
}},
|
||
{"the classifier phase", func(p *Pipeline) {
|
||
p.Gates.Terminology = TerminologyGate{Enabled: true, Model: "draft-model",
|
||
ClassifyTypes: true, ClassifyModel: "classify-model"}
|
||
}},
|
||
{"the repair gate", func(p *Pipeline) {
|
||
p.Gates.Repair = RepairGate{Enabled: true, Model: "repair-model"}
|
||
}},
|
||
} {
|
||
t.Run(c.name, func(t *testing.T) {
|
||
p := base()
|
||
c.mut(p)
|
||
if len(p.ContentLabels) != 0 {
|
||
t.Fatal("premise broken: this test is about the UNLABELLED branch, the shipping default")
|
||
}
|
||
err := m.CheckKeys(p)
|
||
if err == nil {
|
||
t.Fatalf("%s calls a provider whose key is absent and the preflight passed: the run buys the "+
|
||
"draft wave and dies at the gate, which is the definition of a late failure that costs money", c.name)
|
||
}
|
||
if !strings.Contains(err.Error(), "TEST_OTHER_KEY") {
|
||
t.Fatalf("the refusal must name the env var an operator has to set, got: %v", err)
|
||
}
|
||
})
|
||
}
|
||
|
||
// A DISABLED gate must demand nothing — the other half, and the reason this cannot be closed by simply
|
||
// demanding every model in the file.
|
||
for _, c := range []struct {
|
||
name string
|
||
mut func(*Pipeline)
|
||
}{
|
||
{"terminology off", func(p *Pipeline) {
|
||
p.Gates.Terminology = TerminologyGate{Enabled: false, Model: "bank-model"}
|
||
}},
|
||
{"repair off", func(p *Pipeline) {
|
||
p.Gates.Repair = RepairGate{Enabled: false, Model: "repair-model"}
|
||
}},
|
||
{"classifier phase off", func(p *Pipeline) {
|
||
p.Gates.Terminology = TerminologyGate{Enabled: true, Model: "draft-model",
|
||
ClassifyTypes: false, ClassifyModel: "classify-model"}
|
||
}},
|
||
} {
|
||
t.Run(c.name, func(t *testing.T) {
|
||
p := base()
|
||
c.mut(p)
|
||
if err := m.CheckKeys(p); err != nil {
|
||
t.Fatalf("a book that never calls that model must not be made to hold its key: %v", err)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
// TestTheLABELLEDPreflightDemandsEveryCallersKeyToo is the twin of the test above, and it exists because
|
||
// the fix that closed the hole closed ONE branch of it.
|
||
//
|
||
// ⚠ CheckKeys forks on whether the book carries content labels. The unlabelled branch — the shipping
|
||
// default — was the one found open and the one pinned. The labelled branch reaches the gates only through
|
||
// ReachableModels(), and the only thing asserting that was TestABankModelIsAReachableModel, which reads
|
||
// the LIST. «A pin on the list is not a pin on the preflight» is the sentence this pack was written around;
|
||
// leaving the second branch pinned by exactly the thing that sentence rejects is the same defect with the
|
||
// paint changed. Planted — the labelled branch collecting stage models by hand — the whole package stayed
|
||
// green, and a labelled book with its bank role on a keyless provider bought the draft wave and died at
|
||
// the gate, which is where this started.
|
||
func TestTheLABELLEDPreflightDemandsEveryCallersKeyToo(t *testing.T) {
|
||
m := &Models{
|
||
Providers: map[string]Provider{
|
||
"stage-provider": {Kind: "openai", BaseURL: "http://x", APIKeyEnv: "TEST_STAGE_KEY"},
|
||
"other-provider": {Kind: "openai", BaseURL: "http://y", APIKeyEnv: "TEST_OTHER_KEY"},
|
||
},
|
||
Models: map[string]Model{
|
||
"draft-model": {Provider: "stage-provider"},
|
||
"bank-model": {Provider: "other-provider"},
|
||
"classify-model": {Provider: "other-provider"},
|
||
"repair-model": {Provider: "other-provider"},
|
||
"hop-model": {Provider: "other-provider"},
|
||
},
|
||
}
|
||
t.Setenv("TEST_STAGE_KEY", "sk-present")
|
||
t.Setenv("TEST_OTHER_KEY", "")
|
||
|
||
base := func() *Pipeline {
|
||
return &Pipeline{
|
||
ContentLabels: []string{"violence"},
|
||
Stages: []Stage{{Name: "draft", Model: "draft-model", ResolvedModel: "draft-model"}},
|
||
}
|
||
}
|
||
for _, c := range []struct {
|
||
name string
|
||
mut func(*Pipeline)
|
||
}{
|
||
{"the terminologist", func(p *Pipeline) {
|
||
p.Gates.Terminology = TerminologyGate{Enabled: true, Model: "bank-model"}
|
||
}},
|
||
{"the classifier phase", func(p *Pipeline) {
|
||
p.Gates.Terminology = TerminologyGate{Enabled: true, Model: "draft-model",
|
||
ClassifyTypes: true, ClassifyModel: "classify-model"}
|
||
}},
|
||
{"the repair gate", func(p *Pipeline) {
|
||
p.Gates.Repair = RepairGate{Enabled: true, Model: "repair-model"}
|
||
}},
|
||
// ⚠ THE HOP, found by an adversarial pass over this very test: the gates were covered and the
|
||
// stage's own hop was not. A labelled run calls its escalation hop, BuildClient never refuses an
|
||
// empty key (it only becomes a header), so under a preflight blind to the hop the book opens, buys
|
||
// the draft wave, and 401s at the escalation — the same late failure, one field over.
|
||
{"the stage's escalation hop", func(p *Pipeline) {
|
||
p.Stages[0].ResolvedHop = "hop-model"
|
||
}},
|
||
// ⚠ AND THE RESOLVED MODEL, NOT THE CONFIGURED ONE. Routing is what a labelled book is FOR: the
|
||
// model in the yaml may not be the model that answers. A preflight that demanded `Model` would pass
|
||
// here and the first draft call would 401 — the refusal moving from open time to mid-run, which is
|
||
// precisely the distance this whole family of tests exists to keep at zero.
|
||
{"the RESOLVED model when routing replaced the configured one", func(p *Pipeline) {
|
||
p.Stages[0].Model, p.Stages[0].ResolvedModel = "draft-model", "bank-model"
|
||
}},
|
||
} {
|
||
t.Run(c.name, func(t *testing.T) {
|
||
p := base()
|
||
c.mut(p)
|
||
if len(p.ContentLabels) == 0 {
|
||
t.Fatal("premise broken: this test is about the LABELLED branch and must carry labels")
|
||
}
|
||
err := m.CheckKeys(p)
|
||
if err == nil {
|
||
t.Fatalf("a LABELLED book's %s calls a provider whose key is absent and the preflight passed: "+
|
||
"the run buys the draft wave and dies at the gate", c.name)
|
||
}
|
||
if !strings.Contains(err.Error(), "TEST_OTHER_KEY") {
|
||
t.Fatalf("the refusal must name the env var an operator has to set, got: %v", err)
|
||
}
|
||
})
|
||
}
|
||
// And the disabled direction, so this cannot be satisfied by demanding every model in the file.
|
||
t.Run("gates off demand nothing", func(t *testing.T) {
|
||
p := base()
|
||
p.Gates.Terminology = TerminologyGate{Enabled: false, Model: "bank-model"}
|
||
p.Gates.Repair = RepairGate{Enabled: false, Model: "repair-model"}
|
||
if err := m.CheckKeys(p); err != nil {
|
||
t.Fatalf("a labelled book that never calls those models must not be made to hold their keys: %v", err)
|
||
}
|
||
})
|
||
}
|