package config import ( "fmt" "path/filepath" "reflect" "strings" "testing" ) // armparity_test.go: an editor swap-arm exists to isolate ONE variable, and it can only do that if every // other key is the production config's. Three arm headers claimed exactly that and were false by two dozen // keys — the whole bank contour among them (b8154cd gave pipeline-c1.yaml sixty-four lines of it and the // arms a single prompt_version), while the test that called itself the guard of «only the editor differs» // compared three fields. The cost is money, not tidiness, and it runs the OTHER way: with no // mining.contrast_path the arm's own book-level price is zero (priceprojection.go, bookOnceUSD returns 0 // when the terminology gate is off or the contrast path is empty), so a deploy flipped to an arm quietly // bought a cheaper book than the one it reported — the bank contour was configured nowhere and priced // nowhere, and nothing said so. // // ⛔ THE FORM IS THE POINT. This compares the WHOLE loaded configs and exempts the editor stage, instead of // listing the keys that must match. A list is what drifted: it can only check what somebody remembered to // put in it, and a key ADDED to c1 is invisible to it by construction. func TestAnArmIsTheProductionConfigWithADifferentEditor(t *testing.T) { m, err := LoadModels(filepath.Join("..", "..", "configs", "models.yaml")) if err != nil { t.Fatalf("load the shipping models.yaml: %v", err) } // ⛔ THE ARMS ARE DISCOVERED, NOT LISTED. A list here would be the very thing the header refuses one // paragraph up: a new pipeline-arm-*.yaml would join the repository outside every gate, and the file // carrying the money defect (an arm with no mining.contrast_path) would be the one nobody loaded. arms, err := filepath.Glob(filepath.Join("..", "..", "configs", "pipeline-arm-*.yaml")) if err != nil { t.Fatal(err) } if len(arms) == 0 { t.Fatal("no configs/pipeline-arm-*.yaml in the repository — this gate would be enforcing nothing") } load := func(t *testing.T, file string) *Pipeline { t.Helper() p, lerr := LoadPipeline(file, m, "zh-ru", nil) if lerr != nil { // ⚠ t is a PARAMETER, not the enclosing test's: a Fatalf on the outer t from inside a subtest // kills the parent, and every arm after this one goes unchecked. t.Fatalf("load %s: %v", file, lerr) } return p } base := load(t, filepath.Join("..", "..", "configs", "pipeline-c1.yaml")) baseEditor := editorOf(t, base) t.Logf("arms discovered: %d %v; production editor: %s", len(arms), armNames(arms), baseEditor.Model) for _, file := range arms { t.Run(filepath.Base(file), func(t *testing.T) { arm := load(t, file) editor := editorOf(t, arm).Model // An arm whose editor is the production editor isolates nothing, and a file that isolates // nothing is not an arm — it is a second copy of the production config that has to be kept in // step forever. Said as a gate, because it is a state the interim editor can drift INTO // (D39.22 calls the current one interim): the day c1 adopts an arm's editor, that arm has to be // retired or re-pointed, deliberately. if baseEditor.Model == editor { t.Fatalf("the production editor is already %q, so this file isolates no variable: retire it or point it at a different editor", editor) } // The exemptions are NAMED in the log, so a green run is never read as "every key was compared". // reasoning_max_tokens is exempt only where the loader forces it — today that is one model of the // nine in models.yaml (grok-4.3, the only `additive`), and no arm runs it. exempt := "model, resolved model, escalate_to, few_shot" // ⛔ AND THE ONE EXEMPTION THAT IS CONDITIONAL IS ASSERTED TO BE UNUSED, not merely logged. // reasoning_max_tokens is dropped from the comparison only where the loader FORCES a value — // additive billing — because two different models then carry two different forced values and // comparing them would fail for a reason that is not drift. Today no editor on either side is // additive (one model of the nine in models.yaml is, and no config runs it), so the exemption // never fires and its correctness is never exercised. The day an editor moves there it starts // firing silently, and the only trace would be the log line above — which nobody reads on a // green run. So it fails instead: the move is legitimate, the decision it needs is not // automatic, and a gate that quietly stops comparing a key is the defect this file exists for. for side, p := range map[string]*Pipeline{"production": base, filepath.Base(file): arm} { if ed := editorOf(t, p); forcesReasoningBuffer(m, ed.Model) { t.Fatalf("%s now runs editor %q, whose provider bills reasoning additively, so the loader forces reasoning_max_tokens and this gate stops comparing its VALUE. Decide deliberately: either compare the forced values (they are a function of the model, so equal ones mean nothing) or drop the key with a reason written here", side, ed.Model) } } t.Logf("%s: editor %q; exempt from the comparison: %s", filepath.Base(file), editor, exempt) for _, d := range pipelineFieldDiffs(withoutEditorModel(t, m, base), withoutEditorModel(t, m, arm)) { t.Errorf("%s is not pipeline-c1.yaml with a different editor — %s", filepath.Base(file), d) } }) } } // forcesReasoningBuffer is the ONE reading of the loader's rule, used both by the exemption and by the // guard that asserts the exemption is unused. Two copies of one predicate drift, and the drift is silent // in exactly the direction that matters: an exemption firing while the guard says it does not. // It mirrors pipeline.go:771 — the loader refuses a stage resolving to an additive-billing provider with // no reasoning_max_tokens, so there the value is forced and is a function of the model, not of the config. // `additive_total` is deliberately NOT included: the loader does not force the key there either, and a // predicate broader than the rule it reads would exempt a key that nothing forces. func forcesReasoningBuffer(m *Models, model string) bool { return m.providerReasoning(model) == "additive" } // armNames renders the discovered files for a log line. func armNames(paths []string) []string { out := make([]string, 0, len(paths)) for _, p := range paths { out = append(out, filepath.Base(p)) } return out } // editorOf returns the pipeline's editor stage, failing loudly when there is none. func editorOf(t *testing.T, p *Pipeline) *Stage { t.Helper() for i := range p.Stages { if p.Stages[i].Role == "editor" { return &p.Stages[i] } } t.Fatal("the config has no editor stage") return nil } // withoutEditorModel is the config with the editor's MODEL blanked — the model, what it resolves to, and the // knobs the model choice FORCES: the few-shot policy (a reasoning editor drops the hand examples, a // non-reasoning one keeps them; TestSwapArmConfigs pins which is which), and reasoning_max_tokens on an // ADDITIVE-billing provider, where LoadPipeline refuses the stage without a buffer. Everything else — // temperature, the reasoning knob, the prompt and its label — is COMPARED: chosen, not forced, and each is a // second variable the arm would isolate without saying so. // // ⛔ THE CONDITION IS THE POINT, and blanking unconditionally traded one hole for another. On a subset-billed // editor reasoning_max_tokens is optional and unchecked, yet it sizes the spend estimate — so an arm could // carry a buffer production does not have and stay green. Exempt exactly what the other rule forces. // // ⛔ Blanking the WHOLE stage was this gate's own first hole: an arm could then differ from production on // temperature and reasoning too and stay green, isolating three variables while its header claimed one. // // The stage SLOT stays in place, so a stage list of a different length or order still differs. The Stages // slice is copied because a Pipeline value shares its backing array with the original. func withoutEditorModel(t *testing.T, m *Models, p *Pipeline) Pipeline { t.Helper() out := *p out.Stages = append([]Stage(nil), p.Stages...) found := false for i := range out.Stages { if out.Stages[i].Role != "editor" { continue } // Read the model from p: out's copy is blanked two lines below. if forcesReasoningBuffer(m, p.Stages[i].Model) { out.Stages[i].ReasoningMaxTokens = 0 } out.Stages[i].Model, out.Stages[i].ResolvedModel = "", "" out.Stages[i].EscalateTo, out.Stages[i].ResolvedHop = "", "" out.Stages[i].FewShot = nil found = true } if !found { t.Fatal("no editor stage: this comparison would exempt nothing, and the two files would not be an arm and its base") } return out } // pipelineFieldDiffs names the top-level fields of two loaded pipelines that differ, with both values. // Top-level is deliberate: it points a reader at the BLOCK (Gates, Mining, Stages), and the two YAML files // are then a three-line diff apart — where a recursive differ would be a second, hand-written model of a // struct that already knows how to compare itself. // renderField prints a field for the failure message with POINTERS DEREFERENCED. Stage.FewShot is a *bool // — a plain %+v prints its address, which is the one thing an operator cannot compare between two configs, // and few-shot is exactly the key an arm is allowed to differ on for a documented reason. func renderField(v reflect.Value) string { switch v.Kind() { case reflect.Pointer: if v.IsNil() { return "" } return "&" + renderField(v.Elem()) case reflect.Slice, reflect.Array: parts := make([]string, 0, v.Len()) for i := 0; i < v.Len(); i++ { parts = append(parts, renderField(v.Index(i))) } return "[" + strings.Join(parts, " ") + "]" case reflect.Struct: parts := make([]string, 0, v.NumField()) for i := 0; i < v.NumField(); i++ { if !v.Field(i).CanInterface() { continue } parts = append(parts, v.Type().Field(i).Name+":"+renderField(v.Field(i))) } return "{" + strings.Join(parts, " ") + "}" default: return fmt.Sprintf("%+v", v.Interface()) } } func pipelineFieldDiffs(a, b Pipeline) []string { va, vb := reflect.ValueOf(a), reflect.ValueOf(b) var out []string for i := 0; i < va.NumField(); i++ { // CanInterface, because the first unexported field added to Pipeline would otherwise turn this gate // into a panic instead of a message — and an unexported field is invisible to this comparison in // any case, which is worth saying out loud rather than crashing over. if !va.Field(i).CanInterface() { continue } if reflect.DeepEqual(va.Field(i).Interface(), vb.Field(i).Interface()) { continue } out = append(out, fmt.Sprintf("%s differs:\n c1 = %s\n arm = %s", va.Type().Field(i).Name, renderField(va.Field(i)), renderField(vb.Field(i)))) } return out }