textmachine/backend/internal/config/armparity_test.go

package config

import (
	"fmt"
	"path/filepath"
	"reflect"
	"strings"
	"testing"
)

// armparity_test.go: an editor swap-arm exists to isolate ONE variable, and it can only do that if every
// other key is the production config's. Three arm headers claimed exactly that and were false by two dozen
// keys — the whole bank contour among them (b8154cd gave pipeline-c1.yaml sixty-four lines of it and the
// arms a single prompt_version), while the test that called itself the guard of «only the editor differs»
// compared three fields. The cost is money, not tidiness, and it runs the OTHER way: with no
// mining.contrast_path the arm's own book-level price is zero (priceprojection.go, bookOnceUSD returns 0
// when the terminology gate is off or the contrast path is empty), so a deploy flipped to an arm quietly
// bought a cheaper book than the one it reported — the bank contour was configured nowhere and priced
// nowhere, and nothing said so.
//
// ⛔ THE FORM IS THE POINT. This compares the WHOLE loaded configs and exempts the editor stage, instead of
// listing the keys that must match. A list is what drifted: it can only check what somebody remembered to
// put in it, and a key ADDED to c1 is invisible to it by construction.

func TestAnArmIsTheProductionConfigWithADifferentEditor(t *testing.T) {
	m, err := LoadModels(filepath.Join("..", "..", "configs", "models.yaml"))
	if err != nil {
		t.Fatalf("load the shipping models.yaml: %v", err)
	}
	// ⛔ THE ARMS ARE DISCOVERED, NOT LISTED. A list here would be the very thing the header refuses one
	// paragraph up: a new pipeline-arm-*.yaml would join the repository outside every gate, and the file
	// carrying the money defect (an arm with no mining.contrast_path) would be the one nobody loaded.
	arms, err := filepath.Glob(filepath.Join("..", "..", "configs", "pipeline-arm-*.yaml"))
	if err != nil {
		t.Fatal(err)
	}
	if len(arms) == 0 {
		t.Fatal("no configs/pipeline-arm-*.yaml in the repository — this gate would be enforcing nothing")
	}
	load := func(t *testing.T, file string) *Pipeline {
		t.Helper()
		p, lerr := LoadPipeline(file, m, "zh-ru", nil)
		if lerr != nil {
			// ⚠ t is a PARAMETER, not the enclosing test's: a Fatalf on the outer t from inside a subtest
			// kills the parent, and every arm after this one goes unchecked.
			t.Fatalf("load %s: %v", file, lerr)
		}
		return p
	}
	base := load(t, filepath.Join("..", "..", "configs", "pipeline-c1.yaml"))
	baseEditor := editorOf(t, base)
	t.Logf("arms discovered: %d %v; production editor: %s", len(arms), armNames(arms), baseEditor.Model)
	for _, file := range arms {
		t.Run(filepath.Base(file), func(t *testing.T) {
			arm := load(t, file)
			editor := editorOf(t, arm).Model
			// An arm whose editor is the production editor isolates nothing, and a file that isolates
			// nothing is not an arm — it is a second copy of the production config that has to be kept in
			// step forever. Said as a gate, because it is a state the interim editor can drift INTO
			// (D39.22 calls the current one interim): the day c1 adopts an arm's editor, that arm has to be
			// retired or re-pointed, deliberately.
			if baseEditor.Model == editor {
				t.Fatalf("the production editor is already %q, so this file isolates no variable: retire it or point it at a different editor", editor)
			}
			// The exemptions are NAMED in the log, so a green run is never read as "every key was compared".
			// reasoning_max_tokens is exempt only where the loader forces it — today that is one model of the
			// nine in models.yaml (grok-4.3, the only `additive`), and no arm runs it.
			exempt := "model, resolved model, escalate_to, few_shot"
			// ⛔ AND THE ONE EXEMPTION THAT IS CONDITIONAL IS ASSERTED TO BE UNUSED, not merely logged.
			// reasoning_max_tokens is dropped from the comparison only where the loader FORCES a value —
			// additive billing — because two different models then carry two different forced values and
			// comparing them would fail for a reason that is not drift. Today no editor on either side is
			// additive (one model of the nine in models.yaml is, and no config runs it), so the exemption
			// never fires and its correctness is never exercised. The day an editor moves there it starts
			// firing silently, and the only trace would be the log line above — which nobody reads on a
			// green run. So it fails instead: the move is legitimate, the decision it needs is not
			// automatic, and a gate that quietly stops comparing a key is the defect this file exists for.
			for side, p := range map[string]*Pipeline{"production": base, filepath.Base(file): arm} {
				if ed := editorOf(t, p); forcesReasoningBuffer(m, ed.Model) {
					t.Fatalf("%s now runs editor %q, whose provider bills reasoning additively, so the loader forces reasoning_max_tokens and this gate stops comparing its VALUE. Decide deliberately: either compare the forced values (they are a function of the model, so equal ones mean nothing) or drop the key with a reason written here",
						side, ed.Model)
				}
			}
			t.Logf("%s: editor %q; exempt from the comparison: %s", filepath.Base(file), editor, exempt)
			for _, d := range pipelineFieldDiffs(withoutEditorModel(t, m, base), withoutEditorModel(t, m, arm)) {
				t.Errorf("%s is not pipeline-c1.yaml with a different editor — %s", filepath.Base(file), d)
			}
		})
	}
}

// forcesReasoningBuffer is the ONE reading of the loader's rule, used both by the exemption and by the
// guard that asserts the exemption is unused. Two copies of one predicate drift, and the drift is silent
// in exactly the direction that matters: an exemption firing while the guard says it does not.
// It mirrors pipeline.go:771 — the loader refuses a stage resolving to an additive-billing provider with
// no reasoning_max_tokens, so there the value is forced and is a function of the model, not of the config.
// `additive_total` is deliberately NOT included: the loader does not force the key there either, and a
// predicate broader than the rule it reads would exempt a key that nothing forces.
func forcesReasoningBuffer(m *Models, model string) bool {
	return m.providerReasoning(model) == "additive"
}

// armNames renders the discovered files for a log line.
func armNames(paths []string) []string {
	out := make([]string, 0, len(paths))
	for _, p := range paths {
		out = append(out, filepath.Base(p))
	}
	return out
}

// editorOf returns the pipeline's editor stage, failing loudly when there is none.
func editorOf(t *testing.T, p *Pipeline) *Stage {
	t.Helper()
	for i := range p.Stages {
		if p.Stages[i].Role == "editor" {
			return &p.Stages[i]
		}
	}
	t.Fatal("the config has no editor stage")
	return nil
}

// withoutEditorModel is the config with the editor's MODEL blanked — the model, what it resolves to, and the
// knobs the model choice FORCES: the few-shot policy (a reasoning editor drops the hand examples, a
// non-reasoning one keeps them; TestSwapArmConfigs pins which is which), and reasoning_max_tokens on an
// ADDITIVE-billing provider, where LoadPipeline refuses the stage without a buffer. Everything else —
// temperature, the reasoning knob, the prompt and its label — is COMPARED: chosen, not forced, and each is a
// second variable the arm would isolate without saying so.
//
// ⛔ THE CONDITION IS THE POINT, and blanking unconditionally traded one hole for another. On a subset-billed
// editor reasoning_max_tokens is optional and unchecked, yet it sizes the spend estimate — so an arm could
// carry a buffer production does not have and stay green. Exempt exactly what the other rule forces.
//
// ⛔ Blanking the WHOLE stage was this gate's own first hole: an arm could then differ from production on
// temperature and reasoning too and stay green, isolating three variables while its header claimed one.
//
// The stage SLOT stays in place, so a stage list of a different length or order still differs. The Stages
// slice is copied because a Pipeline value shares its backing array with the original.
func withoutEditorModel(t *testing.T, m *Models, p *Pipeline) Pipeline {
	t.Helper()
	out := *p
	out.Stages = append([]Stage(nil), p.Stages...)
	found := false
	for i := range out.Stages {
		if out.Stages[i].Role != "editor" {
			continue
		}
		// Read the model from p: out's copy is blanked two lines below.
		if forcesReasoningBuffer(m, p.Stages[i].Model) {
			out.Stages[i].ReasoningMaxTokens = 0
		}
		out.Stages[i].Model, out.Stages[i].ResolvedModel = "", ""
		out.Stages[i].EscalateTo, out.Stages[i].ResolvedHop = "", ""
		out.Stages[i].FewShot = nil
		found = true
	}
	if !found {
		t.Fatal("no editor stage: this comparison would exempt nothing, and the two files would not be an arm and its base")
	}
	return out
}

// pipelineFieldDiffs names the top-level fields of two loaded pipelines that differ, with both values.
// Top-level is deliberate: it points a reader at the BLOCK (Gates, Mining, Stages), and the two YAML files
// are then a three-line diff apart — where a recursive differ would be a second, hand-written model of a
// struct that already knows how to compare itself.
// renderField prints a field for the failure message with POINTERS DEREFERENCED. Stage.FewShot is a *bool
// — a plain %+v prints its address, which is the one thing an operator cannot compare between two configs,
// and few-shot is exactly the key an arm is allowed to differ on for a documented reason.
func renderField(v reflect.Value) string {
	switch v.Kind() {
	case reflect.Pointer:
		if v.IsNil() {
			return "<nil>"
		}
		return "&" + renderField(v.Elem())
	case reflect.Slice, reflect.Array:
		parts := make([]string, 0, v.Len())
		for i := 0; i < v.Len(); i++ {
			parts = append(parts, renderField(v.Index(i)))
		}
		return "[" + strings.Join(parts, " ") + "]"
	case reflect.Struct:
		parts := make([]string, 0, v.NumField())
		for i := 0; i < v.NumField(); i++ {
			if !v.Field(i).CanInterface() {
				continue
			}
			parts = append(parts, v.Type().Field(i).Name+":"+renderField(v.Field(i)))
		}
		return "{" + strings.Join(parts, " ") + "}"
	default:
		return fmt.Sprintf("%+v", v.Interface())
	}
}

func pipelineFieldDiffs(a, b Pipeline) []string {
	va, vb := reflect.ValueOf(a), reflect.ValueOf(b)
	var out []string
	for i := 0; i < va.NumField(); i++ {
		// CanInterface, because the first unexported field added to Pipeline would otherwise turn this gate
		// into a panic instead of a message — and an unexported field is invisible to this comparison in
		// any case, which is worth saying out loud rather than crashing over.
		if !va.Field(i).CanInterface() {
			continue
		}
		if reflect.DeepEqual(va.Field(i).Interface(), vb.Field(i).Interface()) {
			continue
		}
		out = append(out, fmt.Sprintf("%s differs:\n  c1  = %s\n  arm = %s",
			va.Type().Field(i).Name, renderField(va.Field(i)), renderField(vb.Field(i))))
	}
	return out
}