textmachine/platform/internal/gates/battery_test.go

package gates

import (
	"go/ast"
	"go/parser"
	"go/token"
	"io/fs"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"testing"
)

// The battery's hint under its skips must name EVERY host condition the tests read, must say for each
// whether THIS host meets it, and must name what it opens (PD-374): a hint that names one condition of
// several sends a session to a knob that is already on, and the skips it still gets read as the zone's
// normal. The Makefile's `conditions` target derives that list from the test sources; this test holds
// it to that.
//
// ⚠ The two derive the same facts by DIFFERENT MEANS, and that is the point rather than a detail: the
// recipe greps, this gate PARSES the sources and looks at call expressions. A name that lives only in
// a comment is a reader to the grep and not to the parser, and a recipe narrowed to fewer idioms than
// the tests use makes the two disagree — the disagreement is what goes red. Holding both to one
// anchor would make the gate a mirror of the recipe: green whatever either says.
//
// ⚠ WHAT THE WALK DOES NOT SEE, said here because the next author reads this file and not the report:
// it matches a call with a LITERAL argument — `os.Getenv("TM_PLATFORM_TEST_…")`, its `os.LookupEnv`
// twin, `exec.LookPath("…")` — in a `_test.go` of this module. A name arriving through a constant, a
// variable or a helper's parameter, an aliased or dot-imported `os`, or a helper living in a non-test
// file, is invisible to the walk AND to the recipe, so both fall silent together and PD-374 returns at
// full strength. `systemdOrSkip` is exactly that shape and is why the systemd row is hand-written on
// both sides. The class is narrowed here, not closed.
//
// What it asserts, for EVERY row and not for a sample: the NAME appears; the STATE is the host's
// answer, taken here independently (the environment for a variable, `exec.LookPath` for a binary, the
// same systemctl probe `systemdOrSkip` makes); the PACKAGES are exactly those whose tests read it; and
// nothing is printed that the sources do not read.
//
// Mutation caught: replacing the derivation with a literal list; freezing any state column to a word;
// printing an invented condition; attributing a condition to a package that does not read it; dropping
// the systemd probe.
func TestTheBatteryNamesEveryHostConditionItsTestsRead(t *testing.T) {
	if _, err := exec.LookPath("make"); err != nil {
		t.Skip("make is not on this host: the battery's hint cannot be exercised")
	}
	env, bin := conditionsInTheSources(t)
	if len(env) < 2 || len(bin) < 2 {
		t.Fatalf("parsed %d environment conditions and %d binaries out of the test sources, fewer than the battery is known to read: this walk reads the wrong tree", len(env), len(bin))
	}
	// One variable is forced in both directions so the state column is proven to follow the host
	// rather than to print a word; every other row is checked against the host as it stands.
	unset, unsetEnv := conditions(t, "TM_PLATFORM_TEST_DSN=")
	set, _ := conditions(t, "TM_PLATFORM_TEST_DSN=postgres://somewhere")

	printedEnv := map[string][]string{}
	printedBin := map[string][]string{}
	for _, line := range strings.Split(unset, "\n") {
		if m := envRow.FindStringSubmatch(line); m != nil {
			printedEnv[m[1]] = strings.Fields(m[3])
			// The state, against the environment THE RECIPE WAS GIVEN — not this process's own, which
			// differs the moment a variable is forced for the run, and not "is it exported": NON-EMPTY
			// is the fact. That is what the recipe asks (`[ -n "$(printenv …)" ]`) and, more to the
			// point, what every gated test asks — they skip on `os.Getenv(…) == ""`. A wrapper that
			// always exports a knob and leaves it empty (`TM_PLATFORM_TEST_DSN="${DSN:-}"` in a CI
			// script) is a host under which the live suite skips, and a gate that called that "set"
			// would demand the hint lie about it: red for a state the zone did not cause.
			live := unsetEnv[m[1]] != ""
			if want := map[bool]string{true: "set", false: "UNSET"}[live]; m[2] != want {
				t.Errorf("%s is printed as %q and this host says %q", m[1], m[2], want)
			}
			continue
		}
		if m := binRow.FindStringSubmatch(line); m != nil {
			printedBin[m[1]] = strings.Fields(m[3])
			_, err := exec.LookPath(m[1])
			if want := map[bool]string{true: "present", false: "MISSING"}[err == nil]; m[2] != want {
				t.Errorf("binary %s is printed as %q and this host says %q", m[1], m[2], want)
			}
		}
	}
	// Both directions: everything the sources read is printed, and everything printed is read.
	for name, want := range env {
		got, ok := printedEnv[name]
		if !ok {
			t.Errorf("the list does not name %s, which a test reads:\n%s", name, unset)
			continue
		}
		if !equal(got, want) {
			t.Errorf("%s is announced as read by %v, and the sources say %v", name, got, want)
		}
	}
	for name := range printedEnv {
		if _, ok := env[name]; !ok {
			t.Errorf("the list names %s, and no test reads it: a condition nobody reads sends a session to a knob that opens nothing", name)
		}
	}
	for name, want := range bin {
		got, ok := printedBin[name]
		if !ok {
			t.Errorf("the list does not name the binary %s, which a test helper looks up before it runs:\n%s", name, unset)
			continue
		}
		if !equal(got, want) {
			t.Errorf("binary %s is announced as read by %v, and the sources say %v", name, got, want)
		}
	}
	for name := range printedBin {
		if _, ok := bin[name]; !ok {
			t.Errorf("the list names the binary %s, and no test helper looks it up", name)
		}
	}
	if !regexp.MustCompile(`(?m)^\s*TM_PLATFORM_TEST_DSN\s+set\b`).MatchString(set) {
		t.Errorf("with the variable set the list did not say set:\n%s", set)
	}
	// The systemd row carries no name a walk can find — it is a helper's own probe — so it is checked
	// against that same probe, run here.
	m := regexp.MustCompile(`(?m)^\s*systemctl --user\s+(reachable|UNREACHABLE)\s+read by: (\S+).*systemdOrSkip`).FindStringSubmatch(unset)
	if m == nil {
		t.Fatalf("the reachable-user-systemd condition (systemdOrSkip) is not in the list:\n%s", unset)
	}
	reachable := exec.CommandContext(t.Context(), "systemctl", "--user", "show", "--property=Version").Run() == nil
	if want := map[bool]string{true: "reachable", false: "UNREACHABLE"}[reachable]; m[1] != want {
		t.Errorf("the systemd condition is printed as %q and this host answers %q", m[1], want)
	}
	if !strings.Contains(m[2], "internal/runner") {
		t.Errorf("the systemd condition is announced as read by %q, and systemdOrSkip lives in internal/runner", m[2])
	}
	// The idioms themselves, because a set comparison can only speak about conditions that EXIST: the
	// day a test starts reading one through an idiom the recipe does not grep for, that condition
	// vanishes from the hint and BOTH sides fall silent together — the walk would not see it either if
	// the walk were narrowed to match. So the recipe's own anchors are read here and held to the ones
	// this gate understands; narrowing either is what goes red, before any test uses the idiom.
	// The GREP EXPRESSIONS the recipe runs, not words that appear near them: an idiom named in a
	// comment inside the recipe would satisfy a substring test while the grep beside it had been
	// narrowed back, which is the hole this check exists to close.
	commands := makeRecipe(t, "conditions")
	// EVERY grep that looks for an environment condition must carry BOTH idioms, and the recipe runs
	// more than one of them: the names come from one grep and the packages that read each name from
	// another. An idiom dropped from either is a condition or an attribution that quietly goes
	// missing, and counting whole commands would not see it — the recipe's loops put several greps in
	// one command. So the unit is the grep INVOCATION: each one is taken from its `grep` to the next,
	// and recognised by what it hunts (`TM_PLATFORM_TEST_`, an env idiom, `LookPath`) rather than by
	// the idiom it carries, so narrowing one cannot hide it from this check.
	envGreps, binGreps := 0, 0
	for _, cmd := range commands {
		rest := cmd
		for {
			i := strings.Index(rest, "grep")
			if i < 0 {
				break
			}
			rest = rest[i+len("grep"):]
			invocation := rest
			if j := strings.Index(rest, "grep"); j >= 0 {
				invocation = rest[:j]
			}
			// Only a grep that reads the SOURCES carries an idiom; the pipeline's second grep merely
			// cuts the name out of the first one's output and has no files to look in.
			if !strings.Contains(invocation, "--include") {
				continue
			}
			switch {
			case strings.Contains(invocation, "TM_PLATFORM_TEST_") ||
				strings.Contains(invocation, "Getenv") || strings.Contains(invocation, "LookupEnv"):
				envGreps++
				for _, idiom := range []string{"Getenv", "LookupEnv"} {
					if !strings.Contains(invocation, idiom) {
						t.Errorf("a grep of the `conditions` recipe hunts an environment condition and does not look for %s, which this gate reads out of the sources — a test using that idiom would be a host condition the hint never names:\n\tgrep%s",
							idiom, strings.TrimRight(invocation, " \t"))
					}
				}
			case strings.Contains(invocation, "LookPath"):
				binGreps++
			}
		}
	}
	if envGreps < 2 || binGreps < 1 {
		t.Errorf("the `conditions` recipe runs %d environment greps and %d binary greps; it is known to need at least two and one (names, packages, binaries), so a grep has gone missing or this check reads the wrong lines", envGreps, binGreps)
	}
	// Every printed row must be one of the shapes this gate can judge. A row in a third shape is
	// invisible to both directions above — it can be deleted, or replaced by an invented condition,
	// with the gate green — so the shapes themselves are the assertion.
	for _, line := range strings.Split(unset, "\n") {
		if strings.TrimSpace(line) == "" {
			continue
		}
		switch {
		case envRow.MatchString(line), binRow.MatchString(line),
			strings.Contains(line, "systemctl --user"), strings.Contains(line, "CREATEDB"):
		default:
			t.Errorf("the hint prints a row in a shape this gate cannot judge, so nothing checks it in either direction: %q", line)
		}
	}
}

var (
	envRow = regexp.MustCompile(`^\s*(TM_PLATFORM_TEST_[A-Z_]+)\s+(set|UNSET)\s+read by: (.*)$`)
	binRow = regexp.MustCompile(`^\s*(\S+) \(on PATH\)\s+(present|MISSING)\s+read by: (.*)$`)
)

// conditions runs the target the way `check` does, with some variables forced, and returns BOTH the
// output and the environment the recipe actually saw. The second half is what makes the state column
// checkable: this process's own environment is not the recipe's the moment anything is forced, and a
// gate that compared the two would fail under `make check` (where the live knobs are set) while
// passing on a bare `go test`.
func conditions(t *testing.T, env ...string) (string, map[string]string) {
	t.Helper()
	cmd := exec.CommandContext(t.Context(), "make", "--no-print-directory", "-s", "conditions")
	cmd.Dir = zoneRoot
	cmd.Env = append(os.Environ(), env...)
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("make conditions failed: %v\n%s", err, out)
	}
	// Later entries win in a process environment, so the map is built in the same order.
	effective := map[string]string{}
	for _, kv := range cmd.Env {
		if k, v, ok := strings.Cut(kv, "="); ok {
			effective[k] = v
		}
	}
	return string(out), effective
}

// conditionsInTheSources reads the battery's host conditions out of the test files BY PARSING them:
// the environment variables and the binaries, each mapped to the packages whose tests read it.
//
// A parser rather than a regexp, and that is the whole of its value here: it sees CALLS, so a name in
// a comment or in a string that happens to look like one is not a reader, and a helper that switches
// idioms (`os.Getenv` → `os.LookupEnv`) is still seen. The recipe greps; if the recipe's anchors and
// the language disagree, these two sets disagree and the gate says so.
func conditionsInTheSources(t *testing.T) (env, bin map[string][]string) {
	t.Helper()
	env, bin = map[string][]string{}, map[string][]string{}
	fset := token.NewFileSet()
	err := filepath.WalkDir(zoneRoot, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() || !strings.HasSuffix(path, "_test.go") {
			return nil
		}
		file, perr := parser.ParseFile(fset, path, nil, 0)
		if perr != nil {
			t.Fatalf("a test source of this zone does not parse (%s): %v", path, perr)
		}
		pkg, rerr := filepath.Rel(zoneRoot, filepath.Dir(path))
		if rerr != nil {
			return rerr
		}
		add := func(m map[string][]string, name string) {
			for _, p := range m[name] {
				if p == pkg {
					return
				}
			}
			m[name] = append(m[name], pkg)
		}
		ast.Inspect(file, func(n ast.Node) bool {
			call, ok := n.(*ast.CallExpr)
			if !ok || len(call.Args) != 1 {
				return true
			}
			sel, ok := call.Fun.(*ast.SelectorExpr)
			if !ok {
				return true
			}
			pkgIdent, ok := sel.X.(*ast.Ident)
			if !ok {
				return true
			}
			lit, ok := call.Args[0].(*ast.BasicLit)
			if !ok || lit.Kind != token.STRING {
				return true
			}
			arg, uerr := strconv.Unquote(lit.Value)
			if uerr != nil {
				return true
			}
			switch {
			case pkgIdent.Name == "os" && (sel.Sel.Name == "Getenv" || sel.Sel.Name == "LookupEnv") &&
				strings.HasPrefix(arg, "TM_PLATFORM_TEST_"):
				add(env, arg)
			case pkgIdent.Name == "exec" && sel.Sel.Name == "LookPath":
				add(bin, arg)
			}
			return true
		})
		return nil
	})
	if err != nil {
		t.Fatal(err)
	}
	return env, bin
}

func equal(got, want []string) bool {
	if len(got) != len(want) {
		return false
	}
	g, w := append([]string(nil), got...), append([]string(nil), want...)
	sort.Strings(g)
	sort.Strings(w)
	for i := range g {
		if g[i] != w[i] {
			return false
		}
	}
	return true
}

// makeRecipe is the Makefile's recipe for one target, as the SHELL would see it: one string per
// logical command, comments cut, backslash continuations joined.
//
// It lives in one place because two gates read recipes now, and two hand-written parsers of one
// grammar are a second carrier free to drift from the first — the defect this zone spent a day
// removing from its own money path.
//
// ⚠ The recipe is cut the way MAKE cuts it — the target line, then every line that is blank or
// tab-indented — and NOT at the first blank line: a recipe split into two indented lines with a blank
// between them is legal make with identical output, and a gate that failed on it would send the next
// person who tidies this file looking for a defect that is not there. ⚠ And the joining is what makes
// the unit LOGICAL: a pipeline split across a continuation is one command however many lines it
// occupies, so a legal reformat is not a finding.
func makeRecipe(t *testing.T, target string) []string {
	t.Helper()
	recipe, err := os.ReadFile(filepath.Join(zoneRoot, "Makefile"))
	if err != nil {
		t.Fatal(err)
	}
	lines := strings.Split(string(recipe), "\n")
	start := -1
	for i, l := range lines {
		if strings.HasPrefix(l, target+":") {
			start = i
			break
		}
	}
	if start < 0 {
		t.Fatalf("the Makefile declares no `%s` target: the gate that reads it has no source", target)
	}
	var recipeLines []string
	for _, l := range lines[start+1:] {
		if l != "" && !strings.HasPrefix(l, "\t") {
			break
		}
		recipeLines = append(recipeLines, l)
	}
	commands := []string{}
	pending := ""
	for _, l := range recipeLines {
		code := strings.TrimPrefix(l, "\t")
		if head, _, found := strings.Cut(code, "#"); found {
			code = head
		}
		trimmed := strings.TrimRight(code, " \t")
		if strings.HasSuffix(trimmed, `\`) {
			pending += strings.TrimSuffix(trimmed, `\`) + " "
			continue
		}
		commands = append(commands, pending+code)
		pending = ""
	}
	if pending != "" {
		commands = append(commands, pending)
	}
	return commands
}

// ⛔ THE BATTERY MUST NOT BE ABLE TO REPORT CLEANLINESS OVER A LOG IT DID NOT READ.
//
// Every line `make check` prints about the run — the package list, the ALARM rows, the failure list,
// the skip count — is a GREP over one file. Grep answers a MISSING file exactly as it answers a clean
// one: with nothing. So the recipe could reach its final `else` and print «every test ran: no host
// condition was missing» about a run whose log had vanished, which is a bill of health for a
// measurement that never happened.
//
// It is not hypothetical and it is not rare: observed 06.09 on a run with FIVE skips, three
// `grep: .check.log: No such file or directory` followed by that very line. The cause was a FIXED log
// name shared by every run in the directory — and two batteries in one directory is the normal case
// here, the zone and the orchestrator both run one.
//
// So the recipe must TEST the log before it reads it and leave RED when it is not there. This gate
// holds ONE SAFE SHAPE and says so plainly rather than pretending to prove the property: a test of the
// log, then an exit with a LITERAL NON-ZERO code, then the reads. Within that shape the details are
// free — the log may be named per-run or not, the guard may be `-s`, `-f` or `-e`. A different but
// equally correct shape (say, all the reads nested inside `if [ -s log ]; then … else exit 1; fi`)
// will go red here, and that is deliberate: re-shaping the one recipe the whole battery reports
// through should be a decision somebody takes and re-pins, not a silent pass.
//
// Mutation caught: deleting the guard; moving it after the first grep; dropping the `exit` so the
// recipe notices and carries on; and — the one the first edition missed — `exit 0`, which satisfies
// "there is an exit" while returning SUCCESS from a run whose evidence is gone.
func TestTheBatteryCannotReportCleanlinessWithoutItsLog(t *testing.T) {
	steps := []string{}
	for _, cmd := range makeRecipe(t, "check") {
		for _, s := range strings.Split(cmd, ";") {
			steps = append(steps, strings.TrimSpace(s))
		}
	}
	// WHERE THE LOG COMES FROM, read out of the recipe rather than assumed: the redirect of the test
	// run names it, so a rename cannot make this gate look at the wrong file and pass.
	logRef := ""
	for _, s := range steps {
		if !strings.Contains(s, "test ./...") {
			continue
		}
		if _, after, found := strings.Cut(s, "> "); found {
			logRef = strings.Trim(strings.Fields(after)[0], `"`)
		}
	}
	if logRef == "" {
		t.Fatal("the `check` recipe does not redirect its test run to a file this gate can find: " +
			"either the battery stopped keeping a log, or its shape moved and this gate now reads nothing")
	}
	// One shell command per step, with the leading keyword taken off, so that `then exit 1` is read as
	// `exit 1` and `then echo …` as an echo. Done once here because every check below asks the same
	// question of the same string, and two strippers would drift.
	cmds := make([]string, len(steps))
	for i, s := range steps {
		cmd := strings.TrimSpace(s)
		for {
			stripped := cmd
			for _, kw := range []string{"then", "else", "do", "{"} {
				if rest, found := strings.CutPrefix(stripped, kw); found && strings.TrimLeft(rest, " \t") != rest {
					stripped = strings.TrimLeft(rest, " \t")
				}
			}
			if stripped == cmd {
				break
			}
			cmd = stripped
		}
		cmds[i] = cmd
	}
	reads, guard, exits := -1, -1, -1
	exitCode := ""
	for i, cmd := range cmds {
		if !strings.Contains(cmd, logRef) {
			continue
		}
		// A grep INVOCATION, not the word: the recipe's own guard EXPLAINS itself in an `echo`, and the
		// explanation says «grep». The first edition of this gate counted that sentence as a read and
		// reported the guard as too late — which is the same substring-versus-invocation trap the
		// sibling check above spends a paragraph on, walked into by the test that quotes it.
		greps := strings.Contains(cmd, "$$(grep") ||
			(!strings.HasPrefix(cmd, "echo") && strings.Contains(cmd, "grep"))
		switch {
		case greps && reads < 0:
			reads = i
		case strings.HasPrefix(cmd, "if [") || strings.HasPrefix(cmd, "[ "):
			if guard < 0 && (strings.Contains(cmd, "-s ") || strings.Contains(cmd, "-f ") || strings.Contains(cmd, "-e ")) {
				guard = i
			}
		}
	}
	// ⛔ THE EXIT'S CODE, not merely its presence — and this half was MISSING from the first edition.
	// `exit 0` after the guard satisfied "there is an exit" while leaving the recipe to announce that
	// the evidence is gone and then RETURN SUCCESS. That is the very defect this gate exists for,
	// wearing different clothes and worse in one respect: the old false clean bill was a LINE a person
	// could catch, this one is the EXIT CODE, which is what CI and the landing read. Measured, not
	// argued: with `exit 1` mutated to `exit 0` this gate stayed green and `make check` returned 0
	// while printing «THE BATTERY LEFT NO LOG».
	//
	// A literal is required. `exit` bare carries the previous command's status — here the status of an
	// `echo`, which is zero — and `exit $var` cannot be judged from the recipe at all; both are refused
	// rather than guessed, because a guard whose code is not readable here is a guard this gate cannot
	// promise anything about.
	for i, cmd := range cmds {
		if guard < 0 || i <= guard {
			continue
		}
		if rest, found := strings.CutPrefix(cmd, "exit"); found {
			exits, exitCode = i, strings.TrimSpace(rest)
			break
		}
	}
	if reads < 0 {
		t.Fatalf("no step of the `check` recipe greps %s: the gate is reading the wrong target or the "+
			"battery no longer reports what the log holds", logRef)
	}
	if guard < 0 {
		t.Fatalf("the `check` recipe reads %s at step %d and never TESTS that it is there. A missing log "+
			"is indistinguishable from a clean one to grep, so the recipe will print «every test ran» "+
			"over a run it did not measure — the defect this gate exists for", logRef, reads)
	}
	if guard > reads {
		t.Errorf("the `check` recipe tests %s at step %d but has already read it at step %d: the first "+
			"read is where the false clean bill starts, so the guard must come before it", logRef, guard, reads)
	}
	if exits < 0 || exits > reads {
		t.Errorf("the `check` recipe tests %s at step %d but does not exit before its first read at step "+
			"%d: noticing the absence and carrying on prints the same bill of health as never looking",
			logRef, guard, reads)
		return
	}
	code, err := strconv.Atoi(exitCode)
	if err != nil {
		t.Errorf("the guard of the `check` recipe exits with %q, which this gate cannot read as a number: "+
			"a bare `exit` carries the previous command's status — an `echo`, so zero — and a variable "+
			"cannot be judged from the recipe. Write a literal, so that what the recipe promises is what "+
			"a reader can check", "exit "+exitCode)
		return
	}
	if code == 0 {
		t.Errorf("the guard of the `check` recipe says the log is missing and then exits %d, which is "+
			"SUCCESS: the battery announces that its evidence is gone and reports a green run. That is "+
			"the same false clean bill this gate exists for, moved from a LINE a person reads into the "+
			"EXIT CODE a machine reads — and the landing reads the code", code)
	}
}