package gates import ( "go/ast" "go/parser" "go/token" "io/fs" "os" "os/exec" "path/filepath" "regexp" "sort" "strconv" "strings" "testing" ) // The battery's hint under its skips must name EVERY host condition the tests read, must say for each // whether THIS host meets it, and must name what it opens (PD-374): a hint that names one condition of // several sends a session to a knob that is already on, and the skips it still gets read as the zone's // normal. The Makefile's `conditions` target derives that list from the test sources; this test holds // it to that. // // ⚠ The two derive the same facts by DIFFERENT MEANS, and that is the point rather than a detail: the // recipe greps, this gate PARSES the sources and looks at call expressions. A name that lives only in // a comment is a reader to the grep and not to the parser, and a recipe narrowed to fewer idioms than // the tests use makes the two disagree — the disagreement is what goes red. Holding both to one // anchor would make the gate a mirror of the recipe: green whatever either says. // // ⚠ WHAT THE WALK DOES NOT SEE, said here because the next author reads this file and not the report: // it matches a call with a LITERAL argument — `os.Getenv("TM_PLATFORM_TEST_…")`, its `os.LookupEnv` // twin, `exec.LookPath("…")` — in a `_test.go` of this module. A name arriving through a constant, a // variable or a helper's parameter, an aliased or dot-imported `os`, or a helper living in a non-test // file, is invisible to the walk AND to the recipe, so both fall silent together and PD-374 returns at // full strength. `systemdOrSkip` is exactly that shape and is why the systemd row is hand-written on // both sides. The class is narrowed here, not closed. // // What it asserts, for EVERY row and not for a sample: the NAME appears; the STATE is the host's // answer, taken here independently (the environment for a variable, `exec.LookPath` for a binary, the // same systemctl probe `systemdOrSkip` makes); the PACKAGES are exactly those whose tests read it; and // nothing is printed that the sources do not read. // // Mutation caught: replacing the derivation with a literal list; freezing any state column to a word; // printing an invented condition; attributing a condition to a package that does not read it; dropping // the systemd probe. func TestTheBatteryNamesEveryHostConditionItsTestsRead(t *testing.T) { if _, err := exec.LookPath("make"); err != nil { t.Skip("make is not on this host: the battery's hint cannot be exercised") } env, bin := conditionsInTheSources(t) if len(env) < 2 || len(bin) < 2 { t.Fatalf("parsed %d environment conditions and %d binaries out of the test sources, fewer than the battery is known to read: this walk reads the wrong tree", len(env), len(bin)) } // One variable is forced in both directions so the state column is proven to follow the host // rather than to print a word; every other row is checked against the host as it stands. unset, unsetEnv := conditions(t, "TM_PLATFORM_TEST_DSN=") set, _ := conditions(t, "TM_PLATFORM_TEST_DSN=postgres://somewhere") printedEnv := map[string][]string{} printedBin := map[string][]string{} for _, line := range strings.Split(unset, "\n") { if m := envRow.FindStringSubmatch(line); m != nil { printedEnv[m[1]] = strings.Fields(m[3]) // The state, against the environment THE RECIPE WAS GIVEN — not this process's own, which // differs the moment a variable is forced for the run, and not "is it exported": NON-EMPTY // is the fact. That is what the recipe asks (`[ -n "$(printenv …)" ]`) and, more to the // point, what every gated test asks — they skip on `os.Getenv(…) == ""`. A wrapper that // always exports a knob and leaves it empty (`TM_PLATFORM_TEST_DSN="${DSN:-}"` in a CI // script) is a host under which the live suite skips, and a gate that called that "set" // would demand the hint lie about it: red for a state the zone did not cause. live := unsetEnv[m[1]] != "" if want := map[bool]string{true: "set", false: "UNSET"}[live]; m[2] != want { t.Errorf("%s is printed as %q and this host says %q", m[1], m[2], want) } continue } if m := binRow.FindStringSubmatch(line); m != nil { printedBin[m[1]] = strings.Fields(m[3]) _, err := exec.LookPath(m[1]) if want := map[bool]string{true: "present", false: "MISSING"}[err == nil]; m[2] != want { t.Errorf("binary %s is printed as %q and this host says %q", m[1], m[2], want) } } } // Both directions: everything the sources read is printed, and everything printed is read. for name, want := range env { got, ok := printedEnv[name] if !ok { t.Errorf("the list does not name %s, which a test reads:\n%s", name, unset) continue } if !equal(got, want) { t.Errorf("%s is announced as read by %v, and the sources say %v", name, got, want) } } for name := range printedEnv { if _, ok := env[name]; !ok { t.Errorf("the list names %s, and no test reads it: a condition nobody reads sends a session to a knob that opens nothing", name) } } for name, want := range bin { got, ok := printedBin[name] if !ok { t.Errorf("the list does not name the binary %s, which a test helper looks up before it runs:\n%s", name, unset) continue } if !equal(got, want) { t.Errorf("binary %s is announced as read by %v, and the sources say %v", name, got, want) } } for name := range printedBin { if _, ok := bin[name]; !ok { t.Errorf("the list names the binary %s, and no test helper looks it up", name) } } if !regexp.MustCompile(`(?m)^\s*TM_PLATFORM_TEST_DSN\s+set\b`).MatchString(set) { t.Errorf("with the variable set the list did not say set:\n%s", set) } // The systemd row carries no name a walk can find — it is a helper's own probe — so it is checked // against that same probe, run here. m := regexp.MustCompile(`(?m)^\s*systemctl --user\s+(reachable|UNREACHABLE)\s+read by: (\S+).*systemdOrSkip`).FindStringSubmatch(unset) if m == nil { t.Fatalf("the reachable-user-systemd condition (systemdOrSkip) is not in the list:\n%s", unset) } reachable := exec.CommandContext(t.Context(), "systemctl", "--user", "show", "--property=Version").Run() == nil if want := map[bool]string{true: "reachable", false: "UNREACHABLE"}[reachable]; m[1] != want { t.Errorf("the systemd condition is printed as %q and this host answers %q", m[1], want) } if !strings.Contains(m[2], "internal/runner") { t.Errorf("the systemd condition is announced as read by %q, and systemdOrSkip lives in internal/runner", m[2]) } // The idioms themselves, because a set comparison can only speak about conditions that EXIST: the // day a test starts reading one through an idiom the recipe does not grep for, that condition // vanishes from the hint and BOTH sides fall silent together — the walk would not see it either if // the walk were narrowed to match. So the recipe's own anchors are read here and held to the ones // this gate understands; narrowing either is what goes red, before any test uses the idiom. // The GREP EXPRESSIONS the recipe runs, not words that appear near them: an idiom named in a // comment inside the recipe would satisfy a substring test while the grep beside it had been // narrowed back, which is the hole this check exists to close. commands := makeRecipe(t, "conditions") // EVERY grep that looks for an environment condition must carry BOTH idioms, and the recipe runs // more than one of them: the names come from one grep and the packages that read each name from // another. An idiom dropped from either is a condition or an attribution that quietly goes // missing, and counting whole commands would not see it — the recipe's loops put several greps in // one command. So the unit is the grep INVOCATION: each one is taken from its `grep` to the next, // and recognised by what it hunts (`TM_PLATFORM_TEST_`, an env idiom, `LookPath`) rather than by // the idiom it carries, so narrowing one cannot hide it from this check. envGreps, binGreps := 0, 0 for _, cmd := range commands { rest := cmd for { i := strings.Index(rest, "grep") if i < 0 { break } rest = rest[i+len("grep"):] invocation := rest if j := strings.Index(rest, "grep"); j >= 0 { invocation = rest[:j] } // Only a grep that reads the SOURCES carries an idiom; the pipeline's second grep merely // cuts the name out of the first one's output and has no files to look in. if !strings.Contains(invocation, "--include") { continue } switch { case strings.Contains(invocation, "TM_PLATFORM_TEST_") || strings.Contains(invocation, "Getenv") || strings.Contains(invocation, "LookupEnv"): envGreps++ for _, idiom := range []string{"Getenv", "LookupEnv"} { if !strings.Contains(invocation, idiom) { t.Errorf("a grep of the `conditions` recipe hunts an environment condition and does not look for %s, which this gate reads out of the sources — a test using that idiom would be a host condition the hint never names:\n\tgrep%s", idiom, strings.TrimRight(invocation, " \t")) } } case strings.Contains(invocation, "LookPath"): binGreps++ } } } if envGreps < 2 || binGreps < 1 { t.Errorf("the `conditions` recipe runs %d environment greps and %d binary greps; it is known to need at least two and one (names, packages, binaries), so a grep has gone missing or this check reads the wrong lines", envGreps, binGreps) } // Every printed row must be one of the shapes this gate can judge. A row in a third shape is // invisible to both directions above — it can be deleted, or replaced by an invented condition, // with the gate green — so the shapes themselves are the assertion. for _, line := range strings.Split(unset, "\n") { if strings.TrimSpace(line) == "" { continue } switch { case envRow.MatchString(line), binRow.MatchString(line), strings.Contains(line, "systemctl --user"), strings.Contains(line, "CREATEDB"): default: t.Errorf("the hint prints a row in a shape this gate cannot judge, so nothing checks it in either direction: %q", line) } } } var ( envRow = regexp.MustCompile(`^\s*(TM_PLATFORM_TEST_[A-Z_]+)\s+(set|UNSET)\s+read by: (.*)$`) binRow = regexp.MustCompile(`^\s*(\S+) \(on PATH\)\s+(present|MISSING)\s+read by: (.*)$`) ) // conditions runs the target the way `check` does, with some variables forced, and returns BOTH the // output and the environment the recipe actually saw. The second half is what makes the state column // checkable: this process's own environment is not the recipe's the moment anything is forced, and a // gate that compared the two would fail under `make check` (where the live knobs are set) while // passing on a bare `go test`. func conditions(t *testing.T, env ...string) (string, map[string]string) { t.Helper() cmd := exec.CommandContext(t.Context(), "make", "--no-print-directory", "-s", "conditions") cmd.Dir = zoneRoot cmd.Env = append(os.Environ(), env...) out, err := cmd.CombinedOutput() if err != nil { t.Fatalf("make conditions failed: %v\n%s", err, out) } // Later entries win in a process environment, so the map is built in the same order. effective := map[string]string{} for _, kv := range cmd.Env { if k, v, ok := strings.Cut(kv, "="); ok { effective[k] = v } } return string(out), effective } // conditionsInTheSources reads the battery's host conditions out of the test files BY PARSING them: // the environment variables and the binaries, each mapped to the packages whose tests read it. // // A parser rather than a regexp, and that is the whole of its value here: it sees CALLS, so a name in // a comment or in a string that happens to look like one is not a reader, and a helper that switches // idioms (`os.Getenv` → `os.LookupEnv`) is still seen. The recipe greps; if the recipe's anchors and // the language disagree, these two sets disagree and the gate says so. func conditionsInTheSources(t *testing.T) (env, bin map[string][]string) { t.Helper() env, bin = map[string][]string{}, map[string][]string{} fset := token.NewFileSet() err := filepath.WalkDir(zoneRoot, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } if d.IsDir() || !strings.HasSuffix(path, "_test.go") { return nil } file, perr := parser.ParseFile(fset, path, nil, 0) if perr != nil { t.Fatalf("a test source of this zone does not parse (%s): %v", path, perr) } pkg, rerr := filepath.Rel(zoneRoot, filepath.Dir(path)) if rerr != nil { return rerr } add := func(m map[string][]string, name string) { for _, p := range m[name] { if p == pkg { return } } m[name] = append(m[name], pkg) } ast.Inspect(file, func(n ast.Node) bool { call, ok := n.(*ast.CallExpr) if !ok || len(call.Args) != 1 { return true } sel, ok := call.Fun.(*ast.SelectorExpr) if !ok { return true } pkgIdent, ok := sel.X.(*ast.Ident) if !ok { return true } lit, ok := call.Args[0].(*ast.BasicLit) if !ok || lit.Kind != token.STRING { return true } arg, uerr := strconv.Unquote(lit.Value) if uerr != nil { return true } switch { case pkgIdent.Name == "os" && (sel.Sel.Name == "Getenv" || sel.Sel.Name == "LookupEnv") && strings.HasPrefix(arg, "TM_PLATFORM_TEST_"): add(env, arg) case pkgIdent.Name == "exec" && sel.Sel.Name == "LookPath": add(bin, arg) } return true }) return nil }) if err != nil { t.Fatal(err) } return env, bin } func equal(got, want []string) bool { if len(got) != len(want) { return false } g, w := append([]string(nil), got...), append([]string(nil), want...) sort.Strings(g) sort.Strings(w) for i := range g { if g[i] != w[i] { return false } } return true } // makeRecipe is the Makefile's recipe for one target, as the SHELL would see it: one string per // logical command, comments cut, backslash continuations joined. // // It lives in one place because two gates read recipes now, and two hand-written parsers of one // grammar are a second carrier free to drift from the first — the defect this zone spent a day // removing from its own money path. // // ⚠ The recipe is cut the way MAKE cuts it — the target line, then every line that is blank or // tab-indented — and NOT at the first blank line: a recipe split into two indented lines with a blank // between them is legal make with identical output, and a gate that failed on it would send the next // person who tidies this file looking for a defect that is not there. ⚠ And the joining is what makes // the unit LOGICAL: a pipeline split across a continuation is one command however many lines it // occupies, so a legal reformat is not a finding. func makeRecipe(t *testing.T, target string) []string { t.Helper() recipe, err := os.ReadFile(filepath.Join(zoneRoot, "Makefile")) if err != nil { t.Fatal(err) } lines := strings.Split(string(recipe), "\n") start := -1 for i, l := range lines { if strings.HasPrefix(l, target+":") { start = i break } } if start < 0 { t.Fatalf("the Makefile declares no `%s` target: the gate that reads it has no source", target) } var recipeLines []string for _, l := range lines[start+1:] { if l != "" && !strings.HasPrefix(l, "\t") { break } recipeLines = append(recipeLines, l) } commands := []string{} pending := "" for _, l := range recipeLines { code := strings.TrimPrefix(l, "\t") if head, _, found := strings.Cut(code, "#"); found { code = head } trimmed := strings.TrimRight(code, " \t") if strings.HasSuffix(trimmed, `\`) { pending += strings.TrimSuffix(trimmed, `\`) + " " continue } commands = append(commands, pending+code) pending = "" } if pending != "" { commands = append(commands, pending) } return commands } // ⛔ THE BATTERY MUST NOT BE ABLE TO REPORT CLEANLINESS OVER A LOG IT DID NOT READ. // // Every line `make check` prints about the run — the package list, the ALARM rows, the failure list, // the skip count — is a GREP over one file. Grep answers a MISSING file exactly as it answers a clean // one: with nothing. So the recipe could reach its final `else` and print «every test ran: no host // condition was missing» about a run whose log had vanished, which is a bill of health for a // measurement that never happened. // // It is not hypothetical and it is not rare: observed 06.09 on a run with FIVE skips, three // `grep: .check.log: No such file or directory` followed by that very line. The cause was a FIXED log // name shared by every run in the directory — and two batteries in one directory is the normal case // here, the zone and the orchestrator both run one. // // So the recipe must TEST the log before it reads it and leave RED when it is not there. This gate // holds ONE SAFE SHAPE and says so plainly rather than pretending to prove the property: a test of the // log, then an exit with a LITERAL NON-ZERO code, then the reads. Within that shape the details are // free — the log may be named per-run or not, the guard may be `-s`, `-f` or `-e`. A different but // equally correct shape (say, all the reads nested inside `if [ -s log ]; then … else exit 1; fi`) // will go red here, and that is deliberate: re-shaping the one recipe the whole battery reports // through should be a decision somebody takes and re-pins, not a silent pass. // // Mutation caught: deleting the guard; moving it after the first grep; dropping the `exit` so the // recipe notices and carries on; and — the one the first edition missed — `exit 0`, which satisfies // "there is an exit" while returning SUCCESS from a run whose evidence is gone. func TestTheBatteryCannotReportCleanlinessWithoutItsLog(t *testing.T) { steps := []string{} for _, cmd := range makeRecipe(t, "check") { for _, s := range strings.Split(cmd, ";") { steps = append(steps, strings.TrimSpace(s)) } } // WHERE THE LOG COMES FROM, read out of the recipe rather than assumed: the redirect of the test // run names it, so a rename cannot make this gate look at the wrong file and pass. logRef := "" for _, s := range steps { if !strings.Contains(s, "test ./...") { continue } if _, after, found := strings.Cut(s, "> "); found { logRef = strings.Trim(strings.Fields(after)[0], `"`) } } if logRef == "" { t.Fatal("the `check` recipe does not redirect its test run to a file this gate can find: " + "either the battery stopped keeping a log, or its shape moved and this gate now reads nothing") } // One shell command per step, with the leading keyword taken off, so that `then exit 1` is read as // `exit 1` and `then echo …` as an echo. Done once here because every check below asks the same // question of the same string, and two strippers would drift. cmds := make([]string, len(steps)) for i, s := range steps { cmd := strings.TrimSpace(s) for { stripped := cmd for _, kw := range []string{"then", "else", "do", "{"} { if rest, found := strings.CutPrefix(stripped, kw); found && strings.TrimLeft(rest, " \t") != rest { stripped = strings.TrimLeft(rest, " \t") } } if stripped == cmd { break } cmd = stripped } cmds[i] = cmd } reads, guard, exits := -1, -1, -1 exitCode := "" for i, cmd := range cmds { if !strings.Contains(cmd, logRef) { continue } // A grep INVOCATION, not the word: the recipe's own guard EXPLAINS itself in an `echo`, and the // explanation says «grep». The first edition of this gate counted that sentence as a read and // reported the guard as too late — which is the same substring-versus-invocation trap the // sibling check above spends a paragraph on, walked into by the test that quotes it. greps := strings.Contains(cmd, "$$(grep") || (!strings.HasPrefix(cmd, "echo") && strings.Contains(cmd, "grep")) switch { case greps && reads < 0: reads = i case strings.HasPrefix(cmd, "if [") || strings.HasPrefix(cmd, "[ "): if guard < 0 && (strings.Contains(cmd, "-s ") || strings.Contains(cmd, "-f ") || strings.Contains(cmd, "-e ")) { guard = i } } } // ⛔ THE EXIT'S CODE, not merely its presence — and this half was MISSING from the first edition. // `exit 0` after the guard satisfied "there is an exit" while leaving the recipe to announce that // the evidence is gone and then RETURN SUCCESS. That is the very defect this gate exists for, // wearing different clothes and worse in one respect: the old false clean bill was a LINE a person // could catch, this one is the EXIT CODE, which is what CI and the landing read. Measured, not // argued: with `exit 1` mutated to `exit 0` this gate stayed green and `make check` returned 0 // while printing «THE BATTERY LEFT NO LOG». // // A literal is required. `exit` bare carries the previous command's status — here the status of an // `echo`, which is zero — and `exit $var` cannot be judged from the recipe at all; both are refused // rather than guessed, because a guard whose code is not readable here is a guard this gate cannot // promise anything about. for i, cmd := range cmds { if guard < 0 || i <= guard { continue } if rest, found := strings.CutPrefix(cmd, "exit"); found { exits, exitCode = i, strings.TrimSpace(rest) break } } if reads < 0 { t.Fatalf("no step of the `check` recipe greps %s: the gate is reading the wrong target or the "+ "battery no longer reports what the log holds", logRef) } if guard < 0 { t.Fatalf("the `check` recipe reads %s at step %d and never TESTS that it is there. A missing log "+ "is indistinguishable from a clean one to grep, so the recipe will print «every test ran» "+ "over a run it did not measure — the defect this gate exists for", logRef, reads) } if guard > reads { t.Errorf("the `check` recipe tests %s at step %d but has already read it at step %d: the first "+ "read is where the false clean bill starts, so the guard must come before it", logRef, guard, reads) } if exits < 0 || exits > reads { t.Errorf("the `check` recipe tests %s at step %d but does not exit before its first read at step "+ "%d: noticing the absence and carrying on prints the same bill of health as never looking", logRef, guard, reads) return } code, err := strconv.Atoi(exitCode) if err != nil { t.Errorf("the guard of the `check` recipe exits with %q, which this gate cannot read as a number: "+ "a bare `exit` carries the previous command's status — an `echo`, so zero — and a variable "+ "cannot be judged from the recipe. Write a literal, so that what the recipe promises is what "+ "a reader can check", "exit "+exitCode) return } if code == 0 { t.Errorf("the guard of the `check` recipe says the log is missing and then exits %d, which is "+ "SUCCESS: the battery announces that its evidence is gone and reports a green run. That is "+ "the same false clean bill this gate exists for, moved from a LINE a person reads into the "+ "EXIT CODE a machine reads — and the landing reads the code", code) } }