package gates import ( "go/ast" "go/parser" "go/token" "io/fs" "os" "os/exec" "path/filepath" "regexp" "sort" "strconv" "strings" "testing" ) // The battery's hint under its skips must name EVERY host condition the tests read, must say for each // whether THIS host meets it, and must name what it opens (PD-374): a hint that names one condition of // several sends a session to a knob that is already on, and the skips it still gets read as the zone's // normal. The Makefile's `conditions` target derives that list from the test sources; this test holds // it to that. // // ⚠ The two derive the same facts by DIFFERENT MEANS, and that is the point rather than a detail: the // recipe greps, this gate PARSES the sources and looks at call expressions. A name that lives only in // a comment is a reader to the grep and not to the parser, and a recipe narrowed to fewer idioms than // the tests use makes the two disagree — the disagreement is what goes red. Holding both to one // anchor would make the gate a mirror of the recipe: green whatever either says. // // ⚠ WHAT THE WALK DOES NOT SEE, said here because the next author reads this file and not the report: // it matches a call with a LITERAL argument — `os.Getenv("TM_PLATFORM_TEST_…")`, its `os.LookupEnv` // twin, `exec.LookPath("…")` — in a `_test.go` of this module. A name arriving through a constant, a // variable or a helper's parameter, an aliased or dot-imported `os`, or a helper living in a non-test // file, is invisible to the walk AND to the recipe, so both fall silent together and PD-374 returns at // full strength. `systemdOrSkip` is exactly that shape and is why the systemd row is hand-written on // both sides. The class is narrowed here, not closed. // // What it asserts, for EVERY row and not for a sample: the NAME appears; the STATE is the host's // answer, taken here independently (the environment for a variable, `exec.LookPath` for a binary, the // same systemctl probe `systemdOrSkip` makes); the PACKAGES are exactly those whose tests read it; and // nothing is printed that the sources do not read. // // Mutation caught: replacing the derivation with a literal list; freezing any state column to a word; // printing an invented condition; attributing a condition to a package that does not read it; dropping // the systemd probe. func TestTheBatteryNamesEveryHostConditionItsTestsRead(t *testing.T) { if _, err := exec.LookPath("make"); err != nil { t.Skip("make is not on this host: the battery's hint cannot be exercised") } env, bin := conditionsInTheSources(t) if len(env) < 2 || len(bin) < 2 { t.Fatalf("parsed %d environment conditions and %d binaries out of the test sources, fewer than the battery is known to read: this walk reads the wrong tree", len(env), len(bin)) } // One variable is forced in both directions so the state column is proven to follow the host // rather than to print a word; every other row is checked against the host as it stands. unset, unsetEnv := conditions(t, "TM_PLATFORM_TEST_DSN=") set, _ := conditions(t, "TM_PLATFORM_TEST_DSN=postgres://somewhere") printedEnv := map[string][]string{} printedBin := map[string][]string{} for _, line := range strings.Split(unset, "\n") { if m := envRow.FindStringSubmatch(line); m != nil { printedEnv[m[1]] = strings.Fields(m[3]) // The state, against the environment THE RECIPE WAS GIVEN — not this process's own, which // differs the moment a variable is forced for the run, and not "is it exported": NON-EMPTY // is the fact. That is what the recipe asks (`[ -n "$(printenv …)" ]`) and, more to the // point, what every gated test asks — they skip on `os.Getenv(…) == ""`. A wrapper that // always exports a knob and leaves it empty (`TM_PLATFORM_TEST_DSN="${DSN:-}"` in a CI // script) is a host under which the live suite skips, and a gate that called that "set" // would demand the hint lie about it: red for a state the zone did not cause. live := unsetEnv[m[1]] != "" if want := map[bool]string{true: "set", false: "UNSET"}[live]; m[2] != want { t.Errorf("%s is printed as %q and this host says %q", m[1], m[2], want) } continue } if m := binRow.FindStringSubmatch(line); m != nil { printedBin[m[1]] = strings.Fields(m[3]) _, err := exec.LookPath(m[1]) if want := map[bool]string{true: "present", false: "MISSING"}[err == nil]; m[2] != want { t.Errorf("binary %s is printed as %q and this host says %q", m[1], m[2], want) } } } // Both directions: everything the sources read is printed, and everything printed is read. for name, want := range env { got, ok := printedEnv[name] if !ok { t.Errorf("the list does not name %s, which a test reads:\n%s", name, unset) continue } if !equal(got, want) { t.Errorf("%s is announced as read by %v, and the sources say %v", name, got, want) } } for name := range printedEnv { if _, ok := env[name]; !ok { t.Errorf("the list names %s, and no test reads it: a condition nobody reads sends a session to a knob that opens nothing", name) } } for name, want := range bin { got, ok := printedBin[name] if !ok { t.Errorf("the list does not name the binary %s, which a test helper looks up before it runs:\n%s", name, unset) continue } if !equal(got, want) { t.Errorf("binary %s is announced as read by %v, and the sources say %v", name, got, want) } } for name := range printedBin { if _, ok := bin[name]; !ok { t.Errorf("the list names the binary %s, and no test helper looks it up", name) } } if !regexp.MustCompile(`(?m)^\s*TM_PLATFORM_TEST_DSN\s+set\b`).MatchString(set) { t.Errorf("with the variable set the list did not say set:\n%s", set) } // The systemd row carries no name a walk can find — it is a helper's own probe — so it is checked // against that same probe, run here. m := regexp.MustCompile(`(?m)^\s*systemctl --user\s+(reachable|UNREACHABLE)\s+read by: (\S+).*systemdOrSkip`).FindStringSubmatch(unset) if m == nil { t.Fatalf("the reachable-user-systemd condition (systemdOrSkip) is not in the list:\n%s", unset) } reachable := exec.CommandContext(t.Context(), "systemctl", "--user", "show", "--property=Version").Run() == nil if want := map[bool]string{true: "reachable", false: "UNREACHABLE"}[reachable]; m[1] != want { t.Errorf("the systemd condition is printed as %q and this host answers %q", m[1], want) } if !strings.Contains(m[2], "internal/runner") { t.Errorf("the systemd condition is announced as read by %q, and systemdOrSkip lives in internal/runner", m[2]) } // The idioms themselves, because a set comparison can only speak about conditions that EXIST: the // day a test starts reading one through an idiom the recipe does not grep for, that condition // vanishes from the hint and BOTH sides fall silent together — the walk would not see it either if // the walk were narrowed to match. So the recipe's own anchors are read here and held to the ones // this gate understands; narrowing either is what goes red, before any test uses the idiom. // The GREP EXPRESSIONS the recipe runs, not words that appear near them: an idiom named in a // comment inside the recipe would satisfy a substring test while the grep beside it had been // narrowed back, which is the hole this check exists to close. commands := makeRecipe(t, "conditions") // EVERY grep that looks for an environment condition must carry BOTH idioms, and the recipe runs // more than one of them: the names come from one grep and the packages that read each name from // another. An idiom dropped from either is a condition or an attribution that quietly goes // missing, and counting whole commands would not see it — the recipe's loops put several greps in // one command. So the unit is the grep INVOCATION: each one is taken from its `grep` to the next, // and recognised by what it hunts (`TM_PLATFORM_TEST_`, an env idiom, `LookPath`) rather than by // the idiom it carries, so narrowing one cannot hide it from this check. envGreps, binGreps := 0, 0 for _, cmd := range commands { rest := cmd for { i := strings.Index(rest, "grep") if i < 0 { break } rest = rest[i+len("grep"):] invocation := rest if j := strings.Index(rest, "grep"); j >= 0 { invocation = rest[:j] } // Only a grep that reads the SOURCES carries an idiom; the pipeline's second grep merely // cuts the name out of the first one's output and has no files to look in. if !strings.Contains(invocation, "--include") { continue } switch { case strings.Contains(invocation, "TM_PLATFORM_TEST_") || strings.Contains(invocation, "Getenv") || strings.Contains(invocation, "LookupEnv"): envGreps++ for _, idiom := range []string{"Getenv", "LookupEnv"} { if !strings.Contains(invocation, idiom) { t.Errorf("a grep of the `conditions` recipe hunts an environment condition and does not look for %s, which this gate reads out of the sources — a test using that idiom would be a host condition the hint never names:\n\tgrep%s", idiom, strings.TrimRight(invocation, " \t")) } } case strings.Contains(invocation, "LookPath"): binGreps++ } } } if envGreps < 2 || binGreps < 1 { t.Errorf("the `conditions` recipe runs %d environment greps and %d binary greps; it is known to need at least two and one (names, packages, binaries), so a grep has gone missing or this check reads the wrong lines", envGreps, binGreps) } // Every printed row must be one of the shapes this gate can judge. A row in a third shape is // invisible to both directions above — it can be deleted, or replaced by an invented condition, // with the gate green — so the shapes themselves are the assertion. for _, line := range strings.Split(unset, "\n") { if strings.TrimSpace(line) == "" { continue } switch { case envRow.MatchString(line), binRow.MatchString(line), strings.Contains(line, "systemctl --user"), strings.Contains(line, "CREATEDB"): default: t.Errorf("the hint prints a row in a shape this gate cannot judge, so nothing checks it in either direction: %q", line) } } } var ( envRow = regexp.MustCompile(`^\s*(TM_PLATFORM_TEST_[A-Z_]+)\s+(set|UNSET)\s+read by: (.*)$`) binRow = regexp.MustCompile(`^\s*(\S+) \(on PATH\)\s+(present|MISSING)\s+read by: (.*)$`) ) // conditions runs the target the way `check` does, with some variables forced, and returns BOTH the // output and the environment the recipe actually saw. The second half is what makes the state column // checkable: this process's own environment is not the recipe's the moment anything is forced, and a // gate that compared the two would fail under `make check` (where the live knobs are set) while // passing on a bare `go test`. func conditions(t *testing.T, env ...string) (string, map[string]string) { t.Helper() cmd := exec.CommandContext(t.Context(), "make", "--no-print-directory", "-s", "conditions") cmd.Dir = zoneRoot cmd.Env = append(os.Environ(), env...) out, err := cmd.CombinedOutput() if err != nil { t.Fatalf("make conditions failed: %v\n%s", err, out) } // Later entries win in a process environment, so the map is built in the same order. effective := map[string]string{} for _, kv := range cmd.Env { if k, v, ok := strings.Cut(kv, "="); ok { effective[k] = v } } return string(out), effective } // conditionsInTheSources reads the battery's host conditions out of the test files BY PARSING them: // the environment variables and the binaries, each mapped to the packages whose tests read it. // // A parser rather than a regexp, and that is the whole of its value here: it sees CALLS, so a name in // a comment or in a string that happens to look like one is not a reader, and a helper that switches // idioms (`os.Getenv` → `os.LookupEnv`) is still seen. The recipe greps; if the recipe's anchors and // the language disagree, these two sets disagree and the gate says so. func conditionsInTheSources(t *testing.T) (env, bin map[string][]string) { t.Helper() env, bin = map[string][]string{}, map[string][]string{} fset := token.NewFileSet() err := filepath.WalkDir(zoneRoot, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } if d.IsDir() || !strings.HasSuffix(path, "_test.go") { return nil } file, perr := parser.ParseFile(fset, path, nil, 0) if perr != nil { t.Fatalf("a test source of this zone does not parse (%s): %v", path, perr) } pkg, rerr := filepath.Rel(zoneRoot, filepath.Dir(path)) if rerr != nil { return rerr } add := func(m map[string][]string, name string) { for _, p := range m[name] { if p == pkg { return } } m[name] = append(m[name], pkg) } ast.Inspect(file, func(n ast.Node) bool { call, ok := n.(*ast.CallExpr) if !ok || len(call.Args) != 1 { return true } sel, ok := call.Fun.(*ast.SelectorExpr) if !ok { return true } pkgIdent, ok := sel.X.(*ast.Ident) if !ok { return true } lit, ok := call.Args[0].(*ast.BasicLit) if !ok || lit.Kind != token.STRING { return true } arg, uerr := strconv.Unquote(lit.Value) if uerr != nil { return true } switch { case pkgIdent.Name == "os" && (sel.Sel.Name == "Getenv" || sel.Sel.Name == "LookupEnv") && strings.HasPrefix(arg, "TM_PLATFORM_TEST_"): add(env, arg) case pkgIdent.Name == "exec" && sel.Sel.Name == "LookPath": add(bin, arg) } return true }) return nil }) if err != nil { t.Fatal(err) } return env, bin } func equal(got, want []string) bool { if len(got) != len(want) { return false } g, w := append([]string(nil), got...), append([]string(nil), want...) sort.Strings(g) sort.Strings(w) for i := range g { if g[i] != w[i] { return false } } return true } // makeRecipe is the Makefile's recipe for one target, as the SHELL would see it: one string per // logical command, comments cut, backslash continuations joined. // // It lives in one place because two gates read recipes now, and two hand-written parsers of one // grammar are a second carrier free to drift from the first — the defect this zone spent a day // removing from its own money path. // // ⚠ The recipe is cut the way MAKE cuts it — the target line, then every line that is blank or // tab-indented — and NOT at the first blank line: a recipe split into two indented lines with a blank // between them is legal make with identical output, and a gate that failed on it would send the next // person who tidies this file looking for a defect that is not there. ⚠ And the joining is what makes // the unit LOGICAL: a pipeline split across a continuation is one command however many lines it // occupies, so a legal reformat is not a finding. func makeRecipe(t *testing.T, target string) []string { t.Helper() recipe, err := os.ReadFile(filepath.Join(zoneRoot, "Makefile")) if err != nil { t.Fatal(err) } lines := strings.Split(string(recipe), "\n") start := -1 for i, l := range lines { if strings.HasPrefix(l, target+":") { start = i break } } if start < 0 { t.Fatalf("the Makefile declares no `%s` target: the gate that reads it has no source", target) } var recipeLines []string for _, l := range lines[start+1:] { if l != "" && !strings.HasPrefix(l, "\t") { break } recipeLines = append(recipeLines, l) } commands := []string{} pending := "" for _, l := range recipeLines { code := strings.TrimPrefix(l, "\t") if head, _, found := strings.Cut(code, "#"); found { code = head } trimmed := strings.TrimRight(code, " \t") if strings.HasSuffix(trimmed, `\`) { pending += strings.TrimSuffix(trimmed, `\`) + " " continue } commands = append(commands, pending+code) pending = "" } if pending != "" { commands = append(commands, pending) } return commands } // ⛔ THE BATTERY MUST NOT BE ABLE TO REPORT CLEANLINESS OVER A LOG IT DID NOT READ. // // Every line `make check` prints about the run — the package list, the ALARM rows, the failure list, // the skip count — is a GREP over one file. Grep answers a MISSING file exactly as it answers a clean // one: with nothing. So the recipe could reach its final `else` and print «every test ran: no host // condition was missing» about a run whose log had vanished, which is a bill of health for a // measurement that never happened. // // It is not hypothetical and it is not rare: observed 06.09 on a run with FIVE skips, three // `grep: .check.log: No such file or directory` followed by that very line. The cause was a FIXED log // name shared by every run in the directory — and two batteries in one directory is the normal case // here, the zone and the orchestrator both run one. // // So the recipe must TEST the log before it reads it and leave RED when it is not there. This gate // holds ONE SAFE SHAPE and says so plainly rather than pretending to prove the property: a test of the // log, then an exit with a LITERAL NON-ZERO code, then the reads. Within that shape the details are // free — the log may be named per-run or not, the guard may be `-s`, `-f` or `-e`. A different but // equally correct shape (say, all the reads nested inside `if [ -s log ]; then … else exit 1; fi`) // will go red here, and that is deliberate: re-shaping the one recipe the whole battery reports // through should be a decision somebody takes and re-pins, not a silent pass. // // Mutation caught: deleting the guard; moving it after the first grep; dropping the `exit` so the // recipe notices and carries on; and — the one the first edition missed — `exit 0`, which satisfies // "there is an exit" while returning SUCCESS from a run whose evidence is gone. func TestTheBatteryCannotReportCleanlinessWithoutItsLog(t *testing.T) { steps := []string{} for _, cmd := range makeRecipe(t, "check") { for _, s := range strings.Split(cmd, ";") { steps = append(steps, strings.TrimSpace(s)) } } // WHERE THE LOG COMES FROM, read out of the recipe rather than assumed: the redirect of the test // run names it, so a rename cannot make this gate look at the wrong file and pass. logRef := "" for _, s := range steps { if !strings.Contains(s, "test ./...") { continue } if _, after, found := strings.Cut(s, "> "); found { logRef = strings.Trim(strings.Fields(after)[0], `"`) } } if logRef == "" { t.Fatal("the `check` recipe does not redirect its test run to a file this gate can find: " + "either the battery stopped keeping a log, or its shape moved and this gate now reads nothing") } // One shell command per step, with the leading keyword taken off, so that `then exit 1` is read as // `exit 1` and `then echo …` as an echo. Done once here because every check below asks the same // question of the same string, and two strippers would drift. cmds := make([]string, len(steps)) for i, s := range steps { cmd := strings.TrimSpace(s) for { stripped := cmd for _, kw := range []string{"then", "else", "do", "{"} { if rest, found := strings.CutPrefix(stripped, kw); found && strings.TrimLeft(rest, " \t") != rest { stripped = strings.TrimLeft(rest, " \t") } } if stripped == cmd { break } cmd = stripped } cmds[i] = cmd } reads, guard, exits := -1, -1, -1 exitCode := "" for i, cmd := range cmds { if !strings.Contains(cmd, logRef) { continue } // A grep INVOCATION, not the word: the recipe's own guard EXPLAINS itself in an `echo`, and the // explanation says «grep». The first edition of this gate counted that sentence as a read and // reported the guard as too late — which is the same substring-versus-invocation trap the // sibling check above spends a paragraph on, walked into by the test that quotes it. greps := strings.Contains(cmd, "$$(grep") || (!strings.HasPrefix(cmd, "echo") && strings.Contains(cmd, "grep")) switch { case greps && reads < 0: reads = i case strings.HasPrefix(cmd, "if [") || strings.HasPrefix(cmd, "[ "): if guard < 0 && (strings.Contains(cmd, "-s ") || strings.Contains(cmd, "-f ") || strings.Contains(cmd, "-e ")) { guard = i } } } // ⛔ THE EXIT'S CODE, not merely its presence — and this half was MISSING from the first edition. // `exit 0` after the guard satisfied "there is an exit" while leaving the recipe to announce that // the evidence is gone and then RETURN SUCCESS. That is the very defect this gate exists for, // wearing different clothes and worse in one respect: the old false clean bill was a LINE a person // could catch, this one is the EXIT CODE, which is what CI and the landing read. Measured, not // argued: with `exit 1` mutated to `exit 0` this gate stayed green and `make check` returned 0 // while printing «THE BATTERY LEFT NO LOG». // // A literal is required. `exit` bare carries the previous command's status — here the status of an // `echo`, which is zero — and `exit $var` cannot be judged from the recipe at all; both are refused // rather than guessed, because a guard whose code is not readable here is a guard this gate cannot // promise anything about. for i, cmd := range cmds { if guard < 0 || i <= guard { continue } if rest, found := strings.CutPrefix(cmd, "exit"); found { exits, exitCode = i, strings.TrimSpace(rest) break } } if reads < 0 { t.Fatalf("no step of the `check` recipe greps %s: the gate is reading the wrong target or the "+ "battery no longer reports what the log holds", logRef) } if guard < 0 { t.Fatalf("the `check` recipe reads %s at step %d and never TESTS that it is there. A missing log "+ "is indistinguishable from a clean one to grep, so the recipe will print «every test ran» "+ "over a run it did not measure — the defect this gate exists for", logRef, reads) } if guard > reads { t.Errorf("the `check` recipe tests %s at step %d but has already read it at step %d: the first "+ "read is where the false clean bill starts, so the guard must come before it", logRef, guard, reads) } if exits < 0 || exits > reads { t.Errorf("the `check` recipe tests %s at step %d but does not exit before its first read at step "+ "%d: noticing the absence and carrying on prints the same bill of health as never looking", logRef, guard, reads) return } code, err := strconv.Atoi(exitCode) if err != nil { t.Errorf("the guard of the `check` recipe exits with %q, which this gate cannot read as a number: "+ "a bare `exit` carries the previous command's status — an `echo`, so zero — and a variable "+ "cannot be judged from the recipe. Write a literal, so that what the recipe promises is what "+ "a reader can check", "exit "+exitCode) return } if code == 0 { t.Errorf("the guard of the `check` recipe says the log is missing and then exits %d, which is "+ "SUCCESS: the battery announces that its evidence is gone and reports a green run. That is "+ "the same false clean bill this gate exists for, moved from a LINE a person reads into the "+ "EXIT CODE a machine reads — and the landing reads the code", code) } } // ⛔ THE BATTERY MAY DELETE ITS OWN LOG AND NOBODY ELSE'S. // // The per-run name that removed the collision took a property with it: the fixed name limited itself // to ONE stale file, and per-run names accumulate one per failed or interrupted run. The recipe // sweeps them — and a sweep is exactly where the original defect can walk back in, because // `rm -f .check.log.*` is the obvious way to write it and it deletes a CONCURRENT run's evidence // mid-recipe, which is the collision the per-run name exists to prevent. // // So the shape held here is: any removal aimed at a log OTHER than this run's own must carry a // liveness test in the same step. `kill -0` is what the recipe uses — POSIX, no `/proc` — and a // recycled PID keeps a stale file, which is the conservative direction and the only one that cannot // destroy evidence. // // Mutation caught: replacing the conditional sweep with a bare `rm -f .check.log.*`; dropping the // liveness test while keeping the loop; sweeping after the run has started rather than before. func TestTheBatterySweepsOnlyLogsWhoseWriterIsGone(t *testing.T) { steps := []string{} for _, cmd := range makeRecipe(t, "check") { for _, s := range strings.Split(cmd, ";") { steps = append(steps, strings.TrimSpace(s)) } } logRef := "" for _, s := range steps { if strings.Contains(s, "test ./...") { if _, after, found := strings.Cut(s, "> "); found { logRef = strings.Trim(strings.Fields(after)[0], `"`) } } } if logRef == "" { t.Fatal("the `check` recipe does not redirect its test run to a file this gate can find") } // A removal of THIS run's own log is the ordinary cleanup and needs no liveness test: nothing else // can be writing it. Every OTHER removal is aimed at somebody else's file. swept := false for _, s := range steps { if !removesAFile(s) || strings.Contains(s, logRef) { continue } swept = true if !strings.Contains(s, "kill -0") { t.Errorf("the `check` recipe removes a log that is not its own without asking whether its "+ "writer is still alive:\n\t%s\nA sweep that does not test liveness deletes a concurrent "+ "run's evidence mid-recipe — the collision the per-run name was introduced to remove, "+ "walking back in through the tidy-up", s) } } if !swept { t.Error("the `check` recipe never removes an OLD log. Per-run names do not clean up after " + "themselves: every failed or interrupted run leaves its file behind, and the heap becomes " + "indistinguishable from the one log a failing run deliberately keeps for a reader") } } // removesAFile says whether a recipe step INVOKES `rm`, as opposed to containing those two letters. // // ⚠ Written as a function after a substring test matched «ala`rm m`arkers» inside an echo and blamed // the recipe for a defect it did not have. That is the THIRD time in one shift that a check of this // file confused a word with a command — the sibling gate warns about it in prose, the guard's own // echo tripped the exit check, and now this. Prose warnings evidently do not stop it; a named // function that every such check must go through has a chance to. func removesAFile(step string) bool { // ⚠ MAKE'S SILENCE PREFIX IS GLUED TO THE FIRST COMMAND OF A RECIPE (`@rm -f …`), so the first // token is `@rm` and not `rm`. Found by a mutation that this gate DID catch — but through the // wrong assertion, reporting «never removes an old log» about a recipe whose first act was // removing them all. A right verdict for a wrong reason is a hole, not a pass. fields := strings.Fields(strings.TrimPrefix(strings.TrimSpace(step), "@")) for i, f := range fields { if f != "rm" { continue } if i == 0 { return true } switch fields[i-1] { case "||", "&&", "then", "else", "do", "{": return true } } return false } // A battery that ended without a test reporting a failure must still name the package and the reason // (backlog row 305). Proved by RUNNING the reporter over recorded logs — and, in the sibling below, // by holding `check` to actually calling it: a reporter nothing invokes is a reporter that reports // nothing, and the recipe's own failure branch is where the defect lived. // // The logs are recorded, not hand-written: each came from breaking this module on a copy of the tree // (a test that sleeps past the timeout, a call to a function that does not exist, a plain `t.Fatal`) // and keeping what `go test -v` printed. // // The third case keeps the cure from eating the ordinary path: a log where a test did fail must still // name that test. func TestTheBatteryNamesWhatFailedWhenNoTestSaidItDid(t *testing.T) { for _, c := range []struct { log string wants []string unwants []string }{{ // A package that ran past its deadline: the panic names the deadline, the block under it names // the test still running, and only the package summary says WHERE. log: "battery-timeout.log", // The COUNT is asserted, not just the presence of evidence: a capped grep whose cap says // nothing reads as «that was all there was», which is the reading ENGINEERING_STANDARDS §3.8 // forbids and the one this whole target exists to stop. wants: []string{"textmachine/platform/internal/money", "test timed out", "TestHangReproForRow305", "NO TEST REPORTED A FAILURE", "Evidence, 1 lines"}, }, { // A package that never compiled: `[build failed]` and the compiler's own line, which is the // only thing that says what to fix. log: "battery-buildfail.log", wants: []string{"textmachine/platform/internal/money", "build failed", "undefined: undefinedHelper", "NO TEST REPORTED A FAILURE", "Evidence, 2 lines"}, }, { // The ordinary ending, which the cure must not trade away. log: "battery-testfail.log", wants: []string{"textmachine/platform/internal/money", "--- FAIL: TestPlainFailureRepro"}, unwants: []string{"NO TEST REPORTED A FAILURE"}, }} { t.Run(c.log, func(t *testing.T) { cmd := exec.CommandContext(t.Context(), "make", "--no-print-directory", "-s", "report-failures", "LOG=internal/gates/testdata/"+c.log, "STATUS=1") cmd.Dir = zoneRoot out, err := cmd.CombinedOutput() if err != nil { t.Fatalf("make report-failures failed: %v\n%s", err, out) } for _, w := range c.wants { if !strings.Contains(string(out), w) { t.Errorf("the report of a battery that went red does not carry %q, so an operator reading it "+ "cannot tell what to look at:\n%s", w, out) } } for _, u := range c.unwants { if strings.Contains(string(out), u) { t.Errorf("the report says %q over a log in which a test DID report a failure:\n%s", u, out) } } }) } } // Every field the daemon copies from the store's observations into the metrics struct must come from // the field of the SAME NAME. // // The copy is a composite literal of nine near-identically-typed numbers (cmd/tmplatformd, the // ObserveRunner call), so two of them swapped compiles, passes every test, and publishes one state's number under // another's name — and the two this pack added, `ParkedAttempts` and `QuarantinedAttempts`, are // exactly the pair an operator would act on in opposite ways (PD-438). // // Read from the source rather than exercised, because exercising it needs the daemon: what is being // held is the mapping, and the mapping is visible where it is written. func TestTheDaemonCopiesEveryObservationIntoTheFieldOfItsOwnName(t *testing.T) { const src = zoneRoot + "/cmd/tmplatformd/runner.go" f, err := parser.ParseFile(token.NewFileSet(), src, nil, 0) if err != nil { t.Fatal(err) } var checked int ast.Inspect(f, func(n ast.Node) bool { lit, ok := n.(*ast.CompositeLit) if !ok { return true } sel, ok := lit.Type.(*ast.SelectorExpr) if !ok || sel.Sel.Name != "Runner" { return true } for _, e := range lit.Elts { kv, ok := e.(*ast.KeyValueExpr) if !ok { continue } key, ok := kv.Key.(*ast.Ident) if !ok { continue } from, ok := kv.Value.(*ast.SelectorExpr) if !ok { t.Errorf("metrics.Runner.%s is not copied from an observation field", key.Name) continue } checked++ if from.Sel.Name != key.Name { t.Errorf("metrics.Runner.%s is filled from observations.%s: one state's number is "+ "published under another's name", key.Name, from.Sel.Name) } } return false }) // A negative result of the loop above and «the literal moved» are the same silence, so the count // is asserted: the literal copies nine observations, and every one of them is checked. if checked < 9 { t.Fatalf("only %d fields of metrics.Runner were checked: the composite literal this gate reads "+ "has moved, and the gate is now holding nothing", checked) } } // `check` must REACH the reporter on a red run. The target being correct is half of row 305; the // other half is that the failure branch of the recipe calls it instead of grepping for `--- FAIL` // itself — which is what it did, and what an edit could quietly restore. // // Read out of the recipe rather than exercised: exercising it means a red battery, and the report // itself is already proved by execution above. func TestTheBatterysFailureBranchCallsTheReporter(t *testing.T) { var branch []string inBranch := false for _, cmd := range makeRecipe(t, "check") { for _, step := range strings.Split(cmd, ";") { step = strings.TrimSpace(step) if strings.HasPrefix(step, "if [ $$status -ne 0 ]") { inBranch = true } if inBranch { branch = append(branch, step) } if inBranch && strings.HasPrefix(step, "fi") { inBranch = false } } } if len(branch) == 0 { t.Fatal("the `check` recipe has no branch for a non-zero test run: either it stopped reacting to " + "one, or its shape moved and this gate now reads nothing") } joined := strings.Join(branch, " ; ") if !strings.Contains(joined, "report-failures") { t.Errorf("the failure branch does not call `report-failures`, so a package that timed out is "+ "reported by whatever this branch does instead:\n%s", joined) } // The predecessor of the call, verbatim: a branch that greps for `--- FAIL` on its own is the // defect of row 305 restored, whether or not the target still exists beside it. if strings.Contains(joined, "--- FAIL") { t.Errorf("the failure branch greps for `--- FAIL` itself; that grep is the only shape a FAILING "+ "TEST has, and a package that timed out has none of it:\n%s", joined) } }