81 lines
3.8 KiB
Go
81 lines
3.8 KiB
Go
package gates
|
||
|
||
import (
|
||
"os/exec"
|
||
"path/filepath"
|
||
"regexp"
|
||
"strings"
|
||
"testing"
|
||
)
|
||
|
||
// ⛔ THE HARNESS THAT PRODUCES THE PACK'S NUMBERS IS ITSELF MEASURED, and it had to be: a mutation
|
||
// tool that counts the wrong thing turns «plantings N, caught M» into a statement about another
|
||
// subject, and nothing about the number looks wrong (unified backlog row 486).
|
||
//
|
||
// Four ways to a false number, one fixture record each, and the right answer for every one is known
|
||
// in advance — which is the only way a verdict CAN be checked. Against the zone's own code a wrong
|
||
// verdict is indistinguishable from a finding.
|
||
//
|
||
// - F1 the planting breaks the NAMED pin — the only shape that may be counted as caught;
|
||
// - F2 the planting makes the package red through a DIFFERENT test — a right verdict for a wrong
|
||
// reason, which is a hole and not a catch (D39.217 п.2в);
|
||
// - F3 nothing observes the planting — a survivor, and the tool must say so rather than hide it;
|
||
// - F4 the anchor is not in the file — nothing was planted, so nothing was measured;
|
||
// - F5 the set is RED BEFORE ANY PLANTING — the condition unified backlog row 486 exists for: a
|
||
// package red by its own reason would otherwise report every planting as a catch;
|
||
// - F6 the record names no pin — «something went red» is an answer to a neighbouring question.
|
||
//
|
||
// ⚠ IT ALSO ASSERTS THE RESTORATION, because a harness that leaves a planting behind poisons every
|
||
// later record silently: the copy has to come back byte-identical to its origin.
|
||
func TestTheMutationHarnessCountsOnlyWhatItCanMeasure(t *testing.T) {
|
||
python, err := exec.LookPath("python3")
|
||
if err != nil {
|
||
t.Skip("python3 is not on PATH: the zone's mutation harness cannot be run, so its counting rules are unchecked")
|
||
}
|
||
origin := filepath.Join("testdata", "mutantfixture")
|
||
copyRoot := filepath.Join(t.TempDir(), "tree")
|
||
if out, err := exec.CommandContext(t.Context(), "cp", "-a", origin, copyRoot).CombinedOutput(); err != nil {
|
||
t.Fatalf("copy the fixture: %v: %s", err, out)
|
||
}
|
||
cmd := exec.CommandContext(t.Context(), python, filepath.Join("tools", "mutate.py"),
|
||
filepath.Join("internal", "gates", origin, "catalogue.json"), copyRoot,
|
||
filepath.Join("internal", "gates", origin))
|
||
cmd.Dir = zoneRoot
|
||
out, err := cmd.CombinedOutput()
|
||
if err != nil {
|
||
t.Fatalf("the harness itself failed: %v\n%s", err, out)
|
||
}
|
||
got := map[string]string{}
|
||
line := regexp.MustCompile(`^=== ([^:]+): (?:⚠ |⛔ )?(RED|ВЫЖИЛА|НЕ ИЗМЕРЕНА) `)
|
||
for _, l := range strings.Split(string(out), "\n") {
|
||
if m := line.FindStringSubmatch(l); m != nil {
|
||
got[m[1]] = m[2]
|
||
}
|
||
}
|
||
want := map[string]string{
|
||
"F1-caught-by-its-pin": "RED",
|
||
"F2-red-by-another-test": "НЕ ИЗМЕРЕНА",
|
||
"F3-survives": "ВЫЖИЛА",
|
||
"F4-anchor-is-gone": "НЕ ИЗМЕРЕНА",
|
||
"F5-baseline-is-red": "НЕ ИЗМЕРЕНА",
|
||
"F6-no-pin-named": "НЕ ИЗМЕРЕНА",
|
||
}
|
||
if len(got) != len(want) {
|
||
t.Fatalf("the harness reported %d records and the fixture has %d: %s", len(got), len(want), out)
|
||
}
|
||
for id, w := range want {
|
||
if got[id] != w {
|
||
t.Errorf("record %s came out %q, want %q — the tool's counting rule is not the one the "+
|
||
"pack's numbers were read under\n%s", id, got[id], w, out)
|
||
}
|
||
}
|
||
// The tally, from the tool's own summary: a per-record verdict that is right while the count is
|
||
// wrong is still a false number in an act.
|
||
if !strings.Contains(string(out), "записей 6 · поймано 1 · ВЫЖИЛО 1 · НЕ ИЗМЕРЕНО 4") {
|
||
t.Errorf("the summary line does not match the verdicts above: %s", out)
|
||
}
|
||
if out, err := exec.CommandContext(t.Context(), "diff", "-rq", origin, copyRoot).CombinedOutput(); err != nil {
|
||
t.Errorf("the harness did not restore the copy, so every later record it measures is planted on "+
|
||
"top of this one: %v: %s", err, out)
|
||
}
|
||
}
|