package gates import ( "os/exec" "path/filepath" "regexp" "strings" "testing" ) // ⛔ THE HARNESS THAT PRODUCES THE PACK'S NUMBERS IS ITSELF MEASURED, and it had to be: a mutation // tool that counts the wrong thing turns «plantings N, caught M» into a statement about another // subject, and nothing about the number looks wrong (unified backlog row 486). // // Four ways to a false number, one fixture record each, and the right answer for every one is known // in advance — which is the only way a verdict CAN be checked. Against the zone's own code a wrong // verdict is indistinguishable from a finding. // // - F1 the planting breaks the NAMED pin — the only shape that may be counted as caught; // - F2 the planting makes the package red through a DIFFERENT test — a right verdict for a wrong // reason, which is a hole and not a catch (D39.217 п.2в); // - F3 nothing observes the planting — a survivor, and the tool must say so rather than hide it; // - F4 the anchor is not in the file — nothing was planted, so nothing was measured; // - F5 the set is RED BEFORE ANY PLANTING — the condition unified backlog row 486 exists for: a // package red by its own reason would otherwise report every planting as a catch; // - F6 the record names no pin — «something went red» is an answer to a neighbouring question. // // ⚠ IT ALSO ASSERTS THE RESTORATION, because a harness that leaves a planting behind poisons every // later record silently: the copy has to come back byte-identical to its origin. func TestTheMutationHarnessCountsOnlyWhatItCanMeasure(t *testing.T) { python, err := exec.LookPath("python3") if err != nil { t.Skip("python3 is not on PATH: the zone's mutation harness cannot be run, so its counting rules are unchecked") } origin := filepath.Join("testdata", "mutantfixture") copyRoot := filepath.Join(t.TempDir(), "tree") if out, err := exec.CommandContext(t.Context(), "cp", "-a", origin, copyRoot).CombinedOutput(); err != nil { t.Fatalf("copy the fixture: %v: %s", err, out) } cmd := exec.CommandContext(t.Context(), python, filepath.Join("tools", "mutate.py"), filepath.Join("internal", "gates", origin, "catalogue.json"), copyRoot, filepath.Join("internal", "gates", origin)) cmd.Dir = zoneRoot out, err := cmd.CombinedOutput() if err != nil { t.Fatalf("the harness itself failed: %v\n%s", err, out) } got := map[string]string{} line := regexp.MustCompile(`^=== ([^:]+): (?:⚠ |⛔ )?(RED|ВЫЖИЛА|НЕ ИЗМЕРЕНА) `) for _, l := range strings.Split(string(out), "\n") { if m := line.FindStringSubmatch(l); m != nil { got[m[1]] = m[2] } } want := map[string]string{ "F1-caught-by-its-pin": "RED", "F2-red-by-another-test": "НЕ ИЗМЕРЕНА", "F3-survives": "ВЫЖИЛА", "F4-anchor-is-gone": "НЕ ИЗМЕРЕНА", "F5-baseline-is-red": "НЕ ИЗМЕРЕНА", "F6-no-pin-named": "НЕ ИЗМЕРЕНА", } if len(got) != len(want) { t.Fatalf("the harness reported %d records and the fixture has %d: %s", len(got), len(want), out) } for id, w := range want { if got[id] != w { t.Errorf("record %s came out %q, want %q — the tool's counting rule is not the one the "+ "pack's numbers were read under\n%s", id, got[id], w, out) } } // The tally, from the tool's own summary: a per-record verdict that is right while the count is // wrong is still a false number in an act. if !strings.Contains(string(out), "записей 6 · поймано 1 · ВЫЖИЛО 1 · НЕ ИЗМЕРЕНО 4") { t.Errorf("the summary line does not match the verdicts above: %s", out) } if out, err := exec.CommandContext(t.Context(), "diff", "-rq", origin, copyRoot).CombinedOutput(); err != nil { t.Errorf("the harness did not restore the copy, so every later record it measures is planted on "+ "top of this one: %v: %s", err, out) } }