// Command tmmutate plants a catalogued source mutation, runs one package's tests, and puts the source // back — the adversarial half of the battery, as a tool instead of as a script somebody rewrites. // // WHY IT IS IN THE REPOSITORY. A gate is only worth what it CATCHES, and the only way to know is to // break the thing on purpose and watch the battery go red. Two packs and one acceptance in a row have // each written this harness from scratch in a scratch directory, and it died with the session every // time; the acceptance of the bank-decisions door then found EIGHT places where a gate was blind, which // is what a catalogue that outlives a session is for. The catalogue is the record: every entry names a // property somebody decided was worth guarding, and re-running it says whether that is still true. // // WHY IT FINGERPRINTS. A planted mutation looks exactly like production code. One run of the previous // harness was killed by a tool timeout before its cleanup ran, and the planting sat in the tree for // forty minutes; an independent reviewer took measurements off that tree and reported a defect that did // not exist. So: the sha256 of every target is taken before the edit and checked after the restore, the // process restores on SIGINT/SIGTERM, and a mismatch is a loud failure rather than a log line. // // cp -a backend /tmp/mut/backend && cd /tmp/mut/backend // go run ./cmd/tmmutate -root . # every mutation in the catalogue // go run ./cmd/tmmutate -root . -id A-lock,G-byte-gate // go run ./cmd/tmmutate -root . -logs /tmp/mutlogs # keep each mutation's full test output // // ⚠ `-root` has no default and the copy is not a suggestion: the whole-catalogue fingerprint below is // taken when the tool starts and can only certify a tree that was clean AT THAT MOMENT. A tree another // process is editing, or one a previously crashed run left poisoned, verifies perfectly against its own // poisoned baseline. // // A mutation the battery SURVIVES is the finding. The tool exits non-zero when any mutation survives, // so it is usable as a gate itself. package main import ( "crypto/sha256" "encoding/hex" "encoding/json" "errors" "flag" "fmt" "os" "os/exec" "os/signal" "path/filepath" "regexp" "strings" "sync" "syscall" "time" ) // Mutation is one planting: what to break, and where the pin that must catch it lives. type Mutation struct { ID string `json:"id"` // Why states the PROPERTY the planting attacks, in the words of whoever added it. It is the only part // of an entry that cannot be re-derived, and it is what a reader needs when a mutation survives. Why string `json:"why"` // Package is where the pin is looked for — NOT necessarily where the edit is. A planting judged by // the tests of the file it edits is the classic false "the mutation survived": the pin for a store // lock lives in the pipeline package, and running the store's own tests would prove nothing. Package string `json:"package"` // Run optionally narrows `go test -run`. Left empty the WHOLE package runs, which is the honest // default: a pin nobody named is still a pin. Run string `json:"run,omitempty"` // Expect is "red" (the default) or "survives". A catalogued survivor is not a hole somebody forgot — // it is an argued one, and the argument belongs in Why. Without this the tool could only ever report, // because one known survivor would make it fail forever; with it, the catalogue is a GATE: any // outcome that is not the recorded one fails the run. Expect string `json:"expect,omitempty"` Edits []Edit `json:"edits"` } // Edit is one exact textual replacement. Exact and unique on purpose: a regexp that matches twice // mutates two things and attributes the result to one. type Edit struct { File string `json:"file"` Find string `json:"find"` Replace string `json:"replace"` } type original struct { path string body []byte sum string } func main() { root := flag.String("root", "", "REQUIRED. Module root the mutations are applied in. Point it at a COPY: this tool edits source files, and a parallel session editing the same tree turns a measurement into a corruption. No default, deliberately — the dangerous value must not be the one you get by typing nothing.") logs := flag.String("logs", "", "directory to write each mutation's FULL test output into (empty: keep only the failing test names)") catalog := flag.String("catalog", "", "catalogue path (default /cmd/tmmutate/mutations.json)") ids := flag.String("id", "", "comma-separated mutation ids to run (default: all)") timeout := flag.Duration("timeout", 20*time.Minute, "per-mutation test timeout") flag.Parse() if *root == "" { fmt.Fprintln(os.Stderr, "tmmutate: -root is required (point it at a COPY of the module, not at the tree you are editing)") os.Exit(2) } if err := run(*root, *catalog, *ids, *timeout, *logs); err != nil { fmt.Fprintf(os.Stderr, "tmmutate: %v\n", err) os.Exit(1) } } func run(root, catalog, ids string, timeout time.Duration, logs string) error { if catalog == "" { catalog = filepath.Join(root, "cmd", "tmmutate", "mutations.json") } raw, err := os.ReadFile(catalog) if err != nil { return fmt.Errorf("read the catalogue: %w", err) } var all []Mutation dec := json.NewDecoder(strings.NewReader(string(raw))) dec.DisallowUnknownFields() if err := dec.Decode(&all); err != nil { return fmt.Errorf("parse the catalogue: %w", err) } want := map[string]bool{} for _, id := range strings.Split(ids, ",") { if id = strings.TrimSpace(id); id != "" { want[id] = true } } // The catalogue is data a human writes, so it is VALIDATED rather than trusted. DisallowUnknownFields // catches a misspelled key; it says nothing about a key that is simply absent, and an entry with no // `package` would have run `go test` with an empty pattern and reported whatever came back. seen := map[string]bool{} for i, m := range all { switch { case m.ID == "": return fmt.Errorf("catalogue entry %d has no id", i) case seen[m.ID]: return fmt.Errorf("the catalogue has two entries with id %q", m.ID) case m.Package == "": return fmt.Errorf("%s: no package — the pin is looked for SOMEWHERE, and an empty pattern is not a somewhere", m.ID) case m.Why == "": return fmt.Errorf("%s: no `why` — the property a planting attacks is the only part of an entry that cannot be re-derived", m.ID) case m.Expect != "" && m.Expect != "red" && m.Expect != "survives": return fmt.Errorf("%s: expect is %q, want \"red\" or \"survives\"", m.ID, m.Expect) case len(m.Edits) == 0: return fmt.Errorf("%s: the entry has no edits", m.ID) } for j, e := range m.Edits { if e.File == "" || e.Find == "" { return fmt.Errorf("%s: edit %d has no file or no anchor", m.ID, j) } } seen[m.ID] = true } for id := range want { if !seen[id] { return fmt.Errorf("no mutation %q in the catalogue", id) } } // Every file the catalogue touches, fingerprinted ONCE while the tree is known good and re-checked // before each mutation. Belt and braces on purpose: the per-mutation restore verifies itself too, but // a restore that verifies against the wrong baseline verifies nothing — which is exactly how this // tool failed the first time it was used. // // ⚠ A FILE THIS PASS CANNOT READ IS THAT ENTRY'S ROT, NOT THE RUN'S DEATH. It used to `return` here, // and that quietly undid the whole point of making a moved ANCHOR per-entry: a renamed or deleted file // is the commonest way a catalogue goes stale, and one of them stopped the run BEFORE entry one — // taking every healthy entry behind it down unseen, `-id` runs included, because this loop walks the // whole catalogue and not the selection. Measured: a catalogue with one such entry printed // «fingerprint …: no such file or directory» and ran nothing at all. baseline := map[string]string{} rottedFile := map[string]string{} // mutation id → why its file could not be fingerprinted for _, m := range all { for _, e := range m.Edits { path := filepath.Join(root, e.File) if _, have := baseline[path]; have { continue } body, rerr := os.ReadFile(path) if rerr != nil { // Recorded against the ENTRY, so the run reports it in place and still ends non-zero. The // file is deliberately left out of the baseline: nothing may be measured against a file // this pass never saw. rottedFile[m.ID] = fmt.Sprintf("%s: %v (the file moved or was deleted under the catalogue)", m.ID, rerr) continue } baseline[path] = hash(body) } } pristine := func(when string) error { for path, sum := range baseline { body, rerr := os.ReadFile(path) if rerr != nil { return fmt.Errorf("%s: re-read %s: %w", when, path, rerr) } if hash(body) != sum { return fmt.Errorf("%s: TREE IS NOT PRISTINE at %s — a planting is still in it; every result after this point is measured against mutated code", when, path) } } return nil } // THE PACKAGE MUST BE GREEN BEFORE ANYTHING IS PLANTED, once per package. // // Without this the tool's central sentence — «this pin catches this mutation» — rests on an exit code // that a build failure, a pre-existing red, a flake, a timeout or a mistyped package pattern produce // just as well. Demonstrated against this very tool: an entry whose `find` equals its `replace` (the // source is byte-identical afterwards) was reported RED and the gate exited 0. A baseline is what // makes a later red ATTRIBUTABLE. green := map[string]bool{} rottedPkg := map[string]string{} for _, m := range all { if len(want) > 0 && !want[m.ID] { continue } if green[m.Package] { continue } fmt.Printf("baseline %s …\n", m.Package) base := exec.Command("go", "test", m.Package, "-count=1", fmt.Sprintf("-timeout=%s", timeout)) base.Dir = root if out, berr := base.CombinedOutput(); berr != nil { // ⚠ A PACKAGE PATH THAT NO LONGER RESOLVES IS ROT, and it is the third way a catalogue goes // stale after a moved anchor and a moved file. It used to `return` here, so one renamed package // stopped the run before entry one and took every healthy entry with it — the same shape the // other two were fixed out of. A package that resolves and is genuinely RED still stops the // run: a verdict measured against a package that was already failing is unattributable, which // is the whole reason this baseline exists. if setupFailed(out) { rottedPkg[m.Package] = fmt.Sprintf("package %s does not resolve (it was renamed, moved or deleted under the catalogue)", m.Package) continue } return fmt.Errorf("%s is NOT GREEN before any mutation is planted — every verdict about it would be unattributable:\n%s", m.Package, tailOf(out)) } green[m.Package] = true } survived, ran := []string{}, 0 for _, m := range all { if len(want) > 0 && !want[m.ID] { continue } if why, dead := rottedPkg[m.Package]; dead { ran++ fmt.Printf("ROTTED %-28s %s\n", m.ID, why) survived = append(survived, m.ID+" (package moved)") continue } if why, dead := rottedFile[m.ID]; dead { // Reported in place, before anything is planted: the entry guards nothing, and the run goes on. ran++ fmt.Printf("ROTTED %-28s %s\n", m.ID, why) survived = append(survived, m.ID+" (file moved)") continue } if err := pristine("before " + m.ID); err != nil { return err } ran++ verdict, detail, err := one(root, m, timeout, logs) if err != nil { return err } wantRed := m.Expect != "survives" switch { case verdict == rotted: // Counted as an unexpected outcome, exactly like NOTHING: an entry whose anchor moved guards // nothing, and the run must still end non-zero. What it no longer does is stop the run. fmt.Printf("ROTTED %-28s %s\n", m.ID, detail) // The summary label stays generic: the DETAIL line above already says which kind of rot this // was, and a label naming one kind for all of them would be the same over-claim in miniature — // «anchor moved» printed for an entry whose run filter had been renamed. survived = append(survived, m.ID+" (rotted)") case verdict == inconclusive: // NOT a red. The package did not build, or the run died before a single test reported — so // nothing was judged and the entry proves nothing. Counted as an unexpected outcome on // purpose: a catalogue entry that stopped compiling (because the code it patches moved) is a // property that has silently stopped being guarded. fmt.Printf("NOTHING %-28s %s\n", m.ID, detail) survived = append(survived, m.ID+" (nothing ran)") case verdict == red && wantRed: fmt.Printf("RED %-28s %s\n", m.ID, detail) case verdict == red: fmt.Printf("RED! %-28s %s — the catalogue records this one as a SURVIVOR and it is now caught; update the entry\n", m.ID, detail) survived = append(survived, m.ID+" (unexpectedly caught)") case wantRed: fmt.Printf("SURVIVED %-28s %s — %s\n", m.ID, m.Package, m.Why) survived = append(survived, m.ID) default: fmt.Printf("survives %-28s %s — recorded, argued: %s\n", m.ID, m.Package, m.Why) } } if err := pristine("after the last mutation"); err != nil { return err } fmt.Printf("\n%d mutation(s) run, %d unexpected outcome(s)\n", ran, len(survived)) if len(survived) > 0 { return fmt.Errorf("not what the catalogue records: %s", strings.Join(survived, ", ")) } return nil } var failLine = regexp.MustCompile(`(?m)^\s*--- FAIL: (\S+)`) // The three verdicts. INCONCLUSIVE is the one this tool lacked: «go test exited non-zero» is also what a // build failure, a mistyped package pattern and a timeout produce, and calling any of those RED is how a // property stops being guarded without anybody being told. type verdict int const ( survivedTests verdict = iota red inconclusive // ROTTED is the fourth, and it exists because the third was not enough: an anchor that no longer // occurs exactly once says the SOURCE MOVED under the catalogue, which is neither a survivor nor a // build failure. It used to abort the whole run — so one stale anchor cost the tool the only property // that makes a catalogue a catalogue, «it goes whole», and the rot then hid every entry after it. // Measured: `AF-keys-file-reason-covers-all` rotted at 8adcb86 and stopped the run at entry 33 of 120 // for two packs, while `-id` runs kept passing and nobody saw it. rotted ) // tailOf keeps the last few lines of a command's output for an error message. func tailOf(out []byte) string { lines := strings.Split(strings.TrimRight(string(out), "\n"), "\n") if len(lines) > 12 { lines = lines[len(lines)-12:] } return strings.Join(lines, "\n") } // one plants a mutation, runs the package's tests, and restores the tree whatever happens. func one(root string, m Mutation, timeout time.Duration, logs string) (v verdict, detail string, err error) { if len(m.Edits) == 0 { return survivedTests, "", fmt.Errorf("%s: the entry has no edits", m.ID) } // `saved` is read by the signal goroutine while this one appends to it, so it is guarded. Not // theoretical: the goroutine's whole job is to run DURING the write loop, which is exactly when the // slice is growing, and a torn read there restores the wrong bytes or none. var mu sync.Mutex var saved []original restore := func() error { mu.Lock() snapshot := append([]original(nil), saved...) mu.Unlock() var bad []string for _, o := range snapshot { if werr := os.WriteFile(o.path, o.body, 0o644); werr != nil { bad = append(bad, fmt.Sprintf("%s: %v", o.path, werr)) continue } // The restore is VERIFIED, not assumed. This is the whole reason the tool exists rather than a // shell one-liner: a planting left behind reads as production code. now, rerr := os.ReadFile(o.path) if rerr != nil || hash(now) != o.sum { bad = append(bad, fmt.Sprintf("%s: TREE IS NOT PRISTINE after restore", o.path)) } } if len(bad) > 0 { return fmt.Errorf("%s: could not restore: %s", m.ID, strings.Join(bad, "; ")) } return nil } // A signal must not be able to leave a planting behind either. sig := make(chan os.Signal, 1) signal.Notify(sig, os.Interrupt, syscall.SIGTERM) done := make(chan struct{}) go func() { select { case s := <-sig: // The restore's verdict is REPORTED, not discarded. Printing «restored» after a failed restore // is the one message this tool must never produce: it tells the operator the tree is clean at // the exact moment it is not. if rerr := restore(); rerr != nil { fmt.Fprintf(os.Stderr, "tmmutate: %v — THE TREE STILL CARRIES THE PLANTING, restore it by hand before trusting anything\n", rerr) os.Exit(3) } fmt.Fprintf(os.Stderr, "tmmutate: restored on %v\n", s) os.Exit(2) case <-done: } }() defer func() { // RESTORE FIRST, then stop the handler. The other order leaves a window in which the deferred // restore has not run and the handler that would have covered it is already gone. rerr := restore() close(done) signal.Stop(sig) // A restore failure OUTRANKS whatever else went wrong: every later verdict is measured against a // tree that is not the tree. Swallowing it behind an earlier error was how it could stay quiet. if rerr != nil { if err != nil { err = errors.Join(rerr, err) } else { err = rerr } } }() // ONE READ AND ONE WRITE PER FILE, and the pristine copy is taken from the FIRST read. // // ⚠ The obvious loop — read, patch, write, once per edit — is wrong the moment a mutation carries TWO // edits to ONE file, and it is wrong in the worst possible way: the second edit's "pristine" copy is // the already-patched file, so the restore writes the patched content back and the sha256 check // compares it against the patched hash and passes. This session shipped exactly that, ran two full // rounds with it, and every mutation after the two-edit one measured a tree that still carried a // planting — three pins were recorded as catching mutations they had nothing to do with. The tool // exists to make that impossible, so this ordering is the contract and not an implementation detail. edits := map[string][]Edit{} var order []string for _, e := range m.Edits { if _, seen := edits[e.File]; !seen { order = append(order, e.File) } edits[e.File] = append(edits[e.File], e) } for _, file := range order { path := filepath.Join(root, file) body, rerr := os.ReadFile(path) if rerr != nil { // ROT, not a run failure. The moved-file case itself is caught EARLIER, by the fingerprint pass, // which records it against the entry and lets the run continue; what this branch covers is the // narrow race in between — a file that disappears after that pass and before this read. // ⚠ An earlier version of this comment said the moved-file case "still aborts the whole run". // That was true when it was written and false one edit later, and it is the exact class of // defect this tool exists to find: a sentence that outlived the code it described. return rotted, fmt.Sprintf("%s: %v (the file moved or was deleted under the catalogue)", m.ID, rerr), nil } mu.Lock() saved = append(saved, original{path: path, body: body, sum: hash(body)}) mu.Unlock() patched := string(body) for _, e := range edits[file] { if n := strings.Count(patched, e.Find); n != 1 { // A VERDICT, not an error: the deferred restore has already been armed for every file read // so far, so returning here leaves the tree pristine and lets the remaining entries run. return rotted, fmt.Sprintf("%s: the anchor occurs %d times in %s, it must occur exactly once (the source moved under the catalogue, or two edits of this mutation overlap)", m.ID, n, e.File), nil } patched = strings.Replace(patched, e.Find, e.Replace, 1) } if werr := os.WriteFile(path, []byte(patched), 0o644); werr != nil { return survivedTests, "", fmt.Errorf("%s: %w", m.ID, werr) } } args := []string{"test", m.Package, "-count=1", fmt.Sprintf("-timeout=%s", timeout)} if m.Run != "" { args = append(args, "-run", m.Run) } cmd := exec.Command("go", args...) cmd.Dir = root out, terr := cmd.CombinedOutput() // The OUTPUT, not only the verdict. A mutation this session ran reported a pin that had in fact // failed for an unrelated reason, and with only the test NAMES there was no way to tell the two apart // without re-running by hand — which is the same class of false result the fingerprinting is against. if logs != "" { // Loud: a `-logs` directory that cannot be made is an operator who asked for evidence and will // find none, and this tool exists because a verdict without evidence was believed once already. if werr := os.MkdirAll(logs, 0o755); werr != nil { return survivedTests, "", fmt.Errorf("%s: -logs %s: %w", m.ID, logs, werr) } if werr := os.WriteFile(filepath.Join(logs, m.ID+".log"), out, 0o644); werr != nil { return survivedTests, "", fmt.Errorf("%s: write the log: %w", m.ID, werr) } } // ⚠ A `run` FILTER THAT MATCHES NOTHING IS ROT, NOT A SURVIVOR. `go test -run TestRenamedAway` runs // zero tests and exits 0, so a renamed pin turned every entry that names it green forever: an // expect-RED entry printed SURVIVED (a finding that is really a NOTHING — no test was judged), and an // expect-`survives` entry printed exactly what the catalogue records and kept the whole gate at exit 0. // The catalogue has entries with run filters precisely so a red is attributable to ONE pin, which is // what makes this failure quiet: the narrower the filter, the more a rename costs. if strings.Contains(string(out), "no tests to run") { return rotted, fmt.Sprintf("%s: the run filter %q matched no test — it was renamed or removed, and this entry has been asserting nothing", m.ID, m.Run), nil } if terr == nil { return survivedTests, "", nil } names := failLine.FindAllStringSubmatch(string(out), -1) var caught []string for _, n := range names { caught = append(caught, n[1]) } if len(caught) == 0 { // NOT a red. The package is green by baseline, so a non-zero exit with no test naming itself means // nothing was judged: the source did not build, the pattern matched no package, or the run died. // The first version of this tool called that RED, and two catalogue entries whose anchors had // stopped compiling were recorded for a whole session as caught by tests that never ran. return inconclusive, m.Package + ": no test reported — the planting did not build, or the run died before any test (property: " + m.Why + ")", nil } if len(caught) > 4 { caught = append(caught[:4], fmt.Sprintf("+%d more", len(names)-4)) } return red, m.Package + " " + strings.Join(caught, ", "), nil } func hash(b []byte) string { sum := sha256.Sum256(b) return hex.EncodeToString(sum[:]) } // setupFailed reports whether `go test` refused the PACKAGE PATTERN itself rather than running anything — // the package was renamed, moved or deleted under the catalogue. Distinguishable from a genuine red in the // output, and it must be, because the two get opposite treatment: rot is recorded per entry and the run // goes on, a real red stops everything. func setupFailed(out []byte) bool { s := string(out) return strings.Contains(s, "[setup failed]") || strings.Contains(s, "directory not found") || strings.Contains(s, "matched no packages") }