package runs import ( "bytes" "encoding/json" "errors" "io/fs" "log/slog" "os" "path/filepath" "strings" "testing" "time" "textmachine/platform/internal/ingest" ) // A BOOK'S DIRECTORY IS THAT BOOK'S IDENTITY, so it is held to the same rule as the identifier // itself: it does not reach a line below ERROR (the zone's standard §2, PD-139, PD-99). // // The equivalence is not a reading of the path — it is how the intake writes it: every book taken in // lives at `/` (books.Receive), and five of five books on the live cold-run stand // read that way (`select id, workdir from books`, 11.09). The fixture below therefore puts the book // at its production shape rather than at a temporary directory, so that what the assertion looks for // is the identifier and not a string that merely resembles one. // // The intake has the same pin from its own side (books.TestNoBookIdentifierReachesAnInfoLine); this // is the reconciler's, and the reconciler is where a wedged run repeats its complaint on every pass. func TestNoBookDirectoryReachesALineBelowError(t *testing.T) { f := newFixture(t, "20", 500) book := f.bookID(t) workdir := filepath.Join(t.TempDir(), book) if err := os.MkdirAll(workdir, 0o750); err != nil { t.Fatal(err) } if _, err := f.store.Pool().Exec(f.ctx, `update books set workdir = $2 where id = $1`, book, workdir); err != nil { t.Fatal(err) } var buf bytes.Buffer f.svc.Log = slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: book, Chapters: order(10)}) if err != nil { t.Fatal(err) } if err := f.svc.Spawn(f.ctx, run.ID); err != nil { t.Fatal(err) } // The directory goes away under the live run, and the engine answers the way it does about a // project it cannot open: with the path in its own text. if err := os.RemoveAll(workdir); err != nil { t.Fatal(err) } f.engine.set(ingest.StatusReport{}, errors.New("tmctl: open "+workdir+"/project.db: no such file or directory")) f.runner.alive = false for i := 1; i <= 6; i++ { at := f.now.Add(time.Duration(i) * time.Hour) f.svc.Now = func() time.Time { return at } if err := f.svc.Sweep(f.ctx); err != nil { t.Fatal(err) } } lines := strings.Split(strings.TrimSpace(buf.String()), "\n") var carriesRun, said int for _, line := range lines { if line == "" { continue } var rec map[string]any if err := json.Unmarshal([]byte(line), &rec); err != nil { t.Fatal(err) } if strings.Contains(line, run.ID) { carriesRun++ } if rec["msg"] == "settlement deferred: the book's project directory cannot be read" { said++ } if rec["level"] == "ERROR" { continue // an ERROR may carry a path an operator must repair — see the note below } if strings.Contains(line, book) { t.Errorf("a %v line carries the book's own directory, which is its identifier: %s", rec["level"], line) } } // THE BOUNDARY THE FIXTURE HAD TO CROSS, and both halves of it. Without the first the assertion // above would be true of a run that never reached the settlement at all; without the second, of a // logger that wrote nothing. if said == 0 { t.Fatalf("no pass said the book's directory could not be read, so this fixture never reached the "+ "branch it is about (%d lines)", len(lines)) } if carriesRun == 0 { t.Fatalf("not one of the %d lines carries the run id: the filter that must match found nothing, so "+ "the filter above proves nothing either", len(lines)) } t.Logf("%d lines, %d of them carrying the run id, %d naming the unreadable directory", len(lines), carriesRun, said) } // THE SECOND CARRIER, and it was invisible to the pin above because that one holds the unit DEAD. // // A run whose unit is ALIVE never reaches the settlement: it is RESYNCED instead, on its own interval, // and that call asks the engine about the same directory. Where the pin above sees one complaint at // the end of a run, this path repeats one every `ResyncEvery` for as long as the unit lives — the // loudest shape of the identifier-in-a-log this class is about, and the one the first round of this // pack declared closed while it was open (adversarial pass, 11.09). func TestNoBookDirectoryReachesALineBelowErrorWhileTheUnitIsAlive(t *testing.T) { f := newFixture(t, "20", 500) book := f.bookID(t) workdir := filepath.Join(t.TempDir(), book) if err := os.MkdirAll(workdir, 0o750); err != nil { t.Fatal(err) } if _, err := f.store.Pool().Exec(f.ctx, `update books set workdir = $2 where id = $1`, book, workdir); err != nil { t.Fatal(err) } var buf bytes.Buffer f.svc.Log = slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) f.svc.Cfg.ResyncEvery = time.Second run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: book, Chapters: order(10)}) if err != nil { t.Fatal(err) } if err := f.svc.Spawn(f.ctx, run.ID); err != nil { t.Fatal(err) } // The unit is ALIVE and the book goes away under it: the sweep resyncs rather than settles. f.runner.alive = true if err := os.RemoveAll(workdir); err != nil { t.Fatal(err) } f.engine.set(ingest.StatusReport{}, errors.New("tmctl: open "+workdir+"/project.db: no such file or directory")) for i := 1; i <= 3; i++ { at := f.now.Add(time.Duration(i) * time.Minute) f.svc.Now = func() time.Time { return at } if err := f.svc.Sweep(f.ctx); err != nil { t.Fatal(err) } } lines := strings.Split(strings.TrimSpace(buf.String()), "\n") var carriesRun, said int for _, line := range lines { if line == "" { continue } var rec map[string]any if err := json.Unmarshal([]byte(line), &rec); err != nil { t.Fatal(err) } if strings.Contains(line, run.ID) { carriesRun++ } if rec["msg"] == "resync deferred: the book's project directory cannot be read" { said++ } if rec["level"] == "ERROR" { continue } if strings.Contains(line, book) { t.Errorf("a %v line of a LIVE run carries the book's own directory: %s", rec["level"], line) } } if said == 0 { t.Fatalf("no pass resynced against the missing directory, so this fixture never reached the branch "+ "it is about (%d lines): the unit must be alive and the resync due", len(lines)) } if carriesRun == 0 { t.Fatalf("not one of the %d lines carries the run id: the filter that must match found nothing", len(lines)) } t.Logf("%d lines, %d carrying the run id, %d naming the unreadable directory on a LIVE unit", len(lines), carriesRun, said) } // ⛔ THE HALF THE FIRST ROUND GOT WRONG: THE DIRECTORY IS THERE AND THE ENGINE STILL NAMES IT. // // The rule used to be a classifier — "is the book's directory gone?" — and where the answer was no it // carried the engine's text verbatim. An engine that fails on a FILE INSIDE a directory that is // perfectly present names the path just the same, and this test did not see it because it picked the // ONE engine error that has no path in it (`tmctl: exec: …`). Any other message and the pin next door // would have failed. Found by the acceptance of 11.09 (F3); the rule is a REMOVER now, so this half // asserts both things at once — the diagnosis survives and the identifier does not. func TestAnEngineThatNamesTheBooksPathIsCarriedWithoutIt(t *testing.T) { f := newFixture(t, "20", 500) book := f.bookID(t) workdir := filepath.Join(t.TempDir(), book) if err := os.MkdirAll(workdir, 0o750); err != nil { t.Fatal(err) } if _, err := f.store.Pool().Exec(f.ctx, `update books set workdir = $2 where id = $1`, book, workdir); err != nil { t.Fatal(err) } var buf bytes.Buffer f.svc.Log = slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: book, Chapters: order(10)}) if err != nil { t.Fatal(err) } if err := f.svc.Spawn(f.ctx, run.ID); err != nil { t.Fatal(err) } // THE DIRECTORY STAYS. What the engine cannot open is a file inside it — a project database that // was moved, a permission, a corrupt page: the ordinary shape, and the one the classifier missed. f.engine.set(ingest.StatusReport{}, errors.New("tmctl: open "+filepath.Join(workdir, "project.db")+": permission denied")) f.runner.alive = false for i := 1; i <= 3; i++ { at := f.now.Add(time.Duration(i) * time.Hour) f.svc.Now = func() time.Time { return at } if err := f.svc.Sweep(f.ctx); err != nil { t.Fatal(err) } } logged := buf.String() // The diagnosis: an operator must still be able to tell this from a hung engine. if !strings.Contains(logged, "permission denied") { t.Fatalf("the engine's own diagnosis did not survive the removal, so an operator has nothing to go "+ "on for a failure this platform cannot name itself:\n%s", logged) } // The identity: and it must not be there in any form. for _, line := range strings.Split(strings.TrimSpace(logged), "\n") { if line == "" { continue } var rec map[string]any if err := json.Unmarshal([]byte(line), &rec); err != nil { t.Fatal(err) } if rec["level"] == "ERROR" { continue } if strings.Contains(line, book) || strings.Contains(line, workdir) { t.Errorf("a %v line carries the book over a directory that is PRESENT: %s", rec["level"], line) } } } // ⛔ THE LOUDEST CARRIER, PINNED WHERE IT ACTUALLY LEAKS — and the fixture the first two rounds did not // have. // // The resync line repeats every `ResyncEvery` for as long as the unit lives, and its neighbour test // removes the directory: that makes `sourceThere` answer first, so the sweep says its own sentence and // the line carrying the ENGINE's text is never reached. Removing the redaction there therefore // survived the whole package (acceptance of 11.09, finding 4). Here the directory is PRESENT and the // engine fails on a file inside it — the shape that walks past every guard and into the log. func TestTheResyncOfAPresentDirectoryCarriesTheEnginesTextWithoutTheBook(t *testing.T) { f := newFixture(t, "20", 500) book := f.bookID(t) workdir := filepath.Join(t.TempDir(), book) if err := os.MkdirAll(workdir, 0o750); err != nil { t.Fatal(err) } if _, err := f.store.Pool().Exec(f.ctx, `update books set workdir = $2 where id = $1`, book, workdir); err != nil { t.Fatal(err) } var buf bytes.Buffer f.svc.Log = slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) f.svc.Cfg.ResyncEvery = time.Second run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: book, Chapters: order(10)}) if err != nil { t.Fatal(err) } if err := f.svc.Spawn(f.ctx, run.ID); err != nil { t.Fatal(err) } // The unit is ALIVE, so the sweep RESYNCS; the directory is there, so no guard answers first; and // what the engine cannot open is a file inside it. f.runner.alive = true f.engine.set(ingest.StatusReport{}, errors.New("tmctl: open "+filepath.Join(workdir, "project.db")+": database is locked")) for i := 1; i <= 3; i++ { at := f.now.Add(time.Duration(i) * time.Minute) f.svc.Now = func() time.Time { return at } if err := f.svc.Sweep(f.ctx); err != nil { t.Fatal(err) } } logged := buf.String() // THE BOUNDARY: the line this test is about must actually have been written, and it is the one // carrying the ENGINE's words — not the sweep's own sentence about a missing directory. if !strings.Contains(logged, `"msg":"resync failed"`) { t.Fatalf("no pass reached the resync's own line, so this fixture never met the carrier it is "+ "about:\n%s", logged) } if !strings.Contains(logged, "database is locked") { t.Fatalf("the engine's diagnosis did not survive, so an operator cannot tell this from a hung "+ "engine:\n%s", logged) } for _, line := range strings.Split(strings.TrimSpace(logged), "\n") { if line == "" { continue } var rec map[string]any if err := json.Unmarshal([]byte(line), &rec); err != nil { t.Fatal(err) } if rec["level"] == "ERROR" { continue } if strings.Contains(line, book) || strings.Contains(line, workdir) { t.Errorf("a %v line carries the book over a directory that is PRESENT: %s", rec["level"], line) } } } // What the journal's own stat says when it fails: the operation and the errno, never the path. // // It is the second carrier PD-139 names, and it is one the platform COMPOSES — `os.Stat` answers a // *fs.PathError whose text is the book's directory, and this error is wrapped into a spawn failure // that reaches an ERROR line through the handler. Where the platform writes the sentence it does not // have to name the book; where the sentence is another program's it is carried as written. func TestTheJournalStatFailureNamesTheOperationAndNotTheBook(t *testing.T) { if os.Geteuid() == 0 { t.Skip("the fixture denies itself a directory, which root is not denied") } const book = "bk_THISISTHEBOOKSOWNID" workdir := filepath.Join(t.TempDir(), book) if err := os.Mkdir(workdir, 0o000); err != nil { t.Fatal(err) } t.Cleanup(func() { _ = os.Chmod(workdir, 0o700) }) _, err := journalSize(workdir) if err == nil { t.Fatal("the stat succeeded over a directory with no permissions: the fixture did not produce the " + "failure it is about") } if !errors.Is(err, fs.ErrPermission) { t.Fatalf("the errno did not survive: %v — an operator cannot tell a permission from an absence", err) } if !strings.Contains(err.Error(), "stat") { t.Errorf("the operation did not survive: %v", err) } if strings.Contains(err.Error(), book) || strings.Contains(err.Error(), workdir) { t.Errorf("the error carries the book's own directory: %v", err) } }