335 lines
13 KiB
Go
335 lines
13 KiB
Go
package runs
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"errors"
|
|
"io/fs"
|
|
"log/slog"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"textmachine/platform/internal/ingest"
|
|
)
|
|
|
|
// A BOOK'S DIRECTORY IS THAT BOOK'S IDENTITY, so it is held to the same rule as the identifier
|
|
// itself: it does not reach a line below ERROR (the zone's standard §2, PD-139, PD-99).
|
|
//
|
|
// The equivalence is not a reading of the path — it is how the intake writes it: every book taken in
|
|
// lives at `<books dir>/<book id>` (books.Receive), and five of five books on the live cold-run stand
|
|
// read that way (`select id, workdir from books`, 11.09). The fixture below therefore puts the book
|
|
// at its production shape rather than at a temporary directory, so that what the assertion looks for
|
|
// is the identifier and not a string that merely resembles one.
|
|
//
|
|
// The intake has the same pin from its own side (books.TestNoBookIdentifierReachesAnInfoLine); this
|
|
// is the reconciler's, and the reconciler is where a wedged run repeats its complaint on every pass.
|
|
func TestNoBookDirectoryReachesALineBelowError(t *testing.T) {
|
|
f := newFixture(t, "20", 500)
|
|
book := f.bookID(t)
|
|
workdir := filepath.Join(t.TempDir(), book)
|
|
if err := os.MkdirAll(workdir, 0o750); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := f.store.Pool().Exec(f.ctx, `update books set workdir = $2 where id = $1`, book, workdir); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var buf bytes.Buffer
|
|
f.svc.Log = slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
|
|
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: book, Chapters: order(10)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The directory goes away under the live run, and the engine answers the way it does about a
|
|
// project it cannot open: with the path in its own text.
|
|
if err := os.RemoveAll(workdir); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.engine.set(ingest.StatusReport{}, errors.New("tmctl: open "+workdir+"/project.db: no such file or directory"))
|
|
f.runner.alive = false
|
|
for i := 1; i <= 6; i++ {
|
|
at := f.now.Add(time.Duration(i) * time.Hour)
|
|
f.svc.Now = func() time.Time { return at }
|
|
if err := f.svc.Sweep(f.ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
lines := strings.Split(strings.TrimSpace(buf.String()), "\n")
|
|
var carriesRun, said int
|
|
for _, line := range lines {
|
|
if line == "" {
|
|
continue
|
|
}
|
|
var rec map[string]any
|
|
if err := json.Unmarshal([]byte(line), &rec); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(line, run.ID) {
|
|
carriesRun++
|
|
}
|
|
if rec["msg"] == "settlement deferred: the book's project directory cannot be read" {
|
|
said++
|
|
}
|
|
if rec["level"] == "ERROR" {
|
|
continue // an ERROR may carry a path an operator must repair — see the note below
|
|
}
|
|
if strings.Contains(line, book) {
|
|
t.Errorf("a %v line carries the book's own directory, which is its identifier: %s", rec["level"], line)
|
|
}
|
|
}
|
|
// THE BOUNDARY THE FIXTURE HAD TO CROSS, and both halves of it. Without the first the assertion
|
|
// above would be true of a run that never reached the settlement at all; without the second, of a
|
|
// logger that wrote nothing.
|
|
if said == 0 {
|
|
t.Fatalf("no pass said the book's directory could not be read, so this fixture never reached the "+
|
|
"branch it is about (%d lines)", len(lines))
|
|
}
|
|
if carriesRun == 0 {
|
|
t.Fatalf("not one of the %d lines carries the run id: the filter that must match found nothing, so "+
|
|
"the filter above proves nothing either", len(lines))
|
|
}
|
|
t.Logf("%d lines, %d of them carrying the run id, %d naming the unreadable directory", len(lines), carriesRun, said)
|
|
}
|
|
|
|
// THE SECOND CARRIER, and it was invisible to the pin above because that one holds the unit DEAD.
|
|
//
|
|
// A run whose unit is ALIVE never reaches the settlement: it is RESYNCED instead, on its own interval,
|
|
// and that call asks the engine about the same directory. Where the pin above sees one complaint at
|
|
// the end of a run, this path repeats one every `ResyncEvery` for as long as the unit lives — the
|
|
// loudest shape of the identifier-in-a-log this class is about, and the one the first round of this
|
|
// pack declared closed while it was open (adversarial pass, 11.09).
|
|
func TestNoBookDirectoryReachesALineBelowErrorWhileTheUnitIsAlive(t *testing.T) {
|
|
f := newFixture(t, "20", 500)
|
|
book := f.bookID(t)
|
|
workdir := filepath.Join(t.TempDir(), book)
|
|
if err := os.MkdirAll(workdir, 0o750); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := f.store.Pool().Exec(f.ctx, `update books set workdir = $2 where id = $1`, book, workdir); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var buf bytes.Buffer
|
|
f.svc.Log = slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
|
|
f.svc.Cfg.ResyncEvery = time.Second
|
|
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: book, Chapters: order(10)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The unit is ALIVE and the book goes away under it: the sweep resyncs rather than settles.
|
|
f.runner.alive = true
|
|
if err := os.RemoveAll(workdir); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.engine.set(ingest.StatusReport{}, errors.New("tmctl: open "+workdir+"/project.db: no such file or directory"))
|
|
for i := 1; i <= 3; i++ {
|
|
at := f.now.Add(time.Duration(i) * time.Minute)
|
|
f.svc.Now = func() time.Time { return at }
|
|
if err := f.svc.Sweep(f.ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
lines := strings.Split(strings.TrimSpace(buf.String()), "\n")
|
|
var carriesRun, said int
|
|
for _, line := range lines {
|
|
if line == "" {
|
|
continue
|
|
}
|
|
var rec map[string]any
|
|
if err := json.Unmarshal([]byte(line), &rec); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(line, run.ID) {
|
|
carriesRun++
|
|
}
|
|
if rec["msg"] == "resync deferred: the book's project directory cannot be read" {
|
|
said++
|
|
}
|
|
if rec["level"] == "ERROR" {
|
|
continue
|
|
}
|
|
if strings.Contains(line, book) {
|
|
t.Errorf("a %v line of a LIVE run carries the book's own directory: %s", rec["level"], line)
|
|
}
|
|
}
|
|
if said == 0 {
|
|
t.Fatalf("no pass resynced against the missing directory, so this fixture never reached the branch "+
|
|
"it is about (%d lines): the unit must be alive and the resync due", len(lines))
|
|
}
|
|
if carriesRun == 0 {
|
|
t.Fatalf("not one of the %d lines carries the run id: the filter that must match found nothing", len(lines))
|
|
}
|
|
t.Logf("%d lines, %d carrying the run id, %d naming the unreadable directory on a LIVE unit",
|
|
len(lines), carriesRun, said)
|
|
}
|
|
|
|
// ⛔ THE HALF THE FIRST ROUND GOT WRONG: THE DIRECTORY IS THERE AND THE ENGINE STILL NAMES IT.
|
|
//
|
|
// The rule used to be a classifier — "is the book's directory gone?" — and where the answer was no it
|
|
// carried the engine's text verbatim. An engine that fails on a FILE INSIDE a directory that is
|
|
// perfectly present names the path just the same, and this test did not see it because it picked the
|
|
// ONE engine error that has no path in it (`tmctl: exec: …`). Any other message and the pin next door
|
|
// would have failed. Found by the acceptance of 11.09 (F3); the rule is a REMOVER now, so this half
|
|
// asserts both things at once — the diagnosis survives and the identifier does not.
|
|
func TestAnEngineThatNamesTheBooksPathIsCarriedWithoutIt(t *testing.T) {
|
|
f := newFixture(t, "20", 500)
|
|
book := f.bookID(t)
|
|
workdir := filepath.Join(t.TempDir(), book)
|
|
if err := os.MkdirAll(workdir, 0o750); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := f.store.Pool().Exec(f.ctx, `update books set workdir = $2 where id = $1`, book, workdir); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var buf bytes.Buffer
|
|
f.svc.Log = slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
|
|
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: book, Chapters: order(10)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// THE DIRECTORY STAYS. What the engine cannot open is a file inside it — a project database that
|
|
// was moved, a permission, a corrupt page: the ordinary shape, and the one the classifier missed.
|
|
f.engine.set(ingest.StatusReport{}, errors.New("tmctl: open "+filepath.Join(workdir, "project.db")+": permission denied"))
|
|
f.runner.alive = false
|
|
for i := 1; i <= 3; i++ {
|
|
at := f.now.Add(time.Duration(i) * time.Hour)
|
|
f.svc.Now = func() time.Time { return at }
|
|
if err := f.svc.Sweep(f.ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
logged := buf.String()
|
|
// The diagnosis: an operator must still be able to tell this from a hung engine.
|
|
if !strings.Contains(logged, "permission denied") {
|
|
t.Fatalf("the engine's own diagnosis did not survive the removal, so an operator has nothing to go "+
|
|
"on for a failure this platform cannot name itself:\n%s", logged)
|
|
}
|
|
// The identity: and it must not be there in any form.
|
|
for _, line := range strings.Split(strings.TrimSpace(logged), "\n") {
|
|
if line == "" {
|
|
continue
|
|
}
|
|
var rec map[string]any
|
|
if err := json.Unmarshal([]byte(line), &rec); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if rec["level"] == "ERROR" {
|
|
continue
|
|
}
|
|
if strings.Contains(line, book) || strings.Contains(line, workdir) {
|
|
t.Errorf("a %v line carries the book over a directory that is PRESENT: %s", rec["level"], line)
|
|
}
|
|
}
|
|
}
|
|
|
|
// ⛔ THE LOUDEST CARRIER, PINNED WHERE IT ACTUALLY LEAKS — and the fixture the first two rounds did not
|
|
// have.
|
|
//
|
|
// The resync line repeats every `ResyncEvery` for as long as the unit lives, and its neighbour test
|
|
// removes the directory: that makes `sourceThere` answer first, so the sweep says its own sentence and
|
|
// the line carrying the ENGINE's text is never reached. Removing the redaction there therefore
|
|
// survived the whole package (acceptance of 11.09, finding 4). Here the directory is PRESENT and the
|
|
// engine fails on a file inside it — the shape that walks past every guard and into the log.
|
|
func TestTheResyncOfAPresentDirectoryCarriesTheEnginesTextWithoutTheBook(t *testing.T) {
|
|
f := newFixture(t, "20", 500)
|
|
book := f.bookID(t)
|
|
workdir := filepath.Join(t.TempDir(), book)
|
|
if err := os.MkdirAll(workdir, 0o750); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := f.store.Pool().Exec(f.ctx, `update books set workdir = $2 where id = $1`, book, workdir); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var buf bytes.Buffer
|
|
f.svc.Log = slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
|
|
f.svc.Cfg.ResyncEvery = time.Second
|
|
run, err := f.svc.Start(f.ctx, StartRequest{UserID: "u1", BookID: book, Chapters: order(10)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.svc.Spawn(f.ctx, run.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The unit is ALIVE, so the sweep RESYNCS; the directory is there, so no guard answers first; and
|
|
// what the engine cannot open is a file inside it.
|
|
f.runner.alive = true
|
|
f.engine.set(ingest.StatusReport{}, errors.New("tmctl: open "+filepath.Join(workdir, "project.db")+": database is locked"))
|
|
for i := 1; i <= 3; i++ {
|
|
at := f.now.Add(time.Duration(i) * time.Minute)
|
|
f.svc.Now = func() time.Time { return at }
|
|
if err := f.svc.Sweep(f.ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
logged := buf.String()
|
|
// THE BOUNDARY: the line this test is about must actually have been written, and it is the one
|
|
// carrying the ENGINE's words — not the sweep's own sentence about a missing directory.
|
|
if !strings.Contains(logged, `"msg":"resync failed"`) {
|
|
t.Fatalf("no pass reached the resync's own line, so this fixture never met the carrier it is "+
|
|
"about:\n%s", logged)
|
|
}
|
|
if !strings.Contains(logged, "database is locked") {
|
|
t.Fatalf("the engine's diagnosis did not survive, so an operator cannot tell this from a hung "+
|
|
"engine:\n%s", logged)
|
|
}
|
|
for _, line := range strings.Split(strings.TrimSpace(logged), "\n") {
|
|
if line == "" {
|
|
continue
|
|
}
|
|
var rec map[string]any
|
|
if err := json.Unmarshal([]byte(line), &rec); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if rec["level"] == "ERROR" {
|
|
continue
|
|
}
|
|
if strings.Contains(line, book) || strings.Contains(line, workdir) {
|
|
t.Errorf("a %v line carries the book over a directory that is PRESENT: %s", rec["level"], line)
|
|
}
|
|
}
|
|
}
|
|
|
|
// What the journal's own stat says when it fails: the operation and the errno, never the path.
|
|
//
|
|
// It is the second carrier PD-139 names, and it is one the platform COMPOSES — `os.Stat` answers a
|
|
// *fs.PathError whose text is the book's directory, and this error is wrapped into a spawn failure
|
|
// that reaches an ERROR line through the handler. Where the platform writes the sentence it does not
|
|
// have to name the book; where the sentence is another program's it is carried as written.
|
|
func TestTheJournalStatFailureNamesTheOperationAndNotTheBook(t *testing.T) {
|
|
if os.Geteuid() == 0 {
|
|
t.Skip("the fixture denies itself a directory, which root is not denied")
|
|
}
|
|
const book = "bk_THISISTHEBOOKSOWNID"
|
|
workdir := filepath.Join(t.TempDir(), book)
|
|
if err := os.Mkdir(workdir, 0o000); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = os.Chmod(workdir, 0o700) })
|
|
_, err := journalSize(workdir)
|
|
if err == nil {
|
|
t.Fatal("the stat succeeded over a directory with no permissions: the fixture did not produce the " +
|
|
"failure it is about")
|
|
}
|
|
if !errors.Is(err, fs.ErrPermission) {
|
|
t.Fatalf("the errno did not survive: %v — an operator cannot tell a permission from an absence", err)
|
|
}
|
|
if !strings.Contains(err.Error(), "stat") {
|
|
t.Errorf("the operation did not survive: %v", err)
|
|
}
|
|
if strings.Contains(err.Error(), book) || strings.Contains(err.Error(), workdir) {
|
|
t.Errorf("the error carries the book's own directory: %v", err)
|
|
}
|
|
}
|