package pipeline import ( "fmt" "os" "path/filepath" ) // artifact.go: the shared write discipline of the engine's READ-OUT FILES — the sidecars beside the // project DB that another process (the platform, D39.81/D39.85) reads while this one runs: the // chapter/chunk manifest (row 100), the machine bank-stop table (row 101), the bank export (row 125) and // the engine's auto-bank (row 231 — it joined this list when the read-only surfaces started folding the // bank from the decision files themselves, which is what made it a concurrently-read document). // // They have one property in common that the older sidecars did not have to care about: they are read // CONCURRENTLY with the run that writes them. A plain os.WriteFile truncates first, so a reader that // opens the file in that window gets a valid path, zero bytes and a parse error indistinguishable from // a corrupt artifact. Write-then-rename makes every read see either the previous document or the next // one, whole — the same guarantee the store gets from its transactions, at file granularity. // writeFileAtomic replaces path with data through a temp file in the SAME directory + rename. Same // directory is load-bearing: rename is atomic only within a filesystem, and the project dir is where the // reader is looking anyway. Every RETURNED failure removes its temp file, so a failed write leaves the // previous document in place and nothing beside it; a kill -9 mid-write can still leave one dot-prefixed // temp file, which is inert (no reader looks at it) and is not worth a startup sweep. func writeFileAtomic(path string, data []byte) error { s, err := stageFileAtomic(path, data) if err != nil { return err } return s.commit() } // stagedFile is the write half of writeFileAtomic with the rename still pending: the temp file is fully // written, synced and chmodded, so everything an ENVIRONMENT can refuse — space, permissions, I/O — has // already been met. It exists for the writers of MORE THAN ONE document (writeDecisionFiles): staging // every file before renaming any means an environment failure is discovered before a single byte of any // document has moved. type stagedFile struct { tmp, path string } // stageFileAtomic prepares data for path — see stagedFile. A failure removes the temp file. func stageFileAtomic(path string, data []byte) (*stagedFile, error) { dir, base := filepath.Dir(path), filepath.Base(path) f, err := os.CreateTemp(dir, "."+base+".tmp-*") if err != nil { return nil, fmt.Errorf("pipeline: create temp for %s: %w", path, err) } tmp := f.Name() if _, err := f.Write(data); err != nil { f.Close() _ = os.Remove(tmp) return nil, fmt.Errorf("pipeline: write %s: %w", tmp, err) } // Sync before the rename: without it the rename's metadata can reach disk ahead of the bytes, so a host // crash can leave a present-but-truncated document under the real name. The manifest would survive that // (a parse failure degrades to the re-chunk), but the bank export and the stop table have no such // validation — a consumer would read a short document as a complete one. if err := f.Sync(); err != nil { f.Close() _ = os.Remove(tmp) return nil, fmt.Errorf("pipeline: sync %s: %w", tmp, err) } if err := f.Close(); err != nil { _ = os.Remove(tmp) return nil, fmt.Errorf("pipeline: close %s: %w", tmp, err) } // 0o644 like the other sidecars: CreateTemp makes 0o600, and an artifact another process reads must // not depend on that process running as the same user. if err := os.Chmod(tmp, 0o644); err != nil { _ = os.Remove(tmp) return nil, fmt.Errorf("pipeline: chmod %s: %w", tmp, err) } return &stagedFile{tmp: tmp, path: path}, nil } // commit renames the staged temp over the real name. A failure removes the temp and leaves the previous // document in place. func (s *stagedFile) commit() error { if err := os.Rename(s.tmp, s.path); err != nil { _ = os.Remove(s.tmp) return fmt.Errorf("pipeline: replace %s: %w", s.path, err) } return nil } // abort discards a staged temp without touching the real name. func (s *stagedFile) abort() { _ = os.Remove(s.tmp) } // syncDir flushes a directory's entries, so a rename that this process has already observed survives a // host crash. Rename itself only orders data against metadata (the temp is synced first); the directory // entry is its own write and lives until the kernel flushes it unless someone asks. func syncDir(dir string) error { d, err := os.Open(dir) if err != nil { return fmt.Errorf("pipeline: open %s to sync it: %w", dir, err) } defer d.Close() if err := d.Sync(); err != nil { return fmt.Errorf("pipeline: sync %s: %w", dir, err) } return nil }