104 lines
4.6 KiB
Go
104 lines
4.6 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
)
|
|
|
|
// artifact.go: the shared write discipline of the engine's READ-OUT FILES — the sidecars beside the
|
|
// project DB that another process (the platform, D39.81/D39.85) reads while this one runs: the
|
|
// chapter/chunk manifest (row 100), the machine bank-stop table (row 101), the bank export (row 125) and
|
|
// the engine's auto-bank (row 231 — it joined this list when the read-only surfaces started folding the
|
|
// bank from the decision files themselves, which is what made it a concurrently-read document).
|
|
//
|
|
// They have one property in common that the older sidecars did not have to care about: they are read
|
|
// CONCURRENTLY with the run that writes them. A plain os.WriteFile truncates first, so a reader that
|
|
// opens the file in that window gets a valid path, zero bytes and a parse error indistinguishable from
|
|
// a corrupt artifact. Write-then-rename makes every read see either the previous document or the next
|
|
// one, whole — the same guarantee the store gets from its transactions, at file granularity.
|
|
|
|
// writeFileAtomic replaces path with data through a temp file in the SAME directory + rename. Same
|
|
// directory is load-bearing: rename is atomic only within a filesystem, and the project dir is where the
|
|
// reader is looking anyway. Every RETURNED failure removes its temp file, so a failed write leaves the
|
|
// previous document in place and nothing beside it; a kill -9 mid-write can still leave one dot-prefixed
|
|
// temp file, which is inert (no reader looks at it) and is not worth a startup sweep.
|
|
func writeFileAtomic(path string, data []byte) error {
|
|
s, err := stageFileAtomic(path, data)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return s.commit()
|
|
}
|
|
|
|
// stagedFile is the write half of writeFileAtomic with the rename still pending: the temp file is fully
|
|
// written, synced and chmodded, so everything an ENVIRONMENT can refuse — space, permissions, I/O — has
|
|
// already been met. It exists for the writers of MORE THAN ONE document (writeDecisionFiles): staging
|
|
// every file before renaming any means an environment failure is discovered before a single byte of any
|
|
// document has moved.
|
|
type stagedFile struct {
|
|
tmp, path string
|
|
}
|
|
|
|
// stageFileAtomic prepares data for path — see stagedFile. A failure removes the temp file.
|
|
func stageFileAtomic(path string, data []byte) (*stagedFile, error) {
|
|
dir, base := filepath.Dir(path), filepath.Base(path)
|
|
f, err := os.CreateTemp(dir, "."+base+".tmp-*")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("pipeline: create temp for %s: %w", path, err)
|
|
}
|
|
tmp := f.Name()
|
|
if _, err := f.Write(data); err != nil {
|
|
f.Close()
|
|
_ = os.Remove(tmp)
|
|
return nil, fmt.Errorf("pipeline: write %s: %w", tmp, err)
|
|
}
|
|
// Sync before the rename: without it the rename's metadata can reach disk ahead of the bytes, so a host
|
|
// crash can leave a present-but-truncated document under the real name. The manifest would survive that
|
|
// (a parse failure degrades to the re-chunk), but the bank export and the stop table have no such
|
|
// validation — a consumer would read a short document as a complete one.
|
|
if err := f.Sync(); err != nil {
|
|
f.Close()
|
|
_ = os.Remove(tmp)
|
|
return nil, fmt.Errorf("pipeline: sync %s: %w", tmp, err)
|
|
}
|
|
if err := f.Close(); err != nil {
|
|
_ = os.Remove(tmp)
|
|
return nil, fmt.Errorf("pipeline: close %s: %w", tmp, err)
|
|
}
|
|
// 0o644 like the other sidecars: CreateTemp makes 0o600, and an artifact another process reads must
|
|
// not depend on that process running as the same user.
|
|
if err := os.Chmod(tmp, 0o644); err != nil {
|
|
_ = os.Remove(tmp)
|
|
return nil, fmt.Errorf("pipeline: chmod %s: %w", tmp, err)
|
|
}
|
|
return &stagedFile{tmp: tmp, path: path}, nil
|
|
}
|
|
|
|
// commit renames the staged temp over the real name. A failure removes the temp and leaves the previous
|
|
// document in place.
|
|
func (s *stagedFile) commit() error {
|
|
if err := os.Rename(s.tmp, s.path); err != nil {
|
|
_ = os.Remove(s.tmp)
|
|
return fmt.Errorf("pipeline: replace %s: %w", s.path, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// abort discards a staged temp without touching the real name.
|
|
func (s *stagedFile) abort() { _ = os.Remove(s.tmp) }
|
|
|
|
// syncDir flushes a directory's entries, so a rename that this process has already observed survives a
|
|
// host crash. Rename itself only orders data against metadata (the temp is synced first); the directory
|
|
// entry is its own write and lives until the kernel flushes it unless someone asks.
|
|
func syncDir(dir string) error {
|
|
d, err := os.Open(dir)
|
|
if err != nil {
|
|
return fmt.Errorf("pipeline: open %s to sync it: %w", dir, err)
|
|
}
|
|
defer d.Close()
|
|
if err := d.Sync(); err != nil {
|
|
return fmt.Errorf("pipeline: sync %s: %w", dir, err)
|
|
}
|
|
return nil
|
|
}
|