879 lines
42 KiB
Go
879 lines
42 KiB
Go
// Package config reads the service's settings from the environment.
|
||
package config
|
||
|
||
import (
|
||
"errors"
|
||
"fmt"
|
||
"log/slog"
|
||
"net/url"
|
||
"os"
|
||
"path/filepath"
|
||
"regexp"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
"textmachine/platform/internal/books"
|
||
"textmachine/platform/internal/exports"
|
||
"textmachine/platform/internal/login"
|
||
"textmachine/platform/internal/money"
|
||
"textmachine/platform/internal/pgstore"
|
||
"textmachine/platform/internal/pricing"
|
||
)
|
||
|
||
// Config is the whole configuration surface. Environment only: a control plane is deployed, not
|
||
// hand-run, and a settings file is one more thing to keep in sync with the deployment. Ratified
|
||
// 09.08 (register row PD-114) together with the obligation this file now also carries — an operator
|
||
// must be able to read what actually applied, which is what Settings and LogEffective are for.
|
||
type Config struct {
|
||
Addr string
|
||
// DSN is the Postgres connection string. Empty is allowed and means "start unready": the
|
||
// service answers liveness and reports on /readyz why it cannot serve.
|
||
//
|
||
// ⚠ It carries a password. It is read from the environment, never logged, and never echoed
|
||
// into an error message.
|
||
DSN string
|
||
// TrustedOrigins are origins besides our own that may make unsafe requests.
|
||
TrustedOrigins []string
|
||
// SessionIdleTTL is how long a session survives without use; SessionMaxAge is the ceiling no
|
||
// amount of use can extend. Both are policy, and the policy — the two values, the concurrent
|
||
// session rule and what our session does when the provider's ends — is written down in
|
||
// STACK_DECISIONS §13, because ASVS 5.0 7.1.1 asks for the document, not the number.
|
||
SessionIdleTTL time.Duration
|
||
SessionMaxAge time.Duration
|
||
// Migrate applies pending migrations at boot. Off by default: a rollout should migrate once,
|
||
// deliberately, not once per replica.
|
||
Migrate bool
|
||
// InsecureCookies serves the session over plain HTTP under a different cookie name. A DEV
|
||
// switch: __Host- requires Secure, so localhost cannot use the production name at all (PD-8).
|
||
InsecureCookies bool
|
||
// OIDC is the sign-in provider. Empty issuer means no login surface is mounted — the service
|
||
// still runs, which is what keeps a bare `go run` useful.
|
||
// OIDCProvider is OUR name for the issuer and the first half of the identity key. It is
|
||
// configured next to the issuer because the two must move together: pointing the issuer at a
|
||
// different IdP while keeping the name would file that IdP's subjects under the old provider —
|
||
// the silent account-linking the identity model exists to prevent.
|
||
OIDCProvider string
|
||
OIDCIssuer string
|
||
OIDCClientID string
|
||
OIDCClientSecret string
|
||
OIDCRedirectURL string
|
||
// AfterLogin is where a completed sign-in lands.
|
||
AfterLogin string
|
||
// DevLoginSubject names the ONE identity a development stand can sign in as, and its presence is
|
||
// what mounts that endpoint at all (login.Dev). Empty — the default and what every deployment
|
||
// file in this repository sets — means the endpoint does not exist.
|
||
//
|
||
// ⚠ Setting it TOGETHER with an OIDC provider is refused at boot rather than warned about: a
|
||
// deployment that has a real identity provider and a keyless door into a session is the one shape
|
||
// of this feature that is dangerous, and the operator who would create it is copying a dev unit
|
||
// file onto a real host, where a warning in a log is read by nobody.
|
||
DevLoginSubject string
|
||
// SignupGrantMicroUSD is the credit a new account is created with.
|
||
//
|
||
// ⚠ ZERO by default (owner, 16.08, D39.138 п.2л): during the beta an account is credited BY HAND
|
||
// with `tmplatformctl grant`. The five dollars this used to default to comes back together with
|
||
// the daily aggregate cap that has to guard it, and that cap belongs with payments — until then
|
||
// a self-service grant is an unbounded one, rate-limited only by how fast subjects can be
|
||
// created. An operator who wants the free tier back sets TM_PLATFORM_SIGNUP_GRANT_USD.
|
||
SignupGrantMicroUSD int64
|
||
// MetricsAddr is where the telemetry listener binds. A SEPARATE listener from the API on
|
||
// purpose — see the runner's own note — and empty turns it off.
|
||
MetricsAddr string
|
||
|
||
// Runner is everything about starting engine runs.
|
||
Runner RunnerConfig
|
||
// Intake is everything about receiving a book.
|
||
Intake IntakeConfig
|
||
// Export is everything about handing a built book back.
|
||
Export ExportConfig
|
||
// Backup is everything about keeping a copy of what this deployment cannot re-create.
|
||
Backup BackupConfig
|
||
// LanguagePairs is what this DEPLOYMENT can translate, as `GET /capabilities` answers it and as
|
||
// the intake judges an upload by.
|
||
//
|
||
// It is configuration and not code, and that is the engine-generality invariant rather than a
|
||
// preference: which pairs exist is DATA (a prompt pack and a langpack the operator deploys), and
|
||
// a platform that shipped a default list would be a second place to teach every time a pair is
|
||
// added. An unset list means the deployment declares nothing — see IntakeConfig.
|
||
LanguagePairs []LanguagePair
|
||
|
||
// Settings is every value above with where it came from, in the order it was read. It is what
|
||
// the boot line prints (PD-114) and it carries no secret and no money amount — see Setting.
|
||
Settings []Setting
|
||
}
|
||
|
||
// RunnerConfig is the runner's operator-facing surface.
|
||
type RunnerConfig struct {
|
||
// EngineBinary is the VERSIONED path of tmctl. Empty means no run can be spawned, which is the
|
||
// right default for an instance that only serves reads. A version-bearing path is the point
|
||
// (unified backlog row 139): the engine ships more often than a run finishes, and an attempt is
|
||
// pinned to the path it started with.
|
||
EngineBinary string
|
||
// StateDir is where the platform keeps its own run state (exit markers). Never the book's
|
||
// directory: the engine owns that one.
|
||
StateDir string
|
||
// MarkerBinary is the command systemd runs as ExecStopPost to record how a unit ended. It is
|
||
// this project's own admin CLI; empty means "the tmplatformctl next to the running daemon".
|
||
MarkerBinary string
|
||
// CeilingArg is how the run's ceiling reaches the engine, as an argv template containing {{usd}}.
|
||
// Defaulted to the landed form (row 145, D39.122) and overridable for a build that spells the flag
|
||
// differently; an override that does not carry {{usd}} is refused rather than sent as a literal.
|
||
CeilingArg string
|
||
// KeysFile is the DEPLOYMENT's provider-key file, handed to `translate` as `--keys-file`
|
||
// (row 211). The path travels as an engine argument and the keys never pass through this
|
||
// process or the unit's environment. Empty means the flag is not passed — the engine then
|
||
// depends on a `.env` beside each book, which nothing on the SaaS path writes.
|
||
//
|
||
// ⚠ The variable is `_PATH`, not `_FILE`, and the suffix is load-bearing: in this service's own
|
||
// convention `KEY_FILE` names a file whose CONTENT replaces KEY's value (see loader.secret) —
|
||
// this value IS the path, read by the engine and never by this process.
|
||
KeysFile string
|
||
// MemoryMax and TasksMax bound ONE run's cgroup (PD-13).
|
||
MemoryMax string
|
||
TasksMax int
|
||
// AllowEngineVersionChange lets a RESUME move to the currently deployed engine build instead of
|
||
// the one its run started with (unified backlog row 139: another version only on an explicit say-so).
|
||
AllowEngineVersionChange bool
|
||
// SweepEvery is how often the reconciler reads the world.
|
||
SweepEvery time.Duration
|
||
// SweepBudget is what ONE pass of the run sweep may take, and RunBudget what one run inside it
|
||
// may take.
|
||
//
|
||
// ⚠ They are an operator's numbers because the ratio between them is a policy and it bit: at
|
||
// 120 s and 60 s, TWO runs whose engine hangs are the whole pass. The pass is split between its
|
||
// phases now, so the ratio no longer decides whether money is counted at all — but a deployment
|
||
// whose engine is slower than this one's still needs to be able to say so without a rebuild, and
|
||
// `RunBudget` in particular was a field this code DECLARED and never assigned: a knob that
|
||
// existed in the struct, was documented, and did nothing.
|
||
SweepBudget time.Duration
|
||
RunBudget time.Duration
|
||
// ResyncEvery is how often a LIVE run is refreshed from `tmctl status --json`. Slow on purpose:
|
||
// each call costs seconds of CPU (unified backlog row 100).
|
||
ResyncEvery time.Duration
|
||
// HoldFactorPercent is `k`: the cushion a hold carries over the engine's expected bill, as an
|
||
// integer percentage. Always set: the default is resolved HERE and not by whoever reads the field,
|
||
// so there is one answer to "what cushion is this instance using" instead of one per caller.
|
||
//
|
||
// ⚠ IT REPLACED A PER-CHAPTER RATE, and the difference is not a rename. The old variable set what
|
||
// a chapter COST — a constant, measured 4.47× low (D39.179 §1), which no operator could have set
|
||
// right because the answer differs per book and per chapter. What a chapter costs is now read from
|
||
// the engine's own projection; what remains for an operator to choose is only how much room to
|
||
// leave above it. Provenance and derivation: pricing.DefaultHoldFactorPercent — and it is DERIVED,
|
||
// not measured, until unified backlog row 281 measures it.
|
||
//
|
||
// ⚠ The old variable's NAME is deliberately not written here, and that is not squeamishness: the
|
||
// completeness gate (config_test, TestEverySettingThisServiceReadsIsPrinted) reads every
|
||
// `TM_PLATFORM_*` token out of THIS FILE and demands the boot log print it, so a retired name left
|
||
// in a comment would be reported as a setting nobody records. Its retirement is asserted by name
|
||
// in the test that owns that fact.
|
||
HoldFactorPercent int
|
||
// Workers is how many queue workers spawn units concurrently.
|
||
Workers int
|
||
}
|
||
|
||
// ExportConfig is the export door's operator-facing surface (canon §createExport/§getExport).
|
||
type ExportConfig struct {
|
||
// Formats are the formats this deployment declares its engine can write, in the order
|
||
// `GET /capabilities` announces them. EMPTY means no export door at all: the two routes are not
|
||
// mounted and `export_formats` answers `[]`, which is the same shape every unbuilt part of the
|
||
// contract has.
|
||
//
|
||
// ⚠ A DECLARATION and not a discovery, and the seam's law is why. The engine's own list is a Go
|
||
// variable inside a module this one may never import (D39.85), and no $0 engine command
|
||
// publishes it as data. `TM_PLATFORM_LANGUAGE_PAIRS` is a declaration for exactly the same
|
||
// reason, and this follows it. A format declared here that the deployed engine does not know
|
||
// comes back as a FAILED export naming the operator's configuration — never as a format
|
||
// announced on `/capabilities` that silently produces nothing.
|
||
Formats []string
|
||
// Dir is where built artifacts live; empty takes `<StateDir>/exports`. Platform state and never
|
||
// the book's directory: that one belongs to the engine (D39.110), and a file this platform
|
||
// expires out from under it would be a write into somebody else's tree.
|
||
Dir string
|
||
// TTL is how long a built artifact and its link live. It bounds DISK and not access — the link
|
||
// is authenticated on every request — so what it buys is that a book's whole text does not sit
|
||
// on a control-plane disk forever after the download it was built for.
|
||
TTL time.Duration
|
||
}
|
||
|
||
// BackupConfig is the restore-point maker's operator-facing surface.
|
||
//
|
||
// ⚠ It is OFF by default, and that is the honest default rather than a safe one: a directory this
|
||
// service invented would land on the same disk as the books and the database, which is the one
|
||
// failure it exists to survive. An operator has to say where, and the boot line says loudly when
|
||
// nobody has (runner.go).
|
||
type BackupConfig struct {
|
||
// Dir is where restore points are written. Empty means this deployment takes none.
|
||
//
|
||
// Absolute for the same reason StateDir and BooksDir are, and with one requirement this service
|
||
// cannot check: a DIFFERENT device from the books and the database, or synchronised off the host.
|
||
// A backup beside what it backs up survives a mistake and not a disk.
|
||
Dir string
|
||
// Every is how often a restore point is taken; Keep is how many survive.
|
||
//
|
||
// Six hours and fourteen points — 84 hours of history, of which the oldest LIVE point is 13
|
||
// intervals back, about 3.25 days — because the two costs pull opposite
|
||
// ways and neither is free: each point costs a full `VACUUM INTO` per book plus a whole dump, and
|
||
// each one kept costs that much disk. Four days is what covers a fault discovered after a
|
||
// weekend, which is the interval that actually decides whether a copy is still there when
|
||
// somebody looks.
|
||
Every time.Duration
|
||
Keep int
|
||
// PgDumpBin and PgRestoreBin are PostgreSQL's own tools — a proven third-party tool over
|
||
// anything of ours (ENGINEERING_STANDARDS §1), and the format they read is the one an operator
|
||
// restoring by hand already knows.
|
||
//
|
||
// ⚠ Their MAJOR version must match the SERVER's: pg_dump refuses to dump a server newer than
|
||
// itself, and the symptom is a backup pass that has been failing since an upgrade nobody
|
||
// connected to it.
|
||
PgDumpBin string
|
||
PgRestoreBin string
|
||
}
|
||
|
||
// IntakeConfig is the book upload's operator-facing surface.
|
||
type IntakeConfig struct {
|
||
// BooksDir is the root under which the platform creates one directory per uploaded book. Empty
|
||
// means this deployment takes no uploads, and then POST /books is not mounted at all: an
|
||
// instance that accepted files it has nowhere to put would reject every one of them after the
|
||
// upload rather than before it.
|
||
//
|
||
// Absolute, for the same reason StateDir is: a relative root names a different directory to the
|
||
// daemon and to anything else that reads a book's workdir out of the database.
|
||
BooksDir string
|
||
// MaxUploadBytes caps ONE upload's request body, and it is the per-route limit PD-72 was opened
|
||
// for. It is a refusal and not a sizing: the acceptance corpus is a 23 MB book.
|
||
MaxUploadBytes int64
|
||
// UploadDeadline is how long one upload's body may take to arrive. It REPLACES the server's
|
||
// 30-second ReadTimeout for that route alone and stays finite; clearing a deadline instead is
|
||
// what re-created PD-2 and is forbidden (STACK_DECISIONS §12).
|
||
UploadDeadline time.Duration
|
||
// BookTemplate is the operator's starting `book.yaml`, from which every new book's own is
|
||
// rendered once (form Б, D39.130). A deploy ARTIFACT and not code: the pipeline and models paths,
|
||
// the ceilings and whatever else the engine's schema carries come from it unread, and the
|
||
// platform fills in only the book's identity, its languages and its source file name.
|
||
//
|
||
// Absolute, for the same reason StateDir and BooksDir are — and the paths INSIDE it should be
|
||
// absolute too, because the engine resolves them against each book's own directory rather than
|
||
// against the template's. Empty means this deployment provisions books by hand, which is what
|
||
// every deployment did before the form was ratified.
|
||
BookTemplate string
|
||
}
|
||
|
||
// LanguagePair is one translation direction this deployment knows about.
|
||
//
|
||
// "Knows about" and "can run" are different facts to a user waiting for one, which is why an
|
||
// unavailable pair is LISTED rather than omitted (canon §Capabilities).
|
||
type LanguagePair struct {
|
||
Source string
|
||
Target string
|
||
Available bool
|
||
}
|
||
|
||
// pairSpec is the configured form of one pair: `zh>ru` for a pair this deployment can run, and
|
||
// `ja>ru:unavailable` for one it knows about and cannot.
|
||
var pairSpec = regexp.MustCompile(`^([a-z]{2,3}(?:-[A-Za-z0-9]{2,8})*)>([a-z]{2,3}(?:-[A-Za-z0-9]{2,8})*)(:unavailable)?$`)
|
||
|
||
// parsePairs reads TM_PLATFORM_LANGUAGE_PAIRS.
|
||
//
|
||
// The separator between the two codes is `>` and not `-`, because a language code legally contains
|
||
// a hyphen (`zh-Hans`) and "zh-Hans-ru" has two readings. Refused rather than guessed at boot: a
|
||
// deployment whose declared pairs are wrong refuses uploads it should take, and the operator's
|
||
// symptom would be a rejection with no cause.
|
||
func parsePairs(raw string) ([]LanguagePair, error) {
|
||
var out []LanguagePair
|
||
for _, spec := range strings.Split(raw, ",") {
|
||
spec = strings.TrimSpace(spec)
|
||
if spec == "" {
|
||
continue
|
||
}
|
||
m := pairSpec.FindStringSubmatch(spec)
|
||
if m == nil {
|
||
return nil, fmt.Errorf("config: TM_PLATFORM_LANGUAGE_PAIRS: %q is not `<source>><target>[:unavailable]`, e.g. zh>ru", spec)
|
||
}
|
||
out = append(out, LanguagePair{Source: m[1], Target: m[2], Available: m[3] == ""})
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
// Setting is one resolved value as an operator may read it in a log.
|
||
type Setting struct {
|
||
// Key is the environment variable.
|
||
Key string
|
||
// Value is what applied — or a REDACTION where the value may not be logged. Two classes are
|
||
// redacted and both are rules of this project rather than taste: a secret (a DSN carries a
|
||
// password; §11 keeps it out of the process environment for the same reason), and any amount of
|
||
// MONEY, which does not reach an INFO log in any form (D39.84, PD-99). For those the source is
|
||
// the whole point — "did my file get picked up" is answerable without the value.
|
||
Value string
|
||
// Source is where the value came from: default, environment, or a *_FILE.
|
||
Source string
|
||
}
|
||
|
||
const (
|
||
sourceDefault = "default"
|
||
sourceEnv = "environment"
|
||
sourceFile = "file"
|
||
)
|
||
|
||
// redacted values. They are written out rather than left empty so that a line always says something:
|
||
// an empty value column reads as "unset", which is a different fact.
|
||
const (
|
||
valueSet = "(set)"
|
||
valueUnset = "(unset)"
|
||
valueAmount = "(an amount; not logged)"
|
||
)
|
||
|
||
// Load reads the environment.
|
||
func Load() (Config, error) {
|
||
var l loader
|
||
c := Config{
|
||
Addr: l.env("TM_PLATFORM_ADDR", "127.0.0.1:8080"),
|
||
OIDCProvider: l.env("TM_PLATFORM_OIDC_PROVIDER", "google"),
|
||
OIDCIssuer: l.env("TM_PLATFORM_OIDC_ISSUER", ""),
|
||
OIDCClientID: l.env("TM_PLATFORM_OIDC_CLIENT_ID", ""),
|
||
OIDCRedirectURL: l.env("TM_PLATFORM_OIDC_REDIRECT_URL", ""),
|
||
AfterLogin: l.env("TM_PLATFORM_AFTER_LOGIN", "/"),
|
||
DevLoginSubject: strings.TrimSpace(l.env("TM_PLATFORM_DEV_LOGIN", "")),
|
||
MetricsAddr: metricsAddr(l.env("TM_PLATFORM_METRICS_ADDR", "127.0.0.1:9464")),
|
||
SignupGrantMicroUSD: 0,
|
||
SessionIdleTTL: defaultSessionIdle,
|
||
SessionMaxAge: defaultSessionMaxAge,
|
||
}
|
||
if raw := l.env("TM_PLATFORM_TRUSTED_ORIGINS", ""); raw != "" {
|
||
for _, o := range strings.Split(raw, ",") {
|
||
if o = strings.TrimSpace(o); o != "" {
|
||
c.TrustedOrigins = append(c.TrustedOrigins, o)
|
||
}
|
||
}
|
||
}
|
||
var err error
|
||
if c.Migrate, err = l.boolean("TM_PLATFORM_MIGRATE"); err != nil {
|
||
return Config{}, err
|
||
}
|
||
if c.InsecureCookies, err = l.boolean("TM_PLATFORM_INSECURE_COOKIES"); err != nil {
|
||
return Config{}, err
|
||
}
|
||
if c.DSN, err = l.secret("TM_PLATFORM_DSN"); err != nil {
|
||
return Config{}, err
|
||
}
|
||
if c.OIDCClientSecret, err = l.secret("TM_PLATFORM_OIDC_CLIENT_SECRET"); err != nil {
|
||
return Config{}, err
|
||
}
|
||
if c.SessionIdleTTL, c.SessionMaxAge, err = sessionTTLs(&l); err != nil {
|
||
return Config{}, err
|
||
}
|
||
if c.SignupGrantMicroUSD, err = l.amount("TM_PLATFORM_SIGNUP_GRANT_USD", c.SignupGrantMicroUSD, false); err != nil {
|
||
return Config{}, err
|
||
}
|
||
if c.LanguagePairs, err = parsePairs(l.env("TM_PLATFORM_LANGUAGE_PAIRS", "")); err != nil {
|
||
return Config{}, err
|
||
}
|
||
if err := c.loadRunner(&l); err != nil {
|
||
return Config{}, err
|
||
}
|
||
if err := c.loadExport(&l); err != nil {
|
||
return Config{}, err
|
||
}
|
||
if err := c.loadIntake(&l); err != nil {
|
||
return Config{}, err
|
||
}
|
||
if err := c.loadBackup(&l); err != nil {
|
||
return Config{}, err
|
||
}
|
||
// An intake needs at least one AVAILABLE pair, not merely a non-empty list: the intake judges an
|
||
// upload by the available half, so a deployment declaring only `:unavailable` ones would answer
|
||
// `/capabilities` with nothing translatable and still take every book (canon §Capabilities).
|
||
if c.IntakeEnabled() && len(c.AvailablePairs()) == 0 {
|
||
return Config{}, errors.New("config: an instance that takes uploads must declare an available TM_PLATFORM_LANGUAGE_PAIRS entry, e.g. zh>ru")
|
||
}
|
||
// Half a login configuration is worse than none: the surface would mount and fail at the first
|
||
// click instead of at boot, where an operator is watching.
|
||
oidc := []string{c.OIDCIssuer, c.OIDCClientID, c.OIDCClientSecret, c.OIDCRedirectURL}
|
||
set := 0
|
||
for _, v := range oidc {
|
||
if v != "" {
|
||
set++
|
||
}
|
||
}
|
||
if set != 0 && set != len(oidc) {
|
||
return Config{}, errors.New("config: OIDC needs all of TM_PLATFORM_OIDC_ISSUER, _CLIENT_ID, _CLIENT_SECRET, _REDIRECT_URL, or none")
|
||
}
|
||
// The one shape of the development sign-in that is dangerous, refused where an operator is
|
||
// watching. A deployment with a real identity provider and a keyless door into a session is
|
||
// account takeover, and the way it would come about is real: a dev unit file copied onto a host
|
||
// that has the production environment. A warning in a log would not be read; a service that does
|
||
// not start is.
|
||
if c.DevLoginSubject != "" && set != 0 {
|
||
return Config{}, errors.New("config: TM_PLATFORM_DEV_LOGIN is a development sign-in and cannot be enabled on a deployment that has an OIDC provider: unset one of them")
|
||
}
|
||
// The other half of the same accident, and the acceptance reached it with a live daemon: the
|
||
// development environment sets TWO variables, and dropping only the sign-in from it still leaves
|
||
// a service that serves real Google sessions in cookies without Secure and with HSTS off.
|
||
//
|
||
// What decides is not a guess about the deployment but its own address: the OIDC callback IS this
|
||
// platform's public URL, so its scheme says whether the service is reached over TLS. Plain http
|
||
// there is a stand, where __Host- and Secure cannot work at all and the switch is legitimate —
|
||
// anything else contradicts the switch, and one of the two is then wrong in a way that costs
|
||
// sessions.
|
||
if c.InsecureCookies && set != 0 && !servedOverPlainHTTP(c.OIDCRedirectURL) {
|
||
return Config{}, fmt.Errorf("config: TM_PLATFORM_INSECURE_COOKIES serves the session without Secure and turns HSTS off, which this deployment's own callback contradicts (TM_PLATFORM_OIDC_REDIRECT_URL=%q): unset one of them", c.OIDCRedirectURL)
|
||
}
|
||
// The development provider's NAME is reserved, and this is the guard an adversarial review of
|
||
// this pack asked for. The identity key is (provider, subject) and `TM_PLATFORM_OIDC_PROVIDER` is
|
||
// free-form operator input, so an issuer configured under the name `dev` files its subjects in
|
||
// the same namespace a development stand writes into — and a `sub` that happens to equal the dev
|
||
// subject then resolves to the DEV account, with its seeded credit. That is the account-linking
|
||
// class §10a warns about, and the only reason the development sign-in can claim a namespace of
|
||
// its own is that this refusal exists.
|
||
if c.OIDCProvider == login.DevProvider {
|
||
return Config{}, fmt.Errorf("config: TM_PLATFORM_OIDC_PROVIDER may not be %q: that name is the development sign-in's identity namespace, and sharing it would file an issuer's subjects in it", login.DevProvider)
|
||
}
|
||
c.Settings = l.settings
|
||
return c, nil
|
||
}
|
||
|
||
// defaultSessionIdle and defaultSessionMaxAge are the session policy's two clocks.
|
||
//
|
||
// 14 days idle, 30 days absolute. The absolute one is the NIST SP 800-63B-4 AAL1 figure ("SHOULD be
|
||
// no more than 30 days"), not a preference: it was 90 days, and a deviation from a SHOULD needs a
|
||
// reason that survives inspection, which that one did not (PD-58, §13).
|
||
const (
|
||
defaultSessionIdle = 14 * 24 * time.Hour
|
||
defaultSessionMaxAge = 30 * 24 * time.Hour
|
||
)
|
||
|
||
// SessionTTLs reads the session policy from the environment, for a caller that mints a session
|
||
// without being the daemon — `tmplatformctl token issue`.
|
||
//
|
||
// ⚠ It exists so there is ONE definition of the policy rather than two. A second reader with its own
|
||
// defaults is how a token minted by the operator comes to outlive, or to die before, the sessions
|
||
// the deployment says it issues — and the two would drift in the direction nobody is watching,
|
||
// which is this repository's most repeated failure (PD-374, PD-432, the battery's condition list).
|
||
func SessionTTLs() (idle, maxAge time.Duration, err error) {
|
||
var l loader
|
||
return sessionTTLs(&l)
|
||
}
|
||
|
||
func sessionTTLs(l *loader) (idle, maxAge time.Duration, err error) {
|
||
if idle, err = l.duration("TM_PLATFORM_SESSION_IDLE", defaultSessionIdle); err != nil {
|
||
return 0, 0, err
|
||
}
|
||
if maxAge, err = l.duration("TM_PLATFORM_SESSION_MAX_AGE", defaultSessionMaxAge); err != nil {
|
||
return 0, 0, err
|
||
}
|
||
if idle > maxAge {
|
||
return 0, 0, fmt.Errorf("config: session idle TTL %s exceeds max age %s", idle, maxAge)
|
||
}
|
||
return idle, maxAge, nil
|
||
}
|
||
|
||
// servedOverPlainHTTP reads this deployment's own scheme off the OIDC callback, which is the one
|
||
// configured value that names the platform's public address.
|
||
//
|
||
// A URL that cannot be parsed answers NO: the question is whether this is demonstrably a plain-http
|
||
// stand, and a value nothing can make sense of demonstrates nothing.
|
||
func servedOverPlainHTTP(callback string) bool {
|
||
u, err := url.Parse(callback)
|
||
return err == nil && strings.EqualFold(u.Scheme, "http")
|
||
}
|
||
|
||
// metricsAddr turns the operator's word for "do not serve metrics" into the empty address the
|
||
// listener understands.
|
||
//
|
||
// A word is needed because an EMPTY environment variable is indistinguishable from an unset one to
|
||
// os.Getenv, and unset takes the default — so without this the switch documented as "empty turns it
|
||
// off" could not be reached from the environment at all.
|
||
func metricsAddr(v string) string {
|
||
if strings.EqualFold(v, "off") || strings.EqualFold(v, "none") {
|
||
return ""
|
||
}
|
||
return v
|
||
}
|
||
|
||
// LogEffective prints what actually applied, one line per setting (PD-114, ratified 09.08).
|
||
//
|
||
// It exists because an environment-only configuration has no artifact an operator can read back: a
|
||
// deployment whose EnvironmentFile= did not load looks exactly like one whose values were chosen,
|
||
// and the difference used to be discoverable only from behaviour. The line carries the SOURCE as
|
||
// well as the value, because "did my setting take effect" is the actual question and a value alone
|
||
// cannot answer it — a default and an environment variable holding the same string are the same line
|
||
// otherwise.
|
||
//
|
||
// Secrets and money amounts are redacted here rather than at the call site (see Setting.Value).
|
||
func (c Config) LogEffective(log *slog.Logger) {
|
||
for _, s := range c.Settings {
|
||
log.Info("config", "key", s.Key, "value", s.Value, "source", s.Source)
|
||
}
|
||
}
|
||
|
||
// LoginEnabled reports whether a sign-in provider is configured.
|
||
func (c Config) LoginEnabled() bool { return c.OIDCIssuer != "" }
|
||
|
||
// DevLoginEnabled reports whether this deployment mounts the development sign-in. Load has already
|
||
// refused the combination where that would be dangerous, so this is a plain question.
|
||
func (c Config) DevLoginEnabled() bool { return c.DevLoginSubject != "" }
|
||
|
||
// RunsEnabled reports whether this instance can spawn engine runs at all. An instance without an
|
||
// engine binary still serves every read: the library and the book card are not run machinery.
|
||
func (c Config) RunsEnabled() bool { return c.Runner.EngineBinary != "" }
|
||
|
||
// IntakeEnabled reports whether this instance can receive a book. It needs somewhere to put the file
|
||
// AND an engine to cut it with: parsing is one call of tmctl, so a deployment without the binary
|
||
// could only take uploads it would have to reject.
|
||
func (c Config) IntakeEnabled() bool { return c.Intake.BooksDir != "" && c.RunsEnabled() }
|
||
|
||
// AvailablePairs is what this deployment can actually run — the half the intake judges by. One place
|
||
// computes it, so the boot check and the intake cannot disagree about what "declared" means.
|
||
func (c Config) AvailablePairs() []LanguagePair {
|
||
var out []LanguagePair
|
||
for _, p := range c.LanguagePairs {
|
||
if p.Available {
|
||
out = append(out, p)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// loadRunner reads the runner's settings.
|
||
func (c *Config) loadRunner(l *loader) error {
|
||
r := RunnerConfig{
|
||
EngineBinary: l.env("TM_PLATFORM_ENGINE_BIN", ""),
|
||
StateDir: l.env("TM_PLATFORM_STATE_DIR", "/var/lib/tmplatform"),
|
||
MarkerBinary: l.env("TM_PLATFORM_CTL_BIN", ""),
|
||
CeilingArg: l.env("TM_PLATFORM_ENGINE_CEILING_ARG", "--ceiling-usd {{usd}}"),
|
||
KeysFile: l.env("TM_PLATFORM_ENGINE_KEYS_PATH", ""),
|
||
// A run gets a generous but finite share of the machine. The figure is a BACKSTOP, not a
|
||
// sizing: a translation is bounded by its ceiling in dollars, and this bounds the one way a
|
||
// single run can hurt the others regardless of money (PD-13).
|
||
MemoryMax: l.env("TM_PLATFORM_RUN_MEMORY_MAX", "4G"),
|
||
TasksMax: 64,
|
||
SweepEvery: 15 * time.Second,
|
||
SweepBudget: 2 * time.Minute,
|
||
RunBudget: 60 * time.Second,
|
||
ResyncEvery: 5 * time.Minute,
|
||
HoldFactorPercent: pricing.DefaultHoldFactorPercent,
|
||
Workers: 4,
|
||
}
|
||
// An ABSOLUTE state directory or none. The exit marker is written by a systemd unit whose
|
||
// WorkingDirectory is the BOOK's directory and read by this daemon from its own working directory,
|
||
// so a relative path names two different files: the run finishes, its marker lands somewhere the
|
||
// reconciler never looks, and the run is restarted forever. Refused at boot, where an operator
|
||
// sees it, rather than discovered at the first end of a run.
|
||
if !filepath.IsAbs(r.StateDir) {
|
||
return fmt.Errorf("config: TM_PLATFORM_STATE_DIR must be an absolute path, got %q", r.StateDir)
|
||
}
|
||
// Absolute for the same reason StateDir is: the flag is read by a systemd unit whose working
|
||
// directory is the BOOK's, so a relative path would name a different file per book — and the
|
||
// symptom, a paid run failing at its first provider call, points nowhere near the cause.
|
||
if r.KeysFile != "" && !filepath.IsAbs(r.KeysFile) {
|
||
return fmt.Errorf("config: TM_PLATFORM_ENGINE_KEYS_PATH must be an absolute path, got %q", r.KeysFile)
|
||
}
|
||
var err error
|
||
if r.TasksMax, err = l.number("TM_PLATFORM_RUN_TASKS_MAX", r.TasksMax); err != nil {
|
||
return err
|
||
}
|
||
if r.AllowEngineVersionChange, err = l.boolean("TM_PLATFORM_RESUME_MAY_CHANGE_ENGINE"); err != nil {
|
||
return err
|
||
}
|
||
if r.Workers, err = l.number("TM_PLATFORM_RUN_WORKERS", r.Workers); err != nil {
|
||
return err
|
||
}
|
||
if r.SweepEvery, err = l.duration("TM_PLATFORM_SWEEP_EVERY", r.SweepEvery); err != nil {
|
||
return err
|
||
}
|
||
if r.SweepBudget, err = l.duration("TM_PLATFORM_SWEEP_BUDGET", r.SweepBudget); err != nil {
|
||
return err
|
||
}
|
||
if r.RunBudget, err = l.duration("TM_PLATFORM_RUN_BUDGET", r.RunBudget); err != nil {
|
||
return err
|
||
}
|
||
if r.ResyncEvery, err = l.duration("TM_PLATFORM_RESYNC_EVERY", r.ResyncEvery); err != nil {
|
||
return err
|
||
}
|
||
if r.HoldFactorPercent, err = l.number("TM_PLATFORM_HOLD_FACTOR_PERCENT", r.HoldFactorPercent); err != nil {
|
||
return err
|
||
}
|
||
// Judged HERE and not at the first admission: a cushion below the engine's own expected bill is a
|
||
// discount, every purchase under it latches before it delivers what it quoted, and an operator who
|
||
// mistyped it should learn so at boot rather than from a user's paused run.
|
||
if _, err := pricing.New(r.HoldFactorPercent); err != nil {
|
||
return fmt.Errorf("config: TM_PLATFORM_HOLD_FACTOR_PERCENT: %w", err)
|
||
}
|
||
c.Runner = r
|
||
return nil
|
||
}
|
||
|
||
// loadBackup reads the restore-point maker's settings.
|
||
func (c *Config) loadBackup(l *loader) error {
|
||
b := BackupConfig{
|
||
Dir: l.env("TM_PLATFORM_BACKUP_DIR", ""),
|
||
Every: 6 * time.Hour,
|
||
Keep: 14,
|
||
PgDumpBin: l.env("TM_PLATFORM_PGDUMP_BIN", "pg_dump"),
|
||
PgRestoreBin: l.env("TM_PLATFORM_PGRESTORE_BIN", "pg_restore"),
|
||
}
|
||
// Absolute, and for a sharper reason than the other directories have: this one is written by a
|
||
// unit whose WorkingDirectory is not the daemon's, and a relative path would put the deployment's
|
||
// only copy of itself somewhere nobody looks for it.
|
||
if b.Dir != "" && !filepath.IsAbs(b.Dir) {
|
||
return fmt.Errorf("config: TM_PLATFORM_BACKUP_DIR must be an absolute path, got %q", b.Dir)
|
||
}
|
||
var err error
|
||
if b.Every, err = l.duration("TM_PLATFORM_BACKUP_EVERY", b.Every); err != nil {
|
||
return err
|
||
}
|
||
if b.Keep, err = l.number("TM_PLATFORM_BACKUP_KEEP", b.Keep); err != nil {
|
||
return err
|
||
}
|
||
// ⚠ NO SECOND GUARD ON THESE TWO. Both would-be dangers — a schedule of zero (a full copy of the
|
||
// deployment on every sweep tick) and keeping zero points (the pass deleting its own work) — are
|
||
// already refused by the loader itself: `duration` and `number` both take only positive values.
|
||
// A check here would be dead code whose message no operator could ever read, and the way that was
|
||
// found is worth recording: it was written, and the test asserting the OTHER behaviour failed.
|
||
c.Backup = b
|
||
return nil
|
||
}
|
||
|
||
// BackupEnabled reports whether this instance keeps restore points.
|
||
func (c Config) BackupEnabled() bool { return c.Backup.Dir != "" }
|
||
|
||
// loadExport reads the export door's settings.
|
||
//
|
||
// It runs AFTER loadRunner because its directory defaults out of the runner's state directory: the
|
||
// artifacts are platform state of exactly that kind, and an operator who already told this service
|
||
// where its state lives should not have to say it twice.
|
||
func (c *Config) loadExport(l *loader) error {
|
||
e := ExportConfig{
|
||
Dir: l.env("TM_PLATFORM_EXPORTS_DIR", ""),
|
||
// A day. It bounds disk rather than access, and a day is what a reader who asked for a book
|
||
// needs to fetch it — long enough to survive a closed laptop, short enough that a
|
||
// control-plane disk is not a library.
|
||
TTL: 24 * time.Hour,
|
||
}
|
||
if raw := l.env("TM_PLATFORM_EXPORT_FORMATS", ""); raw != "" {
|
||
for _, f := range strings.Split(raw, ",") {
|
||
if f = strings.TrimSpace(f); f != "" {
|
||
e.Formats = append(e.Formats, f)
|
||
}
|
||
}
|
||
}
|
||
if err := exports.ValidateFormats(e.Formats); err != nil {
|
||
return err
|
||
}
|
||
if e.Dir == "" && c.Runner.StateDir != "" {
|
||
e.Dir = filepath.Join(c.Runner.StateDir, "exports")
|
||
}
|
||
// Absolute for the same reason StateDir is, and with a sharper consequence: the door hands the
|
||
// path to the ENGINE as `--out`, and the engine runs with the BOOK's directory as its working
|
||
// directory — so a relative path would write every artifact inside somebody's book project,
|
||
// which is the one tree this platform must not write into (D39.110).
|
||
if e.Dir != "" && !filepath.IsAbs(e.Dir) {
|
||
return fmt.Errorf("config: TM_PLATFORM_EXPORTS_DIR must be an absolute path, got %q", e.Dir)
|
||
}
|
||
var err error
|
||
if e.TTL, err = l.duration("TM_PLATFORM_EXPORT_TTL", e.TTL); err != nil {
|
||
return err
|
||
}
|
||
if e.TTL <= 0 {
|
||
return fmt.Errorf("config: TM_PLATFORM_EXPORT_TTL must be positive, got %s: a link that expires at once is a door that never opens", e.TTL)
|
||
}
|
||
c.Export = e
|
||
return nil
|
||
}
|
||
|
||
// ExportsEnabled reports whether this instance builds exports at all. It needs a format to build, an
|
||
// engine to build it with and a place to put it — and the CAPABILITY and the mounted routes are the
|
||
// same fact, or a client learns the truth by failing a user's download.
|
||
func (c Config) ExportsEnabled() bool {
|
||
return len(c.Export.Formats) > 0 && c.RunsEnabled() && c.Export.Dir != ""
|
||
}
|
||
|
||
// loadIntake reads the book upload's settings.
|
||
func (c *Config) loadIntake(l *loader) error {
|
||
in := IntakeConfig{
|
||
BooksDir: l.env("TM_PLATFORM_BOOKS_DIR", ""),
|
||
BookTemplate: l.env("TM_PLATFORM_BOOK_TEMPLATE", ""),
|
||
// 64 MiB and ten minutes: the acceptance corpus is a 23 MB book, and ten minutes is what a
|
||
// file that size takes on an uplink an ordinary reader has. Both are refusals rather than
|
||
// promises — the numbers an operator raises when their users hit them.
|
||
MaxUploadBytes: 64 << 20,
|
||
UploadDeadline: 10 * time.Minute,
|
||
}
|
||
if in.BooksDir != "" && !filepath.IsAbs(in.BooksDir) {
|
||
return fmt.Errorf("config: TM_PLATFORM_BOOKS_DIR must be an absolute path, got %q", in.BooksDir)
|
||
}
|
||
// The template is read on every render, from a process whose working directory is the deployment's
|
||
// and not the book's. A relative path would therefore name a different file depending on how the
|
||
// daemon was started, and the symptom — every uploaded book waiting for a configuration — points
|
||
// nowhere near the cause.
|
||
if in.BookTemplate != "" && !filepath.IsAbs(in.BookTemplate) {
|
||
return fmt.Errorf("config: TM_PLATFORM_BOOK_TEMPLATE must be an absolute path, got %q", in.BookTemplate)
|
||
}
|
||
var err error
|
||
if in.MaxUploadBytes, err = l.bytes("TM_PLATFORM_MAX_UPLOAD_BYTES", in.MaxUploadBytes); err != nil {
|
||
return err
|
||
}
|
||
if in.UploadDeadline, err = l.duration("TM_PLATFORM_UPLOAD_DEADLINE", in.UploadDeadline); err != nil {
|
||
return err
|
||
}
|
||
// An upload has to FINISH inside two windows, and the boot is where a deployment that breaks
|
||
// either is cheap to find. The intake sweep treats a book that has been `uploading` longer than
|
||
// its grace as an upload whose request is gone, and deletes the row and the directory under a
|
||
// request still writing into them; past the claim window a retry takes the idempotency claim from
|
||
// an upload that is still running, and the user gets the second book the key exists to prevent.
|
||
//
|
||
// Against the TIGHTER of the two rather than against each: written as two checks, whichever is
|
||
// looser could never fire, and the pin that was supposed to cover it was passing on the other one.
|
||
// And the deadline is only the BODY — what an upload still has to do afterwards is added here,
|
||
// or a 29m59s deadline settled its key a moment after the claim became stealable.
|
||
if window := min(books.UploadGrace, pgstore.ClaimStale); in.UploadDeadline+books.UploadSettle >= window {
|
||
return fmt.Errorf("config: TM_PLATFORM_UPLOAD_DEADLINE (%s) plus what follows an upload (%s) must fit inside the tighter of the intake sweep's grace (%s) and the idempotency claim window (%s)",
|
||
in.UploadDeadline, books.UploadSettle, books.UploadGrace, pgstore.ClaimStale)
|
||
}
|
||
c.Intake = in
|
||
return nil
|
||
}
|
||
|
||
// loader reads the environment and REMEMBERS what it read, so that the boot line can say where each
|
||
// value came from. Every setting of this service goes through it: a print that covers most of the
|
||
// configuration is worse than none, because the variable an operator is hunting is exactly the one
|
||
// nobody thought to record.
|
||
type loader struct{ settings []Setting }
|
||
|
||
func (l *loader) record(key, value, source string) {
|
||
l.settings = append(l.settings, Setting{Key: key, Value: value, Source: source})
|
||
}
|
||
|
||
func (l *loader) env(key, def string) string {
|
||
if v := os.Getenv(key); v != "" {
|
||
l.record(key, v, sourceEnv)
|
||
return v
|
||
}
|
||
if def == "" {
|
||
l.record(key, valueUnset, sourceDefault)
|
||
} else {
|
||
l.record(key, def, sourceDefault)
|
||
}
|
||
return def
|
||
}
|
||
|
||
// Secret is the shared way to read a credential: from KEY, or preferably from the file named by
|
||
// KEY_FILE. Exported so the admin CLI reads the DSN the same way the daemon does — an operator who
|
||
// followed the deploy notes has it in a file, not in the environment.
|
||
func Secret(key string) (string, error) {
|
||
var l loader
|
||
return l.secret(key)
|
||
}
|
||
|
||
// secret reads a value from KEY, or — preferably — from the file named by KEY_FILE. A secret in a
|
||
// file does not show up in /proc/<pid>/environ, is not inherited by child processes, and is exactly
|
||
// what systemd's LoadCredential= hands over (deploy/tmplatformd.service).
|
||
//
|
||
// What is recorded is that it was set and WHERE from, never the value: a DSN carries a password.
|
||
func (l *loader) secret(key string) (string, error) {
|
||
if path := os.Getenv(key + "_FILE"); path != "" {
|
||
b, err := os.ReadFile(path)
|
||
if err != nil {
|
||
// The path, never the content: an unreadable secret file is an operator's problem and
|
||
// the error goes to the log.
|
||
return "", fmt.Errorf("config: %s_FILE: %w", key, err)
|
||
}
|
||
l.record(key, valueSet, sourceFile)
|
||
return strings.TrimSpace(string(b)), nil
|
||
}
|
||
v := os.Getenv(key)
|
||
switch v {
|
||
case "":
|
||
l.record(key, valueUnset, sourceDefault)
|
||
default:
|
||
l.record(key, valueSet, sourceEnv)
|
||
}
|
||
return v, nil
|
||
}
|
||
|
||
// boolean reads a flag. strconv.ParseBool rather than a comparison with "1": an operator who wrote
|
||
// `true` deserves an error or the truth, not a silent no.
|
||
func (l *loader) boolean(key string) (bool, error) {
|
||
raw := os.Getenv(key)
|
||
if raw == "" {
|
||
l.record(key, "false", sourceDefault)
|
||
return false, nil
|
||
}
|
||
v, err := strconv.ParseBool(raw)
|
||
if err != nil {
|
||
return false, fmt.Errorf("config: %s: %q is not a boolean", key, raw)
|
||
}
|
||
l.record(key, strconv.FormatBool(v), sourceEnv)
|
||
return v, nil
|
||
}
|
||
|
||
func (l *loader) number(key string, def int) (int, error) {
|
||
raw := os.Getenv(key)
|
||
if raw == "" {
|
||
l.record(key, strconv.Itoa(def), sourceDefault)
|
||
return def, nil
|
||
}
|
||
v, err := strconv.Atoi(raw)
|
||
if err != nil || v <= 0 {
|
||
return 0, fmt.Errorf("config: %s must be a positive whole number, got %q", key, raw)
|
||
}
|
||
l.record(key, strconv.Itoa(v), sourceEnv)
|
||
return v, nil
|
||
}
|
||
|
||
func (l *loader) bytes(key string, def int64) (int64, error) {
|
||
raw := os.Getenv(key)
|
||
if raw == "" {
|
||
l.record(key, strconv.FormatInt(def, 10), sourceDefault)
|
||
return def, nil
|
||
}
|
||
v, err := strconv.ParseInt(raw, 10, 64)
|
||
if err != nil || v <= 0 {
|
||
return 0, fmt.Errorf("config: %s must be a positive number of bytes, got %q", key, raw)
|
||
}
|
||
l.record(key, strconv.FormatInt(v, 10), sourceEnv)
|
||
return v, nil
|
||
}
|
||
|
||
func (l *loader) duration(key string, def time.Duration) (time.Duration, error) {
|
||
raw := os.Getenv(key)
|
||
if raw == "" {
|
||
l.record(key, def.String(), sourceDefault)
|
||
return def, nil
|
||
}
|
||
d, err := time.ParseDuration(raw)
|
||
if err != nil {
|
||
return 0, fmt.Errorf("config: %s: %w", key, err)
|
||
}
|
||
if d <= 0 {
|
||
return 0, fmt.Errorf("config: %s must be positive", key)
|
||
}
|
||
l.record(key, d.String(), sourceEnv)
|
||
return d, nil
|
||
}
|
||
|
||
// amount reads a sum in dollars into micro-USD. positiveOnly refuses zero as well as negatives,
|
||
// which is the difference between a rate (a zero rate makes every scale infinite) and a grant (a
|
||
// deployment that grants nothing is a legitimate choice).
|
||
//
|
||
// The VALUE is never recorded — money does not reach an INFO log in any form (D39.84, PD-99) — and
|
||
// the source is, which is what an operator needs to see that their figure was picked up.
|
||
func (l *loader) amount(key string, def int64, positiveOnly bool) (int64, error) {
|
||
raw := os.Getenv(key)
|
||
if raw == "" {
|
||
l.record(key, valueAmount, sourceDefault)
|
||
return def, nil
|
||
}
|
||
v, err := money.ParseUSD(raw)
|
||
if err != nil {
|
||
return 0, fmt.Errorf("config: %s: %w", key, err)
|
||
}
|
||
switch {
|
||
case positiveOnly && v <= 0:
|
||
return 0, fmt.Errorf("config: %s must be positive", key)
|
||
case v < 0:
|
||
return 0, fmt.Errorf("config: %s cannot be negative", key)
|
||
}
|
||
l.record(key, valueAmount, sourceEnv)
|
||
return int64(v), nil
|
||
}
|