package pgstore import ( "context" "fmt" ) // Observations is one reading of the state an operator cannot otherwise see. // // Every figure here used to be answerable only by opening psql (zone backlog П-11): how much work is // queued, whether money is stuck in a hold nothing closes, whether a live run's projection has been // blinded, how many books are stuck in intake. They are read in ONE statement because a set of them // taken at different moments describes a state the system was never in. type Observations struct { QueueDepth int64 OldestHoldSeconds float64 QuarantinedAttempts int64 LiveRuns int64 BooksUploading int64 BooksParsing int64 // StalledRuns is how many attempts the reconciler has failed on repeatedly, and AbandonedSurfaces // how many books it has given up materializing. Both were invisible: the only signal was // `tm_platform_sweep_unfinished_total`, which says a PASS did not finish and never says on what — // so an operator could watch it rise and had nothing to look at and nothing to do (register row // PD-169). These two name the state itself; `tmplatformctl runs` and `books --abandoned` name the // rows. // // ⚠ ATTEMPTS and not "live runs", which is what this counted until PD-385. The two phases of the // reconciler share one failure counter, and this predicate was the live phase's alone — so an // attempt that had ENDED with its money still open raised `oldest_open_hold_seconds` without ever // raising this, and the one gauge that names a HANDLE stayed at zero over a frozen hold. // // ⚠ The floors and the SHAPE are the same as StalledRuns', and deliberately so twice over. The // floors, because this gauge and that table are the number and the names of one population, and // two predicates that drift are how an operator ends up paging on a row the command cannot show // them. The shape — two counts added rather than one count over an OR — because a disjunction // costs both halves their index: measured on 200 000 attempts, the OR form made this a sequential // scan of every attempt ever, on a query the daemon runs every fifteen seconds forever. StalledRuns int64 AbandonedSurfaces int64 } // Observe reads the control plane's own state. // // The queue's table is asked for through to_regclass rather than assumed: River owns that schema and // migrates it separately (STACK_DECISIONS §19), so an instance whose queue migration has not run // answers a readiness probe about it and must not fail its telemetry over it as well. func (s *Store) Observe(ctx context.Context, stalledAt int) (Observations, error) { const q = ` select (select case when to_regclass('river_job') is null then 0 else (select count(*) from river_job where state in ('available', 'running', 'scheduled', 'retryable', 'pending')) end), (select coalesce(extract(epoch from (now() - min(opened_at))), 0) from reservations where state = 'open'), (select count(*) from run_attempts where quarantine_reason is not null and ended_at is null), (select count(*) from runs where finished_at is null), (select count(*) from books where status = 'uploading'), (select count(*) from books where status = 'parsing'), (select (select count(*) from run_attempts a join runs r on r.id = a.run_id where a.ended_at is null and r.finished_at is null and a.reconcile_failures >= $1) + (select count(*) from run_attempts a where a.ended_at is not null and a.reconcile_failures >= 1 and a.reconcile_failures >= greatest($1, 1) and exists (select 1 from reservations res where res.engine_run_id = a.run_id || '#' || a.attempt_no and res.state = 'open'))), (select count(*) from books where read_model_abandoned_at is not null)` var o Observations err := s.pool.QueryRow(ctx, q, stalledAt).Scan(&o.QueueDepth, &o.OldestHoldSeconds, &o.QuarantinedAttempts, &o.LiveRuns, &o.BooksUploading, &o.BooksParsing, &o.StalledRuns, &o.AbandonedSurfaces) if err != nil { return Observations{}, fmt.Errorf("pgstore: observe: %w", err) } return o, nil }