87 lines
4.7 KiB
Go
87 lines
4.7 KiB
Go
package pgstore
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
)
|
|
|
|
// Observations is one reading of the state an operator cannot otherwise see.
|
|
//
|
|
// Every figure here used to be answerable only by opening psql (zone backlog П-11): how much work is
|
|
// queued, whether money is stuck in a hold nothing closes, whether a live run's projection has been
|
|
// blinded, how many books are stuck in intake. They are read in ONE statement because a set of them
|
|
// taken at different moments describes a state the system was never in.
|
|
type Observations struct {
|
|
QueueDepth int64
|
|
OldestHoldSeconds float64
|
|
QuarantinedAttempts int64
|
|
LiveRuns int64
|
|
BooksUploading int64
|
|
BooksParsing int64
|
|
// StalledRuns is how many attempts the reconciler has failed on repeatedly, and AbandonedSurfaces
|
|
// how many books it has given up materializing. Both were invisible: the only signal was
|
|
// `tm_platform_sweep_unfinished_total`, which says a PASS did not finish and never says on what —
|
|
// so an operator could watch it rise and had nothing to look at and nothing to do (register row
|
|
// PD-169). These two name the state itself; `tmplatformctl runs` and `books --abandoned` name the
|
|
// rows.
|
|
//
|
|
// ⚠ ATTEMPTS and not "live runs", which is what this counted until PD-385. The two phases of the
|
|
// reconciler share one failure counter, and this predicate was the live phase's alone — so an
|
|
// attempt that had ENDED with its money still open raised `oldest_open_hold_seconds` without ever
|
|
// raising this, and the one gauge that names a HANDLE stayed at zero over a frozen hold.
|
|
//
|
|
// ⚠ The floors and the SHAPE are the same as StalledRuns', and deliberately so twice over. The
|
|
// floors, because this gauge and that table are the number and the names of one population, and
|
|
// two predicates that drift are how an operator ends up paging on a row the command cannot show
|
|
// them. The shape — two counts added rather than one count over an OR — because a disjunction
|
|
// costs both halves their index: measured on 200 000 attempts, the OR form made this a sequential
|
|
// scan of every attempt ever, on a query the daemon runs every fifteen seconds forever.
|
|
StalledRuns int64
|
|
AbandonedSurfaces int64
|
|
// ParkedAttempts is how many live attempts stopped materializing because the journal continued
|
|
// under a foreign stream id. Its own series rather than part of QuarantinedAttempts: the remedies
|
|
// are opposite — a quarantine waits for a person, a park clears itself (PD-438).
|
|
ParkedAttempts int64
|
|
}
|
|
|
|
// Observe reads the control plane's own state.
|
|
//
|
|
// The queue's table is asked for through to_regclass rather than assumed: River owns that schema and
|
|
// migrates it separately (STACK_DECISIONS §19), so an instance whose queue migration has not run
|
|
// answers a readiness probe about it and must not fail its telemetry over it as well.
|
|
func (s *Store) Observe(ctx context.Context, stalledAt int) (Observations, error) {
|
|
const q = `
|
|
select
|
|
(select case when to_regclass('river_job') is null then 0 else
|
|
(select count(*) from river_job
|
|
where state in ('available', 'running', 'scheduled', 'retryable', 'pending')) end),
|
|
(select coalesce(extract(epoch from (now() - min(opened_at))), 0)
|
|
from reservations where state = 'open'),
|
|
(select count(*) from run_attempts
|
|
where quarantine_reason is not null and ended_at is null),
|
|
-- Same shape and same plan as the quarantine count above: an index scan over
|
|
-- run_attempts_live_idx with the column as a filter.
|
|
(select count(*) from run_attempts
|
|
where parked_at is not null and ended_at is null),
|
|
(select count(*) from runs where finished_at is null),
|
|
(select count(*) from books where status = 'uploading'),
|
|
(select count(*) from books where status = 'parsing'),
|
|
(select
|
|
(select count(*) from run_attempts a join runs r on r.id = a.run_id
|
|
where a.ended_at is null and r.finished_at is null and a.reconcile_failures >= $1)
|
|
+ (select count(*) from run_attempts a
|
|
where a.ended_at is not null
|
|
and a.reconcile_failures >= 1
|
|
and a.reconcile_failures >= greatest($1, 1)
|
|
and exists (select 1 from reservations res
|
|
where res.engine_run_id = a.run_id || '#' || a.attempt_no
|
|
and res.state = 'open'))),
|
|
(select count(*) from books where read_model_abandoned_at is not null)`
|
|
var o Observations
|
|
err := s.pool.QueryRow(ctx, q, stalledAt).Scan(&o.QueueDepth, &o.OldestHoldSeconds, &o.QuarantinedAttempts,
|
|
&o.ParkedAttempts, &o.LiveRuns, &o.BooksUploading, &o.BooksParsing, &o.StalledRuns, &o.AbandonedSurfaces)
|
|
if err != nil {
|
|
return Observations{}, fmt.Errorf("pgstore: observe: %w", err)
|
|
}
|
|
return o, nil
|
|
}
|