229 lines
9.1 KiB
Go
229 lines
9.1 KiB
Go
package metrics
|
|
|
|
import (
|
|
"math"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// scrape renders the exposition the way a scraper would read it.
|
|
func scrape(t *testing.T, m *Metrics) string {
|
|
t.Helper()
|
|
w := httptest.NewRecorder()
|
|
m.Handler().ServeHTTP(w, httptest.NewRequest("GET", "/metrics", nil))
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("scrape: %d %s", w.Code, w.Body)
|
|
}
|
|
return w.Body.String()
|
|
}
|
|
|
|
// Labels carry the route PATTERN and never the path. A raw path is a user's library in an operator's
|
|
// index (PD-3) and an unbounded number of series besides — the label would be chosen by whoever
|
|
// sends the request.
|
|
func TestRequestsAreCountedByRoutePatternAndNeverByPath(t *testing.T) {
|
|
m := New()
|
|
mux := http.NewServeMux()
|
|
mux.Handle("GET /v0/books/{bookId}", m.Middleware()(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusNotFound)
|
|
})))
|
|
mux.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "/v0/books/bk_secret", nil))
|
|
|
|
body := scrape(t, m)
|
|
if !strings.Contains(body, `tm_platform_http_requests_total{code="404",method="GET",route="GET /v0/books/{bookId}"} 1`) {
|
|
t.Errorf("the counter does not carry the route pattern:\n%s", body)
|
|
}
|
|
if strings.Contains(body, "bk_secret") {
|
|
t.Errorf("a book id reached the exposition:\n%s", body)
|
|
}
|
|
if !strings.Contains(body, "tm_platform_http_request_duration_seconds_bucket") {
|
|
t.Error("no latency histogram: an average is the one statistic that hides the tail")
|
|
}
|
|
}
|
|
|
|
// An unrouted request is where the label would be chosen by an attacker, so there is exactly one
|
|
// series for all of them.
|
|
func TestAnUnroutedRequestGetsOneSeriesAndNotOnePerPath(t *testing.T) {
|
|
m := New()
|
|
h := m.Middleware()(http.NotFoundHandler())
|
|
for _, p := range []string{"/../etc/passwd", "/random-1", "/random-2"} {
|
|
h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", p, nil))
|
|
}
|
|
body := scrape(t, m)
|
|
if !strings.Contains(body, `route="(unmatched)"} 3`) {
|
|
t.Errorf("unmatched requests are not folded into one series:\n%s", body)
|
|
}
|
|
if strings.Contains(body, "random-1") {
|
|
t.Errorf("a caller-chosen path became a label:\n%s", body)
|
|
}
|
|
// The METHOD is a token the caller chooses too, and on an unrouted request it is theirs to invent.
|
|
for _, verb := range []string{"FOOBAR", "WHATEVER"} {
|
|
r := httptest.NewRequest("GET", "/nope", nil)
|
|
r.Method = verb
|
|
h.ServeHTTP(httptest.NewRecorder(), r)
|
|
}
|
|
body = scrape(t, m)
|
|
for _, verb := range []string{"FOOBAR", "WHATEVER"} {
|
|
if strings.Contains(body, verb) {
|
|
t.Errorf("a caller-chosen method became a label:\n%s", body)
|
|
}
|
|
}
|
|
if !strings.Contains(body, `method="(other)"`) {
|
|
t.Errorf("unknown methods are not folded into one series:\n%s", body)
|
|
}
|
|
}
|
|
|
|
// The wrapper must stay transparent to http.ResponseController, which is how the upload route
|
|
// reaches its own read deadline. Asserted here as well as through the real server, because this is
|
|
// where the property lives.
|
|
func TestTheWrapperKeepsTheResponseControllerReachable(t *testing.T) {
|
|
m := New()
|
|
var sawFlush bool
|
|
h := m.Middleware()(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
// httptest's recorder supports Flush and nothing else, which is enough: what is being
|
|
// asserted is that the controller finds the wrapped writer at all.
|
|
sawFlush = http.NewResponseController(w).Flush() == nil
|
|
}))
|
|
h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "/", nil))
|
|
if !sawFlush {
|
|
t.Fatal("the wrapper hides the ResponseWriter from http.ResponseController: Unwrap is missing")
|
|
}
|
|
}
|
|
|
|
// The numbers П-11 was opened for: each one used to be answerable only in psql.
|
|
func TestTheRunnersStateIsExposedWithItsUnits(t *testing.T) {
|
|
m := New()
|
|
// The two «the projection stopped» states carry DIFFERENT numbers on purpose: a park and a
|
|
// quarantine freeze a run's figures the same way and take opposite actions, so one series
|
|
// standing in for both would page an operator into the wrong remedy (PD-438).
|
|
m.ObserveRunner(Runner{QueueDepth: 3, OldestHoldSeconds: 61.5, QuarantinedAttempts: 1,
|
|
ParkedAttempts: 7, LiveRuns: 2, BooksUploading: 4, BooksParsing: 5})
|
|
m.ObserveTailerLag(4096)
|
|
m.ObserveSweep("runs", 1500*time.Millisecond, true)
|
|
body := scrape(t, m)
|
|
for _, want := range []string{
|
|
"tm_platform_queue_depth 3",
|
|
"tm_platform_oldest_open_hold_seconds 61.5",
|
|
"tm_platform_quarantined_attempts 1",
|
|
"tm_platform_parked_attempts 7",
|
|
"tm_platform_live_runs 2",
|
|
"tm_platform_tailer_lag_bytes 4096",
|
|
`tm_platform_books_in_intake{status="uploading"} 4`,
|
|
`tm_platform_books_in_intake{status="parsing"} 5`,
|
|
`tm_platform_sweep_unfinished_total{sweep="runs"} 1`,
|
|
`tm_platform_sweep_duration_seconds_sum{sweep="runs"} 1.5`,
|
|
} {
|
|
if !strings.Contains(body, want) {
|
|
t.Errorf("the exposition is missing %q", want)
|
|
}
|
|
}
|
|
// Base units and no unit inside a label, per the naming practices this axis is measured against.
|
|
// Judged over OUR namespace only: the Go collector's own series are its to name (and one of them
|
|
// is `go_memstats_mspan_inuse_bytes`, which is how this assertion first failed).
|
|
for _, line := range strings.Split(body, "\n") {
|
|
if !strings.HasPrefix(line, namespace+"_") {
|
|
continue
|
|
}
|
|
for _, wrong := range []string{"_ms ", "_ms{", "_millis", `unit="`} {
|
|
if strings.Contains(line, wrong) {
|
|
t.Errorf("%q is not a base-unit name", line)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// A process that measures nothing must still serve: every instrument is nil-safe, because the
|
|
// alternative is a telemetry failure taking down the thing it measures.
|
|
func TestAServiceWithoutTelemetryStillWorks(t *testing.T) {
|
|
var m *Metrics
|
|
m.ObserveRunner(Runner{QueueDepth: 1})
|
|
m.ObserveTailerLag(1)
|
|
m.ObserveSweep("runs", time.Second, false)
|
|
h := m.Middleware()(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusTeapot)
|
|
}))
|
|
w := httptest.NewRecorder()
|
|
h.ServeHTTP(w, httptest.NewRequest("GET", "/", nil))
|
|
if w.Code != http.StatusTeapot {
|
|
t.Fatalf("status %d through a nil registry", w.Code)
|
|
}
|
|
}
|
|
|
|
// The gauge's whole argument for being alertable is that a deployment with NO complete restore point
|
|
// reads +Inf rather than 0 or NaN: `> threshold` is FALSE for both of those, so an instance that has
|
|
// never made a backup would sit under every alert ever written for this series. The doc comment names
|
|
// that as the wrong answer; nothing checked it.
|
|
func TestTheBackupAgeGaugeIsInfiniteWhenThereIsNoPoint(t *testing.T) {
|
|
m := New()
|
|
read := func() float64 {
|
|
fams, err := m.registry.Gather()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, f := range fams {
|
|
if f.GetName() == "tm_platform_backup_age_seconds" {
|
|
return f.GetMetric()[0].GetGauge().GetValue()
|
|
}
|
|
}
|
|
t.Fatal("the gauge is not registered")
|
|
return 0
|
|
}
|
|
// Before anything has observed: a fresh process must already say "no copy", not "zero seconds old".
|
|
if v := read(); !math.IsInf(v, 1) {
|
|
t.Errorf("a fresh registry reads %v; an alert on `> threshold` would never fire", v)
|
|
}
|
|
m.ObserveBackupAge(0, false)
|
|
if v := read(); !math.IsInf(v, 1) {
|
|
t.Errorf("no restore point reads %v, want +Inf", v)
|
|
}
|
|
m.ObserveBackupAge(90*time.Minute, true)
|
|
if v := read(); v != 5400 {
|
|
t.Errorf("age = %v seconds, want 5400 (the base unit is seconds, not minutes)", v)
|
|
}
|
|
}
|
|
|
|
// The cap's five series carry the five DIFFERENT numbers they are given.
|
|
//
|
|
// Five distinct values on purpose: with a fixture where any two coincide, a pair of gauges wired to
|
|
// each other's readings is invisible, and «how saturated is the host» would be answered by «how deep
|
|
// is the line» without anybody noticing. The counters are published as INCREMENTS from cumulative
|
|
// readings, so a second observation is made and the counters have to have moved by the difference
|
|
// while the gauges simply carry the latest.
|
|
func TestTheCutCapacityIsExposedWithEveryFigureItWasGiven(t *testing.T) {
|
|
m := New()
|
|
m.ObserveCuts(Cuts{Limit: 6, InFlight: 4, Waiting: 3, Waited: 9, GaveUp: 2})
|
|
body := scrape(t, m)
|
|
for _, want := range []string{
|
|
"tm_platform_cut_slots 6",
|
|
"tm_platform_cuts_in_flight 4",
|
|
"tm_platform_cuts_waiting 3",
|
|
"tm_platform_cut_waits_total 9",
|
|
"tm_platform_cut_slot_timeouts_total 2",
|
|
} {
|
|
if !strings.Contains(body, want) {
|
|
t.Errorf("the exposition is missing %q", want)
|
|
}
|
|
}
|
|
// A second reading: the gauges take the new value, the counters advance by the difference rather
|
|
// than being set to it — a counter set to a cumulative reading would double-count on every scrape.
|
|
m.ObserveCuts(Cuts{Limit: 6, InFlight: 1, Waiting: 0, Waited: 11, GaveUp: 2})
|
|
body = scrape(t, m)
|
|
for _, want := range []string{
|
|
"tm_platform_cuts_in_flight 1",
|
|
"tm_platform_cuts_waiting 0",
|
|
"tm_platform_cut_waits_total 11",
|
|
"tm_platform_cut_slot_timeouts_total 2",
|
|
} {
|
|
if !strings.Contains(body, want) {
|
|
t.Errorf("after a second reading the exposition is missing %q", want)
|
|
}
|
|
}
|
|
// And a reading that went BACKWARDS — which cannot happen in this process and would break every
|
|
// rate() over the series if it did — leaves the counter where it stood.
|
|
m.ObserveCuts(Cuts{Limit: 6, InFlight: 0, Waiting: 0, Waited: 1, GaveUp: 0})
|
|
if body = scrape(t, m); !strings.Contains(body, "tm_platform_cut_waits_total 11") {
|
|
t.Error("a reading below what was already counted moved the counter backwards")
|
|
}
|
|
}
|