package metrics import ( "math" "net/http" "net/http/httptest" "strings" "testing" "time" ) // scrape renders the exposition the way a scraper would read it. func scrape(t *testing.T, m *Metrics) string { t.Helper() w := httptest.NewRecorder() m.Handler().ServeHTTP(w, httptest.NewRequest("GET", "/metrics", nil)) if w.Code != http.StatusOK { t.Fatalf("scrape: %d %s", w.Code, w.Body) } return w.Body.String() } // Labels carry the route PATTERN and never the path. A raw path is a user's library in an operator's // index (PD-3) and an unbounded number of series besides — the label would be chosen by whoever // sends the request. func TestRequestsAreCountedByRoutePatternAndNeverByPath(t *testing.T) { m := New() mux := http.NewServeMux() mux.Handle("GET /v0/books/{bookId}", m.Middleware()(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNotFound) }))) mux.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "/v0/books/bk_secret", nil)) body := scrape(t, m) if !strings.Contains(body, `tm_platform_http_requests_total{code="404",method="GET",route="GET /v0/books/{bookId}"} 1`) { t.Errorf("the counter does not carry the route pattern:\n%s", body) } if strings.Contains(body, "bk_secret") { t.Errorf("a book id reached the exposition:\n%s", body) } if !strings.Contains(body, "tm_platform_http_request_duration_seconds_bucket") { t.Error("no latency histogram: an average is the one statistic that hides the tail") } } // An unrouted request is where the label would be chosen by an attacker, so there is exactly one // series for all of them. func TestAnUnroutedRequestGetsOneSeriesAndNotOnePerPath(t *testing.T) { m := New() h := m.Middleware()(http.NotFoundHandler()) for _, p := range []string{"/../etc/passwd", "/random-1", "/random-2"} { h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", p, nil)) } body := scrape(t, m) if !strings.Contains(body, `route="(unmatched)"} 3`) { t.Errorf("unmatched requests are not folded into one series:\n%s", body) } if strings.Contains(body, "random-1") { t.Errorf("a caller-chosen path became a label:\n%s", body) } // The METHOD is a token the caller chooses too, and on an unrouted request it is theirs to invent. for _, verb := range []string{"FOOBAR", "WHATEVER"} { r := httptest.NewRequest("GET", "/nope", nil) r.Method = verb h.ServeHTTP(httptest.NewRecorder(), r) } body = scrape(t, m) for _, verb := range []string{"FOOBAR", "WHATEVER"} { if strings.Contains(body, verb) { t.Errorf("a caller-chosen method became a label:\n%s", body) } } if !strings.Contains(body, `method="(other)"`) { t.Errorf("unknown methods are not folded into one series:\n%s", body) } } // The wrapper must stay transparent to http.ResponseController, which is how the upload route // reaches its own read deadline. Asserted here as well as through the real server, because this is // where the property lives. func TestTheWrapperKeepsTheResponseControllerReachable(t *testing.T) { m := New() var sawFlush bool h := m.Middleware()(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { // httptest's recorder supports Flush and nothing else, which is enough: what is being // asserted is that the controller finds the wrapped writer at all. sawFlush = http.NewResponseController(w).Flush() == nil })) h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "/", nil)) if !sawFlush { t.Fatal("the wrapper hides the ResponseWriter from http.ResponseController: Unwrap is missing") } } // The numbers П-11 was opened for: each one used to be answerable only in psql. func TestTheRunnersStateIsExposedWithItsUnits(t *testing.T) { m := New() m.ObserveRunner(Runner{QueueDepth: 3, OldestHoldSeconds: 61.5, QuarantinedAttempts: 1, LiveRuns: 2, BooksUploading: 4, BooksParsing: 5}) m.ObserveTailerLag(4096) m.ObserveSweep("runs", 1500*time.Millisecond, true) body := scrape(t, m) for _, want := range []string{ "tm_platform_queue_depth 3", "tm_platform_oldest_open_hold_seconds 61.5", "tm_platform_quarantined_attempts 1", "tm_platform_live_runs 2", "tm_platform_tailer_lag_bytes 4096", `tm_platform_books_in_intake{status="uploading"} 4`, `tm_platform_books_in_intake{status="parsing"} 5`, `tm_platform_sweep_unfinished_total{sweep="runs"} 1`, `tm_platform_sweep_duration_seconds_sum{sweep="runs"} 1.5`, } { if !strings.Contains(body, want) { t.Errorf("the exposition is missing %q", want) } } // Base units and no unit inside a label, per the naming practices this axis is measured against. // Judged over OUR namespace only: the Go collector's own series are its to name (and one of them // is `go_memstats_mspan_inuse_bytes`, which is how this assertion first failed). for _, line := range strings.Split(body, "\n") { if !strings.HasPrefix(line, namespace+"_") { continue } for _, wrong := range []string{"_ms ", "_ms{", "_millis", `unit="`} { if strings.Contains(line, wrong) { t.Errorf("%q is not a base-unit name", line) } } } } // A process that measures nothing must still serve: every instrument is nil-safe, because the // alternative is a telemetry failure taking down the thing it measures. func TestAServiceWithoutTelemetryStillWorks(t *testing.T) { var m *Metrics m.ObserveRunner(Runner{QueueDepth: 1}) m.ObserveTailerLag(1) m.ObserveSweep("runs", time.Second, false) h := m.Middleware()(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusTeapot) })) w := httptest.NewRecorder() h.ServeHTTP(w, httptest.NewRequest("GET", "/", nil)) if w.Code != http.StatusTeapot { t.Fatalf("status %d through a nil registry", w.Code) } } // The gauge's whole argument for being alertable is that a deployment with NO complete restore point // reads +Inf rather than 0 or NaN: `> threshold` is FALSE for both of those, so an instance that has // never made a backup would sit under every alert ever written for this series. The doc comment names // that as the wrong answer; nothing checked it. func TestTheBackupAgeGaugeIsInfiniteWhenThereIsNoPoint(t *testing.T) { m := New() read := func() float64 { fams, err := m.registry.Gather() if err != nil { t.Fatal(err) } for _, f := range fams { if f.GetName() == "tm_platform_backup_age_seconds" { return f.GetMetric()[0].GetGauge().GetValue() } } t.Fatal("the gauge is not registered") return 0 } // Before anything has observed: a fresh process must already say "no copy", not "zero seconds old". if v := read(); !math.IsInf(v, 1) { t.Errorf("a fresh registry reads %v; an alert on `> threshold` would never fire", v) } m.ObserveBackupAge(0, false) if v := read(); !math.IsInf(v, 1) { t.Errorf("no restore point reads %v, want +Inf", v) } m.ObserveBackupAge(90*time.Minute, true) if v := read(); v != 5400 { t.Errorf("age = %v seconds, want 5400 (the base unit is seconds, not minutes)", v) } }