textmachine/platform/internal/metrics/metrics_test.go

package metrics

import (
	"math"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"
)

// scrape renders the exposition the way a scraper would read it.
func scrape(t *testing.T, m *Metrics) string {
	t.Helper()
	w := httptest.NewRecorder()
	m.Handler().ServeHTTP(w, httptest.NewRequest("GET", "/metrics", nil))
	if w.Code != http.StatusOK {
		t.Fatalf("scrape: %d %s", w.Code, w.Body)
	}
	return w.Body.String()
}

// Labels carry the route PATTERN and never the path. A raw path is a user's library in an operator's
// index (PD-3) and an unbounded number of series besides — the label would be chosen by whoever
// sends the request.
func TestRequestsAreCountedByRoutePatternAndNeverByPath(t *testing.T) {
	m := New()
	mux := http.NewServeMux()
	mux.Handle("GET /v0/books/{bookId}", m.Middleware()(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		w.WriteHeader(http.StatusNotFound)
	})))
	mux.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "/v0/books/bk_secret", nil))

	body := scrape(t, m)
	if !strings.Contains(body, `tm_platform_http_requests_total{code="404",method="GET",route="GET /v0/books/{bookId}"} 1`) {
		t.Errorf("the counter does not carry the route pattern:\n%s", body)
	}
	if strings.Contains(body, "bk_secret") {
		t.Errorf("a book id reached the exposition:\n%s", body)
	}
	if !strings.Contains(body, "tm_platform_http_request_duration_seconds_bucket") {
		t.Error("no latency histogram: an average is the one statistic that hides the tail")
	}
}

// An unrouted request is where the label would be chosen by an attacker, so there is exactly one
// series for all of them.
func TestAnUnroutedRequestGetsOneSeriesAndNotOnePerPath(t *testing.T) {
	m := New()
	h := m.Middleware()(http.NotFoundHandler())
	for _, p := range []string{"/../etc/passwd", "/random-1", "/random-2"} {
		h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", p, nil))
	}
	body := scrape(t, m)
	if !strings.Contains(body, `route="(unmatched)"} 3`) {
		t.Errorf("unmatched requests are not folded into one series:\n%s", body)
	}
	if strings.Contains(body, "random-1") {
		t.Errorf("a caller-chosen path became a label:\n%s", body)
	}
	// The METHOD is a token the caller chooses too, and on an unrouted request it is theirs to invent.
	for _, verb := range []string{"FOOBAR", "WHATEVER"} {
		r := httptest.NewRequest("GET", "/nope", nil)
		r.Method = verb
		h.ServeHTTP(httptest.NewRecorder(), r)
	}
	body = scrape(t, m)
	for _, verb := range []string{"FOOBAR", "WHATEVER"} {
		if strings.Contains(body, verb) {
			t.Errorf("a caller-chosen method became a label:\n%s", body)
		}
	}
	if !strings.Contains(body, `method="(other)"`) {
		t.Errorf("unknown methods are not folded into one series:\n%s", body)
	}
}

// The wrapper must stay transparent to http.ResponseController, which is how the upload route
// reaches its own read deadline. Asserted here as well as through the real server, because this is
// where the property lives.
func TestTheWrapperKeepsTheResponseControllerReachable(t *testing.T) {
	m := New()
	var sawFlush bool
	h := m.Middleware()(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		// httptest's recorder supports Flush and nothing else, which is enough: what is being
		// asserted is that the controller finds the wrapped writer at all.
		sawFlush = http.NewResponseController(w).Flush() == nil
	}))
	h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "/", nil))
	if !sawFlush {
		t.Fatal("the wrapper hides the ResponseWriter from http.ResponseController: Unwrap is missing")
	}
}

// The numbers П-11 was opened for: each one used to be answerable only in psql.
func TestTheRunnersStateIsExposedWithItsUnits(t *testing.T) {
	m := New()
	m.ObserveRunner(Runner{QueueDepth: 3, OldestHoldSeconds: 61.5, QuarantinedAttempts: 1,
		LiveRuns: 2, BooksUploading: 4, BooksParsing: 5})
	m.ObserveTailerLag(4096)
	m.ObserveSweep("runs", 1500*time.Millisecond, true)
	body := scrape(t, m)
	for _, want := range []string{
		"tm_platform_queue_depth 3",
		"tm_platform_oldest_open_hold_seconds 61.5",
		"tm_platform_quarantined_attempts 1",
		"tm_platform_live_runs 2",
		"tm_platform_tailer_lag_bytes 4096",
		`tm_platform_books_in_intake{status="uploading"} 4`,
		`tm_platform_books_in_intake{status="parsing"} 5`,
		`tm_platform_sweep_unfinished_total{sweep="runs"} 1`,
		`tm_platform_sweep_duration_seconds_sum{sweep="runs"} 1.5`,
	} {
		if !strings.Contains(body, want) {
			t.Errorf("the exposition is missing %q", want)
		}
	}
	// Base units and no unit inside a label, per the naming practices this axis is measured against.
	// Judged over OUR namespace only: the Go collector's own series are its to name (and one of them
	// is `go_memstats_mspan_inuse_bytes`, which is how this assertion first failed).
	for _, line := range strings.Split(body, "\n") {
		if !strings.HasPrefix(line, namespace+"_") {
			continue
		}
		for _, wrong := range []string{"_ms ", "_ms{", "_millis", `unit="`} {
			if strings.Contains(line, wrong) {
				t.Errorf("%q is not a base-unit name", line)
			}
		}
	}
}

// A process that measures nothing must still serve: every instrument is nil-safe, because the
// alternative is a telemetry failure taking down the thing it measures.
func TestAServiceWithoutTelemetryStillWorks(t *testing.T) {
	var m *Metrics
	m.ObserveRunner(Runner{QueueDepth: 1})
	m.ObserveTailerLag(1)
	m.ObserveSweep("runs", time.Second, false)
	h := m.Middleware()(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		w.WriteHeader(http.StatusTeapot)
	}))
	w := httptest.NewRecorder()
	h.ServeHTTP(w, httptest.NewRequest("GET", "/", nil))
	if w.Code != http.StatusTeapot {
		t.Fatalf("status %d through a nil registry", w.Code)
	}
}

// The gauge's whole argument for being alertable is that a deployment with NO complete restore point
// reads +Inf rather than 0 or NaN: `> threshold` is FALSE for both of those, so an instance that has
// never made a backup would sit under every alert ever written for this series. The doc comment names
// that as the wrong answer; nothing checked it.
func TestTheBackupAgeGaugeIsInfiniteWhenThereIsNoPoint(t *testing.T) {
	m := New()
	read := func() float64 {
		fams, err := m.registry.Gather()
		if err != nil {
			t.Fatal(err)
		}
		for _, f := range fams {
			if f.GetName() == "tm_platform_backup_age_seconds" {
				return f.GetMetric()[0].GetGauge().GetValue()
			}
		}
		t.Fatal("the gauge is not registered")
		return 0
	}
	// Before anything has observed: a fresh process must already say "no copy", not "zero seconds old".
	if v := read(); !math.IsInf(v, 1) {
		t.Errorf("a fresh registry reads %v; an alert on `> threshold` would never fire", v)
	}
	m.ObserveBackupAge(0, false)
	if v := read(); !math.IsInf(v, 1) {
		t.Errorf("no restore point reads %v, want +Inf", v)
	}
	m.ObserveBackupAge(90*time.Minute, true)
	if v := read(); v != 5400 {
		t.Errorf("age = %v seconds, want 5400 (the base unit is seconds, not minutes)", v)
	}
}