Track the number of slow requests

sync3/handler/handler.go

@@ -63,6 +63,7 @@ type SyncLiveHandler struct {
 	numConns     prometheus.Gauge
 	setupHistVec *prometheus.HistogramVec
 	histVec      *prometheus.HistogramVec
+	slowReqs     prometheus.Counter
 }
 
 func NewSync3Handler(
@@ -137,6 +138,9 @@ func (h *SyncLiveHandler) Teardown() {
 	if h.histVec != nil {
 		prometheus.Unregister(h.histVec)
 	}
+	if h.slowReqs != nil {
+		prometheus.Unregister(h.slowReqs)
+	}
 }
 
 func (h *SyncLiveHandler) updateMetrics() {
@@ -167,9 +171,16 @@ func (h *SyncLiveHandler) addPrometheusMetrics() {
 		Help:      "Time taken in seconds for the sliding sync response to be calculated, excludes long polling",
 		Buckets:   []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
 	}, []string{"initial"})
+	h.slowReqs = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "sliding_sync",
+		Subsystem: "api",
+		Name:      "slow_requests",
+		Help:      "Counter of slow (>=50s) requests, initial or otherwise.",
+	})
 	prometheus.MustRegister(h.numConns)
 	prometheus.MustRegister(h.setupHistVec)
 	prometheus.MustRegister(h.histVec)
+	prometheus.MustRegister(h.slowReqs)
 }
 
 func (h *SyncLiveHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
@@ -197,6 +208,11 @@ func (h *SyncLiveHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 // Entry point for sync v3
 func (h *SyncLiveHandler) serve(w http.ResponseWriter, req *http.Request) error {
 	start := time.Now()
+	defer func() {
+		if time.Since(start) > 50*time.Second {
+			h.slowReqs.Add(1.0)
+		}
+	}()
 	var requestBody sync3.Request
 	if req.ContentLength != 0 {
 		defer req.Body.Close()