From 799204fe6fa639924abb0f0cf0534853d6378a7c Mon Sep 17 00:00:00 2001
From: jiceatscion <139873336+jiceatscion@users.noreply.github.com>
Date: Tue, 16 Jul 2024 13:14:23 +0200
Subject: [PATCH] monitoring: give a timeout to our prometheus request handler
 (#4575)

Set a 1 m timeout to all prometheus incoming requests.
Fixes #1971
---
 private/env/BUILD.bazel |  1 +
 private/env/env.go      | 14 +++++++++++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/private/env/BUILD.bazel b/private/env/BUILD.bazel
index 796ad79db1..57482bffd9 100644
--- a/private/env/BUILD.bazel
+++ b/private/env/BUILD.bazel
@@ -19,6 +19,7 @@ go_library(
         "//pkg/scrypto:go_default_library",
         "//private/config:go_default_library",
         "@com_github_opentracing_opentracing_go//:go_default_library",
+        "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prometheus_client_golang//prometheus/promhttp:go_default_library",
         "@com_github_uber_jaeger_client_go//:go_default_library",
         "@com_github_uber_jaeger_client_go//config:go_default_library",
diff --git a/private/env/env.go b/private/env/env.go
index c8b0843216..fdba5dc47f 100644
--- a/private/env/env.go
+++ b/private/env/env.go
@@ -34,6 +34,7 @@ import (
 	"time"
 
 	opentracing "github.com/opentracing/opentracing-go"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	jaeger "github.com/uber/jaeger-client-go"
 	jaegercfg "github.com/uber/jaeger-client-go/config"
@@ -61,6 +62,10 @@ const (
 	// ShutdownGraceInterval is the time applications wait after issuing a
 	// clean shutdown signal, before forcerfully tearing down the application.
 	ShutdownGraceInterval = 5 * time.Second
+
+	// HandlerTimeout is the time after which the http handler gives up on a request and
+	// returns an error instead.
+	HandlerTimeout = time.Minute
 )
 
 var sighupC chan os.Signal
@@ -185,7 +190,14 @@ func (cfg *Metrics) ServePrometheus(ctx context.Context) error {
 	if cfg.Prometheus == "" {
 		return nil
 	}
-	http.Handle("/metrics", promhttp.Handler())
+	handler := promhttp.InstrumentMetricHandler(
+		prometheus.DefaultRegisterer,
+		promhttp.HandlerFor(
+			prometheus.DefaultGatherer,
+			promhttp.HandlerOpts{Timeout: HandlerTimeout},
+		),
+	)
+	http.Handle("/metrics", handler)
 	log.Info("Exporting prometheus metrics", "addr", cfg.Prometheus)
 
 	server := &http.Server{Addr: cfg.Prometheus}