From d66d50eb63a7edc0322b8e3f898f1c0175db5a21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Rold=C3=A1n=20Betancort?= Date: Fri, 15 Mar 2024 13:09:18 +0000 Subject: [PATCH] make registration of gRPC prom metrics not fail if already registered this allows custom builds of SpiceDB to register their own grpc prom metrics with custom labels --- pkg/cmd/devtools.go | 3 ++- pkg/cmd/server/defaults.go | 36 +++++++++++++++++++++++++----------- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/pkg/cmd/devtools.go b/pkg/cmd/devtools.go index 1130a54e8d..898da1c54e 100644 --- a/pkg/cmd/devtools.go +++ b/pkg/cmd/devtools.go @@ -56,12 +56,13 @@ func NewDevtoolsCommand(programName string) *cobra.Command { } func runfunc(cmd *cobra.Command, _ []string) error { + grpcUnaryInterceptor, _ := server.GRPCMetrics() grpcBuilder := grpcServiceBuilder() grpcServer, err := grpcBuilder.ServerFromFlags(cmd, grpc.ChainUnaryInterceptor( grpclog.UnaryServerInterceptor(server.InterceptorLogger(log.Logger)), otelgrpc.UnaryServerInterceptor(), // nolint: staticcheck - server.GRPCMetricsUnaryInterceptor, + grpcUnaryInterceptor, )) if err != nil { log.Ctx(cmd.Context()).Fatal().Err(err).Msg("failed to create gRPC server") diff --git a/pkg/cmd/server/defaults.go b/pkg/cmd/server/defaults.go index 10e16f8aa0..ac2428594b 100644 --- a/pkg/cmd/server/defaults.go +++ b/pkg/cmd/server/defaults.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "net/http/pprof" + "sync" "time" "github.com/fatih/color" @@ -164,14 +165,21 @@ type MiddlewareOption struct { enableResponseLog bool } -// GRPCMetricsUnaryInterceptor creates the default prometheus metrics interceptor for unary gRPCs -var GRPCMetricsUnaryInterceptor grpc.UnaryServerInterceptor +// gRPCMetricsUnaryInterceptor creates the default prometheus metrics interceptor for unary gRPCs +var gRPCMetricsUnaryInterceptor grpc.UnaryServerInterceptor -// GRPCMetricsStreamingInterceptor creates the default prometheus metrics interceptor for streaming gRPCs -var GRPCMetricsStreamingInterceptor grpc.StreamServerInterceptor +// gRPCMetricsStreamingInterceptor creates the default prometheus metrics interceptor for streaming gRPCs +var gRPCMetricsStreamingInterceptor grpc.StreamServerInterceptor -func init() { - GRPCMetricsUnaryInterceptor, GRPCMetricsStreamingInterceptor = createServerMetrics() +var serverMetricsOnce sync.Once + +// GRPCMetrics returns the interceptors used for the default gRPC metrics from grpc-ecosystem/go-grpc-middleware +func GRPCMetrics() (grpc.UnaryServerInterceptor, grpc.StreamServerInterceptor) { + serverMetricsOnce.Do(func() { + gRPCMetricsUnaryInterceptor, gRPCMetricsStreamingInterceptor = createServerMetrics() + }) + + return gRPCMetricsUnaryInterceptor, gRPCMetricsStreamingInterceptor } const healthCheckRoute = "/grpc.health.v1.Health/Check" @@ -190,6 +198,7 @@ func doesNotMatchRoute(route string) func(_ context.Context, c interceptors.Call // DefaultUnaryMiddleware generates the default middleware chain used for the public SpiceDB Unary gRPC methods func DefaultUnaryMiddleware(opts MiddlewareOption) (*MiddlewareChain[grpc.UnaryServerInterceptor], error) { + grpcMetricsUnaryInterceptor, _ := GRPCMetrics() chain, err := NewMiddlewareChain([]ReferenceableMiddleware[grpc.UnaryServerInterceptor]{ NewUnaryMiddleware(). WithName(DefaultMiddlewareRequestID). @@ -224,7 +233,7 @@ func DefaultUnaryMiddleware(opts MiddlewareOption) (*MiddlewareChain[grpc.UnaryS NewUnaryMiddleware(). WithName(DefaultMiddlewareGRPCProm). - WithInterceptor(GRPCMetricsUnaryInterceptor). + WithInterceptor(grpcMetricsUnaryInterceptor). Done(), NewUnaryMiddleware(). @@ -267,6 +276,7 @@ func DefaultUnaryMiddleware(opts MiddlewareOption) (*MiddlewareChain[grpc.UnaryS // DefaultStreamingMiddleware generates the default middleware chain used for the public SpiceDB Streaming gRPC methods func DefaultStreamingMiddleware(opts MiddlewareOption) (*MiddlewareChain[grpc.StreamServerInterceptor], error) { + _, grpcMetricsStreamingInterceptor := GRPCMetrics() chain, err := NewMiddlewareChain([]ReferenceableMiddleware[grpc.StreamServerInterceptor]{ NewStreamMiddleware(). WithName(DefaultMiddlewareRequestID). @@ -301,7 +311,7 @@ func DefaultStreamingMiddleware(opts MiddlewareOption) (*MiddlewareChain[grpc.St NewStreamMiddleware(). WithName(DefaultMiddlewareGRPCProm). - WithInterceptor(GRPCMetricsStreamingInterceptor). + WithInterceptor(grpcMetricsStreamingInterceptor). Done(), NewStreamMiddleware(). @@ -357,12 +367,13 @@ func determineEventsToLog(opts MiddlewareOption) grpclog.Option { // DefaultDispatchMiddleware generates the default middleware chain used for the internal dispatch SpiceDB gRPC API func DefaultDispatchMiddleware(logger zerolog.Logger, authFunc grpcauth.AuthFunc, ds datastore.Datastore) ([]grpc.UnaryServerInterceptor, []grpc.StreamServerInterceptor) { + grpcMetricsUnaryInterceptor, grpcMetricsStreamingInterceptor := GRPCMetrics() return []grpc.UnaryServerInterceptor{ requestid.UnaryServerInterceptor(requestid.GenerateIfMissing(true)), logmw.UnaryServerInterceptor(logmw.ExtractMetadataField("x-request-id", "requestID")), otelgrpc.UnaryServerInterceptor(), // nolint: staticcheck grpclog.UnaryServerInterceptor(InterceptorLogger(logger), defaultCodeToLevel, durationFieldOption, traceIDFieldOption), - GRPCMetricsUnaryInterceptor, + grpcMetricsUnaryInterceptor, grpcauth.UnaryServerInterceptor(authFunc), datastoremw.UnaryServerInterceptor(ds), servicespecific.UnaryServerInterceptor, @@ -371,7 +382,7 @@ func DefaultDispatchMiddleware(logger zerolog.Logger, authFunc grpcauth.AuthFunc logmw.StreamServerInterceptor(logmw.ExtractMetadataField("x-request-id", "requestID")), otelgrpc.StreamServerInterceptor(), // nolint: staticcheck grpclog.StreamServerInterceptor(InterceptorLogger(logger), defaultCodeToLevel, durationFieldOption, traceIDFieldOption), - GRPCMetricsStreamingInterceptor, + grpcMetricsStreamingInterceptor, grpcauth.StreamServerInterceptor(authFunc), datastoremw.StreamServerInterceptor(ds), servicespecific.StreamServerInterceptor, @@ -406,7 +417,10 @@ func createServerMetrics() (grpc.UnaryServerInterceptor, grpc.StreamServerInterc ), ) - prometheus.DefaultRegisterer.MustRegister(srvMetrics) + // deliberately ignore if these metrics were already registered, so that + // custom builds of SpiceDB can register these metrics with custom labels + _ = prometheus.DefaultRegisterer.Register(srvMetrics) + exemplarFromContext := func(ctx context.Context) prometheus.Labels { if span := trace.SpanContextFromContext(ctx); span.IsSampled() { return prometheus.Labels{"traceID": span.TraceID().String()}