From d56fe10ae550405d8ea03af56b9b60103f3171f1 Mon Sep 17 00:00:00 2001 From: Joe Wogan Date: Thu, 23 May 2024 18:03:42 +0100 Subject: [PATCH 1/3] fix: aligning the interceptor metrics vars with the OTEL spec Signed-off-by: Joe Wogan --- CHANGELOG.md | 1 + .../interceptor/e2e-test/otel/deployment.yaml | 14 ++++---- docs/operate.md | 8 ++--- interceptor/config/metrics.go | 14 ++------ interceptor/metrics/otelmetrics.go | 32 ++----------------- .../interceptor_otel_metrics_test.go | 2 +- 6 files changed, 16 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16297bc0..9275cc77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ This changelog keeps track of work items that have been completed and are ready ### Fixes +- **General**: Align the interceptor metrics env var configuration with the OTEL spec ([#1031](https://github.com/kedacore/http-add-on/issues/1031)) - **General**: TODO ([#TODO](https://github.com/kedacore/http-add-on/issues/TODO)) ### Deprecations diff --git a/config/interceptor/e2e-test/otel/deployment.yaml b/config/interceptor/e2e-test/otel/deployment.yaml index 0f6b3ec8..f3712fc5 100644 --- a/config/interceptor/e2e-test/otel/deployment.yaml +++ b/config/interceptor/e2e-test/otel/deployment.yaml @@ -9,15 +9,13 @@ spec: containers: - name: interceptor env: - - name: KEDA_HTTP_OTEL_PROM_EXPORTER_ENABLED + - name: OTEL_PROM_EXPORTER_ENABLED value: "true" - - name: KEDA_HTTP_OTEL_PROM_EXPORTER_PORT + - name: OTEL_PROM_EXPORTER_PORT value: "2223" - - name: KEDA_HTTP_OTEL_HTTP_EXPORTER_ENABLED + - name: OTEL_EXPORTER_OTLP_METRICS_ENABLED value: "true" - - name: KEDA_HTTP_OTEL_HTTP_COLLECTOR_ENDPOINT - value: "opentelemetry-collector.open-telemetry-system:4318" - - name: KEDA_HTTP_OTEL_HTTP_COLLECTOR_INSECURE - value: "true" - - name: KEDA_HTTP_OTEL_METRIC_EXPORT_INTERVAL + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://opentelemetry-collector.open-telemetry-system:4318" + - name: OTEL_METRIC_EXPORT_INTERVAL value: "1" diff --git a/docs/operate.md b/docs/operate.md index a3b67a9c..691c3a32 100644 --- a/docs/operate.md +++ b/docs/operate.md @@ -9,16 +9,14 @@ There are currently 2 supported methods for exposing metrics from the intercepto ### Configuring the Prometheus compatible metrics endpoint When configured, the interceptor proxy can expose metrics on a Prometheus compatible endpoint. -This endpoint can be enabled by setting the `KEDA_HTTP_OTEL_PROM_EXPORTER_ENABLED` environment variable to `true` on the interceptor deployment (`true` by default) and by setting `KEDA_HTTP_OTEL_PROM_EXPORTER_PORT` to an unused port for the endpoint to be made avaialble on (`2223` by default). +This endpoint can be enabled by setting the `OTEL_PROM_EXPORTER_ENABLED` environment variable to `true` on the interceptor deployment (`true` by default) and by setting `OTEL_PROM_EXPORTER_PORT` to an unused port for the endpoint to be made avaialble on (`2223` by default). ### Configuring the OTEL HTTP exporter When configured, the interceptor proxy can export metrics to a OTEL HTTP collector. -The OTEL exporter can be enabled by setting the `KEDA_HTTP_OTEL_HTTP_EXPORTER_ENABLED` environment variable to `true` on the interceptor deployment (`false` by default). When enabled the `KEDA_HTTP_OTEL_HTTP_COLLECTOR_ENDPOINT` environment variable must also be configured so the exporter knows what collector to send the metrics to (e.g. opentelemetry-collector.open-telemetry-system:4318). +The OTEL exporter can be enabled by setting the `OTEL_EXPORTER_OTLP_METRICS_ENABLED` environment variable to `true` on the interceptor deployment (`false` by default). When enabled the `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable must also be configured so the exporter knows what collector to send the metrics to (e.g. http://opentelemetry-collector.open-telemetry-system:4318). -If the collector is exposed on a unsecured endpoint then you can set the `KEDA_HTTP_OTEL_HTTP_COLLECTOR_INSECURE` environment variable to `true` (`false` by default) which will disable client security on the exporter. - -If you need to provide any headers such as authentication details in order to utilise your OTEL collector you can add them into the `KEDA_HTTP_OTEL_HTTP_HEADERS` environment variable. The frequency at which the metrics are exported can be configured by setting `KEDA_HTTP_OTEL_METRIC_EXPORT_INTERVAL` to the number of seconds you require between each export interval (`30` by default). +If you need to provide any headers such as authentication details in order to utilise your OTEL collector you can add them into the `OTEL_EXPORTER_OTLP_HEADERS` environment variable. The frequency at which the metrics are exported can be configured by setting `OTEL_METRIC_EXPORT_INTERVAL` to the number of seconds you require between each export interval (`30` by default). # Configuring TLS for the KEDA HTTP Add-on interceptor proxy diff --git a/interceptor/config/metrics.go b/interceptor/config/metrics.go index 3def7730..d1c3e05e 100644 --- a/interceptor/config/metrics.go +++ b/interceptor/config/metrics.go @@ -7,19 +7,11 @@ import ( // Metrics is the configuration for configuring metrics in the interceptor. type Metrics struct { // Sets whether or not to enable the Prometheus metrics exporter - OtelPrometheusExporterEnabled bool `envconfig:"KEDA_HTTP_OTEL_PROM_EXPORTER_ENABLED" default:"true"` + OtelPrometheusExporterEnabled bool `envconfig:"OTEL_PROM_EXPORTER_ENABLED" default:"true"` // Sets the port which the Prometheus compatible metrics endpoint should be served on - OtelPrometheusExporterPort int `envconfig:"KEDA_HTTP_OTEL_PROM_EXPORTER_PORT" default:"2223"` + OtelPrometheusExporterPort int `envconfig:"OTEL_PROM_EXPORTER_PORT" default:"2223"` // Sets whether or not to enable the OTEL metrics exporter - OtelHTTPExporterEnabled bool `envconfig:"KEDA_HTTP_OTEL_HTTP_EXPORTER_ENABLED" default:"false"` - // Sets the HTTP endpoint where metrics should be sent to - OtelHTTPCollectorEndpoint string `envconfig:"KEDA_HTTP_OTEL_HTTP_COLLECTOR_ENDPOINT" default:"localhost:4318"` - // Sets the OTLP headers required by the otel exporter - OtelHTTPHeaders string `envconfig:"KEDA_HTTP_OTEL_HTTP_HEADERS" default:""` - // Set the connection to the otel HTTP collector endpoint to use HTTP rather than HTTPS - OtelHTTPCollectorInsecure bool `envconfig:"KEDA_HTTP_OTEL_HTTP_COLLECTOR_INSECURE" default:"false"` - // Set the interval in seconds to export otel metrics to the configured collector endpoint - OtelMetricExportInterval int `envconfig:"KEDA_HTTP_OTEL_METRIC_EXPORT_INTERVAL" default:"30"` + OtelHTTPExporterEnabled bool `envconfig:"OTEL_EXPORTER_OTLP_METRICS_ENABLED" default:"false"` } // Parse parses standard configs using envconfig and returns a pointer to the diff --git a/interceptor/metrics/otelmetrics.go b/interceptor/metrics/otelmetrics.go index 91f4b8a3..5c505cb5 100644 --- a/interceptor/metrics/otelmetrics.go +++ b/interceptor/metrics/otelmetrics.go @@ -3,8 +3,6 @@ package metrics import ( "context" "log" - "strings" - "time" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" @@ -25,19 +23,8 @@ type OtelMetrics struct { func NewOtelMetrics(metricsConfig *config.Metrics, options ...metric.Option) *OtelMetrics { ctx := context.Background() - var exporter *otlpmetrichttp.Exporter - var err error - endpoint := otlpmetrichttp.WithEndpoint(metricsConfig.OtelHTTPCollectorEndpoint) - headersFromEnvVar := getHeaders(metricsConfig.OtelHTTPHeaders) - headers := otlpmetrichttp.WithHeaders(headersFromEnvVar) - - if metricsConfig.OtelHTTPCollectorInsecure { - insecure := otlpmetrichttp.WithInsecure() - exporter, err = otlpmetrichttp.New(ctx, endpoint, headers, insecure) - } else { - exporter, err = otlpmetrichttp.New(ctx, endpoint, headers) - } + exporter, err := otlpmetrichttp.New(ctx) if err != nil { log.Fatalf("could not create otelmetrichttp exporter: %v", err) } @@ -50,7 +37,7 @@ func NewOtelMetrics(metricsConfig *config.Metrics, options ...metric.Option) *Ot ) options = []metric.Option{ - metric.WithReader(metric.NewPeriodicReader(exporter, metric.WithInterval(time.Duration(metricsConfig.OtelMetricExportInterval)*time.Second))), + metric.WithReader(metric.NewPeriodicReader(exporter)), metric.WithResource(res), } } @@ -98,18 +85,3 @@ func (om *OtelMetrics) RecordPendingRequestCount(host string, value int64) { om.pendingRequestCounter.Add(ctx, value, opt) } - -func getHeaders(s string) map[string]string { - // Get the headers in key-value pair from the KEDA_HTTP_OTEL_HTTP_HEADERS environment variable - var m = map[string]string{} - - if s != "" { - h := strings.Split(s, ",") - for _, v := range h { - x := strings.Split(v, "=") - m[x[0]] = x[1] - } - } - - return m -} diff --git a/tests/checks/interceptor_otel_metrics/interceptor_otel_metrics_test.go b/tests/checks/interceptor_otel_metrics/interceptor_otel_metrics_test.go index bceb5411..c0caae9c 100644 --- a/tests/checks/interceptor_otel_metrics/interceptor_otel_metrics_test.go +++ b/tests/checks/interceptor_otel_metrics/interceptor_otel_metrics_test.go @@ -171,7 +171,7 @@ func TestMetricGeneration(t *testing.T) { // If the metric is not found first time around then retry with a delay. if !ok { // Add a small sleep to allow metrics to be pushed from the exporter to the collector - time.Sleep(5 * time.Second) + time.Sleep(10 * time.Second) // Fetch metrics and validate them family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", otelCollectorPromURL)) val, ok = family["interceptor_request_count_total"] From 0578ef2ba1a91acb4942de32e5d923e5518c08c2 Mon Sep 17 00:00:00 2001 From: Joe Wogan Date: Wed, 12 Jun 2024 14:53:27 +0100 Subject: [PATCH 2/3] chore: update skip urls in linkinator GithubAction Signed-off-by: Joe Wogan --- .github/workflows/linkinator.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linkinator.yaml b/.github/workflows/linkinator.yaml index 3b0e2a54..bb469d18 100644 --- a/.github/workflows/linkinator.yaml +++ b/.github/workflows/linkinator.yaml @@ -19,4 +19,4 @@ jobs: paths: "**/*.md" markdown: true retry: true - linksToSkip: "https://github.com/kedacore/http-add-on/pkgs/container/http-add-on-interceptor, https://github.com/kedacore/http-add-on/pkgs/container/http-add-on-operator, https://github.com/kedacore/http-add-on/pkgs/container/http-add-on-scaler" + linksToSkip: "https://github.com/kedacore/http-add-on/pkgs/container/http-add-on-interceptor, https://github.com/kedacore/http-add-on/pkgs/container/http-add-on-operator, https://github.com/kedacore/http-add-on/pkgs/container/http-add-on-scaler, http://opentelemetry-collector.open-telemetry-system:4318" From 5eb768320676799208d1e4f46e68c8fd88b119b4 Mon Sep 17 00:00:00 2001 From: Joe Wogan Date: Wed, 12 Jun 2024 15:00:16 +0100 Subject: [PATCH 3/3] chore: removing unused metricsCfg references Signed-off-by: Joe Wogan --- interceptor/metrics/metricscollector.go | 2 +- interceptor/metrics/otelmetrics.go | 3 +-- interceptor/metrics/otelmetrics_test.go | 5 +---- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/interceptor/metrics/metricscollector.go b/interceptor/metrics/metricscollector.go index 4921d702..0d0c50b8 100644 --- a/interceptor/metrics/metricscollector.go +++ b/interceptor/metrics/metricscollector.go @@ -22,7 +22,7 @@ func NewMetricsCollectors(metricsConfig *config.Metrics) { } if metricsConfig.OtelHTTPExporterEnabled { - otelhttpmetrics := NewOtelMetrics(metricsConfig) + otelhttpmetrics := NewOtelMetrics() collectors = append(collectors, otelhttpmetrics) } } diff --git a/interceptor/metrics/otelmetrics.go b/interceptor/metrics/otelmetrics.go index 5c505cb5..6ad5581f 100644 --- a/interceptor/metrics/otelmetrics.go +++ b/interceptor/metrics/otelmetrics.go @@ -11,7 +11,6 @@ import ( "go.opentelemetry.io/otel/sdk/resource" semconv "go.opentelemetry.io/otel/semconv/v1.4.0" - "github.com/kedacore/http-add-on/interceptor/config" "github.com/kedacore/http-add-on/pkg/build" ) @@ -21,7 +20,7 @@ type OtelMetrics struct { pendingRequestCounter api.Int64UpDownCounter } -func NewOtelMetrics(metricsConfig *config.Metrics, options ...metric.Option) *OtelMetrics { +func NewOtelMetrics(options ...metric.Option) *OtelMetrics { ctx := context.Background() exporter, err := otlpmetrichttp.New(ctx) diff --git a/interceptor/metrics/otelmetrics_test.go b/interceptor/metrics/otelmetrics_test.go index af5ac7c1..69c4ca31 100644 --- a/interceptor/metrics/otelmetrics_test.go +++ b/interceptor/metrics/otelmetrics_test.go @@ -7,8 +7,6 @@ import ( "github.com/stretchr/testify/assert" "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/metricdata" - - "github.com/kedacore/http-add-on/interceptor/config" ) var ( @@ -19,8 +17,7 @@ var ( func init() { testReader = metric.NewManualReader() options := metric.WithReader(testReader) - metricsCfg := config.Metrics{} - testOtel = NewOtelMetrics(&metricsCfg, options) + testOtel = NewOtelMetrics(options) } func TestRequestCounter(t *testing.T) {