Merge pull request #327 from pyrra-dev/fix-latency-burnrates-regexp

slo: Fix latency burnrate recording rules with regexp matchers
pyrra-dev · Jun 16, 2022 · 70b3a1a · 70b3a1a
2 parents 1f6478a + ef1494b
commit 70b3a1a
Show file tree

Hide file tree

Showing 3 changed files with 45 additions and 40 deletions.
diff --git a/slo/promql_test.go b/slo/promql_test.go
@@ -229,6 +229,10 @@ var (
 			Window: model.Duration(14 * 24 * time.Hour),
 			Indicator: Indicator{
 				Latency: &LatencyIndicator{
+					Grouping: []string{
+						"resource",
+						"verb",
+					},
 					Success: Metric{
 						Name: "apiserver_request_duration_seconds_bucket",
 						LabelMatchers: []*labels.Matcher{
@@ -312,7 +316,7 @@ func TestObjective_QueryTotal(t *testing.T) {
 	}, {
 		name:      "apiserver-read-resource-latency",
 		objective: objectiveAPIServerLatency(),
-		expected:  `sum(apiserver_request_duration_seconds:increase2w{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"})`,
+		expected:  `sum by(resource, verb) (apiserver_request_duration_seconds:increase2w{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"})`,
 	}}
 	for _, tc := range testcases {
 		t.Run(tc.name, func(t *testing.T) {
@@ -381,7 +385,7 @@ func TestObjective_QueryErrors(t *testing.T) {
 	}, {
 		name:      "apiserver-read-resource-latency",
 		objective: objectiveAPIServerLatency(),
-		expected:  `sum(apiserver_request_duration_seconds:increase2w{job="apiserver",le="",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"}) - sum(apiserver_request_duration_seconds:increase2w{job="apiserver",le="0.1",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"})`,
+		expected:  `sum by(resource, verb) (apiserver_request_duration_seconds:increase2w{job="apiserver",le="",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"}) - sum by(resource, verb) (apiserver_request_duration_seconds:increase2w{job="apiserver",le="0.1",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"})`,
 	}}
 	for _, tc := range testcases {
 		t.Run(tc.name, func(t *testing.T) {

diff --git a/slo/rules.go b/slo/rules.go
@@ -194,6 +194,12 @@ func (o Objective) Burnrates() (monitoringv1.RuleGroup, error) {
 			if m.Name == labels.MetricName {
 				continue
 			}
+			if _, ok := groupingMap[m.Name]; !ok {
+				if m.Type == labels.MatchRegexp || m.Type == labels.MatchNotRegexp {
+					continue
+				}
+			}
+
 			alertMatchers = append(alertMatchers, m.String())
 		}
 		alertMatchers = append(alertMatchers, fmt.Sprintf(`slo="%s"`, sloName))
@@ -299,11 +305,6 @@ func (o Objective) Burnrate(timerange time.Duration) string {
 		for _, s := range o.Indicator.Latency.Grouping {
 			groupingMap[s] = struct{}{}
 		}
-		for _, m := range o.Indicator.Latency.Total.LabelMatchers {
-			if m.Type == labels.MatchRegexp || m.Type == labels.MatchNotRegexp {
-				groupingMap[m.Name] = struct{}{}
-			}
-		}
 
 		grouping := make([]string, 0, len(groupingMap))
 		for s := range groupingMap {