utils prometheusmetrics: convert gauges to counters (#3093)

go.mod

@@ -104,6 +104,7 @@ require (
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
 	github.com/lib/pq v1.10.9 // indirect

pkg/awsutils/awsutils_test.go

@@ -26,13 +26,16 @@ import (
 	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/golang/mock/gomock"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 
 	mock_ec2wrapper "github.com/aws/amazon-vpc-cni-k8s/pkg/ec2wrapper/mocks"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/utils/eventrecorder"
+	"github.com/aws/amazon-vpc-cni-k8s/utils/prometheusmetrics"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -780,6 +783,33 @@ func TestFreeENIRetry(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestAwsAPIErrInc(t *testing.T) {
+	// Reset metrics before test
+	prometheusmetrics.AwsAPIErr.Reset()
+
+	// Test case 1: AWS error
+	awsErr := awserr.New("InvalidParameterException", "The parameter is invalid", nil)
+	awsAPIErrInc("CreateNetworkInterface", awsErr)
+
+	// Verify metric was incremented with correct labels
+	count := testutil.ToFloat64(prometheusmetrics.AwsAPIErr.With(prometheus.Labels{
+		"api":   "CreateNetworkInterface",
+		"error": "InvalidParameterException",
+	}))
+	assert.Equal(t, float64(1), count)
+
+	// Test case 2: Non-AWS error
+	regularErr := errors.New("some other error")
+	awsAPIErrInc("CreateNetworkInterface", regularErr)
+
+	// Verify metric was not incremented for non-AWS error
+	count = testutil.ToFloat64(prometheusmetrics.AwsAPIErr.With(prometheus.Labels{
+		"api":   "CreateNetworkInterface",
+		"error": "InvalidParameterException",
+	}))
+	assert.Equal(t, float64(1), count)
+}
+
 func TestFreeENIRetryMax(t *testing.T) {
 	ctrl, mockEC2 := setup(t)
 	defer ctrl.Finish()

pkg/ipamd/datastore/data_store_test.go

@@ -22,6 +22,7 @@ import (
 
 	mock_netlinkwrapper "github.com/aws/amazon-vpc-cni-k8s/pkg/netlinkwrapper/mocks"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils"
+	"github.com/aws/amazon-vpc-cni-k8s/utils/prometheusmetrics"
 	"github.com/golang/mock/gomock"
 	"github.com/vishvananda/netlink"
 	"golang.org/x/sys/unix"
@@ -31,6 +32,8 @@ import (
 
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -1542,3 +1545,73 @@ func TestDataStore_validateAllocationByPodVethExistence(t *testing.T) {
 		})
 	}
 }
+
+func TestForceRemovalMetrics(t *testing.T) {
+	// Reset metrics by creating new counters
+	prometheusmetrics.ForceRemovedENIs = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "awscni_force_removed_enis_total",
+		Help: "The total number of ENIs force removed",
+	})
+	prometheusmetrics.ForceRemovedIPs = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "awscni_force_removed_ips_total",
+		Help: "The total number of IPs force removed",
+	})
+
+	ds := NewDataStore(Testlog, NullCheckpoint{}, false)
+
+	// Add an ENI and IP
+	err := ds.AddENI("eni-1", 1, true, false, false)
+	assert.NoError(t, err)
+
+	ipv4Addr := net.IPNet{IP: net.ParseIP("1.1.1.1"), Mask: net.IPv4Mask(255, 255, 255, 255)}
+	err = ds.AddIPv4CidrToStore("eni-1", ipv4Addr, false)
+	assert.NoError(t, err)
+
+	// Assign IP to a pod
+	key := IPAMKey{"net0", "sandbox-1", "eth0"}
+	ip, device, err := ds.AssignPodIPv4Address(key, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"})
+	assert.NoError(t, err)
+	assert.Equal(t, "1.1.1.1", ip)
+	assert.Equal(t, 1, device)
+
+	// Test force removal of IP
+	err = ds.DelIPv4CidrFromStore("eni-1", ipv4Addr, false)
+	assert.Error(t, err) // Should fail without force
+	assert.Contains(t, err.Error(), "IP is used and can not be deleted")
+
+	ipCount := testutil.ToFloat64(prometheusmetrics.ForceRemovedIPs)
+	assert.Equal(t, float64(0), ipCount)
+
+	// Force remove the IP
+	err = ds.DelIPv4CidrFromStore("eni-1", ipv4Addr, true)
+	assert.NoError(t, err) // Should succeed with force
+
+	ipCount = testutil.ToFloat64(prometheusmetrics.ForceRemovedIPs)
+	assert.Equal(t, float64(1), ipCount)
+
+	// Add another IP and assign to pod for ENI removal test
+	ipv4Addr2 := net.IPNet{IP: net.ParseIP("1.1.1.2"), Mask: net.IPv4Mask(255, 255, 255, 255)}
+	err = ds.AddIPv4CidrToStore("eni-1", ipv4Addr2, false)
+	assert.NoError(t, err)
+
+	key2 := IPAMKey{"net0", "sandbox-2", "eth0"}
+	ip, device, err = ds.AssignPodIPv4Address(key2, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"})
+	assert.NoError(t, err)
+	assert.Equal(t, "1.1.1.2", ip)
+	assert.Equal(t, 1, device)
+
+	// Test force removal of ENI
+	err = ds.RemoveENIFromDataStore("eni-1", false)
+	assert.Error(t, err)                                                             // Should fail without force
+	assert.Contains(t, err.Error(), "datastore: ENI is used and can not be deleted") // Updated error message
+
+	eniCount := testutil.ToFloat64(prometheusmetrics.ForceRemovedENIs)
+	assert.Equal(t, float64(0), eniCount)
+
+	// Force remove the ENI
+	err = ds.RemoveENIFromDataStore("eni-1", true)
+	assert.NoError(t, err) // Should succeed with force
+
+	eniCount = testutil.ToFloat64(prometheusmetrics.ForceRemovedENIs)
+	assert.Equal(t, float64(1), eniCount)
+}