iptables operation add timeout and retry exponentially

cmd/egress-cni-plugin/egressContext.go

@@ -29,6 +29,7 @@ import (
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/hostipamwrapper"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/iptableswrapper"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/netlinkwrapper"
+	"github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/nswrapper"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/procsyswrapper"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/utils/cniutils"
@@ -82,7 +83,7 @@ func NewEgressAddContext(nsPath, ifName string) egressContext {
 		ArgsIfName: ifName,
 		Veth:       vethwrapper.NewSetupVeth(),
 		IptCreator: func(protocol iptables.Protocol) (iptableswrapper.IPTablesIface, error) {
-			return iptableswrapper.NewIPTables(protocol)
+			return networkutils.NewIPTables(protocol)
 		},
 	}
 }
@@ -95,7 +96,7 @@ func NewEgressDelContext(nsPath string) egressContext {
 		Ns:     nswrapper.NewNS(),
 		NsPath: nsPath,
 		IptCreator: func(protocol iptables.Protocol) (iptableswrapper.IPTablesIface, error) {
-			return iptableswrapper.NewIPTables(protocol)
+			return networkutils.NewIPTables(protocol)
 		},
 	}
 }

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/eks v1.52.1
 	github.com/aws/aws-sdk-go-v2/service/iam v1.38.1
 	github.com/aws/smithy-go v1.22.1
+	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/containernetworking/cni v1.2.3
 	github.com/containernetworking/plugins v1.5.1
 	github.com/coreos/go-iptables v0.8.0

go.sum

@@ -91,6 +91,8 @@ github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembj
 github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50=
 github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 h1:nvj0OLI3YqYXer/kZD8Ri1aaunCxIEsOst1BVJswV0o=
 github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
+github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
+github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=

pkg/networkutils/iptables.go

@@ -0,0 +1,141 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+// Package networkutils is a collection of iptables and netlink functions
+package networkutils
+
+import (
+	"time"
+
+	"github.com/aws/amazon-vpc-cni-k8s/pkg/iptableswrapper"
+	"github.com/cenkalti/backoff/v4"
+	"github.com/coreos/go-iptables/iptables"
+)
+
+type ipTables struct {
+	ipt     iptableswrapper.IPTablesIface
+	backoff *backoff.ExponentialBackOff
+}
+
+// NewIPTables return a ipTables struct that implements IPTablesIface
+func NewIPTables(protocol iptables.Protocol) (iptableswrapper.IPTablesIface, error) {
+	ipt, err := iptables.New(iptables.IPFamily(protocol), iptables.Timeout(1))
+	if err != nil {
+		return nil, err
+	}
+	return &ipTables{
+		ipt:     ipt,
+		backoff: backoff.NewExponentialBackOff(backoff.WithMaxElapsedTime(0)), // Never stop retrying as backward compatibility
+	}, nil
+}
+
+func (i ipTables) Exists(table, chain string, rulespec ...string) (bool, error) {
+	operation := func() (bool, error) {
+		return i.ipt.Exists(table, chain, rulespec...)
+	}
+	result, err := backoff.RetryNotifyWithData(operation, i.backoff, logRetryError)
+	if err != nil {
+		return true, err
+	}
+	return result, nil
+}
+
+func (i ipTables) Insert(table, chain string, pos int, rulespec ...string) error {
+	operation := func() error {
+		return i.ipt.Insert(table, chain, pos, rulespec...)
+	}
+	return backoff.RetryNotify(operation, i.backoff, logRetryError)
+}
+
+func (i ipTables) Append(table, chain string, rulespec ...string) error {
+	operation := func() error {
+		return i.ipt.Append(table, chain, rulespec...)
+	}
+	return backoff.RetryNotify(operation, i.backoff, logRetryError)
+}
+
+func (i ipTables) AppendUnique(table, chain string, rulespec ...string) error {
+	operation := func() error {
+		return i.ipt.AppendUnique(table, chain, rulespec...)
+	}
+	return backoff.RetryNotify(operation, i.backoff, logRetryError)
+}
+
+func (i ipTables) Delete(table, chain string, rulespec ...string) error {
+	operation := func() error {
+		return i.ipt.Delete(table, chain, rulespec...)
+	}
+	return backoff.RetryNotify(operation, i.backoff, logRetryError)
+}
+
+func (i ipTables) List(table, chain string) ([]string, error) {
+	operation := func() ([]string, error) {
+		return i.ipt.List(table, chain)
+	}
+	result, err := backoff.RetryNotifyWithData(operation, i.backoff, logRetryError)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (i ipTables) NewChain(table, chain string) error {
+	operation := func() error {
+		return i.ipt.NewChain(table, chain)
+	}
+	return backoff.RetryNotify(operation, i.backoff, logRetryError)
+}
+
+func (i ipTables) ClearChain(table, chain string) error {
+	operation := func() error {
+		return i.ipt.ClearChain(table, chain)
+	}
+	return backoff.RetryNotify(operation, i.backoff, logRetryError)
+}
+
+func (i ipTables) DeleteChain(table, chain string) error {
+	operation := func() error {
+		return i.ipt.DeleteChain(table, chain)
+	}
+	return backoff.RetryNotify(operation, i.backoff, logRetryError)
+}
+
+func (i ipTables) ListChains(table string) ([]string, error) {
+	operation := func() ([]string, error) {
+		return i.ipt.ListChains(table)
+	}
+	result, err := backoff.RetryNotifyWithData(operation, i.backoff, logRetryError)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (i ipTables) ChainExists(table, chain string) (bool, error) {
+	operation := func() (bool, error) {
+		return i.ipt.ChainExists(table, chain)
+	}
+	result, err := backoff.RetryNotifyWithData(operation, i.backoff, logRetryError)
+	if err != nil {
+		return true, err
+	}
+	return result, nil
+}
+
+func (i ipTables) HasRandomFully() bool {
+	return i.ipt.HasRandomFully()
+}
+
+func logRetryError(err error, t time.Duration) {
+	log.Errorf("Another app is currently holding the xtables lock. Retrying in %f seconds", t.Seconds())
+}