update: autoscaling-karpenter to v0.32.1 (#129)

awslabs · Nov 2, 2023 · 7ba827a · 7ba827a
1 parent 6b9cc95
commit 7ba827a
Show file tree

Hide file tree

Showing 5 changed files with 154 additions and 107 deletions.
diff --git a/pkg/application/autoscaling/karpenter/default_nodepool.go b/pkg/application/autoscaling/karpenter/default_nodepool.go
@@ -0,0 +1,77 @@
+package karpenter
+
+import (
+	"github.com/awslabs/eksdemo/pkg/manifest"
+	"github.com/awslabs/eksdemo/pkg/resource"
+	"github.com/awslabs/eksdemo/pkg/template"
+)
+
+type NodePoolOptions struct {
+	resource.CommonOptions
+	*KarpenterOptions
+}
+
+func karpenterDefaultNodePool(o *KarpenterOptions) *resource.Resource {
+	res := &resource.Resource{
+		Options: &NodePoolOptions{
+			CommonOptions: resource.CommonOptions{
+				Name: "karpenter-default-nodepool",
+			},
+			KarpenterOptions: o,
+		},
+
+		Manager: &manifest.ResourceManager{
+			Template: &template.TextTemplate{
+				Template: yamlTemplate,
+			},
+		},
+	}
+	return res
+}
+
+const yamlTemplate = `---
+apiVersion: karpenter.sh/v1beta1
+kind: NodePool
+metadata:
+  name: default
+spec:
+  template:
+    spec:
+      requirements:
+        - key: kubernetes.io/arch
+          operator: In
+          values: ["amd64"]
+        - key: kubernetes.io/os
+          operator: In
+          values: ["linux"]
+        - key: karpenter.sh/capacity-type
+          operator: In
+          values: ["on-demand", "spot"]
+        - key: karpenter.k8s.aws/instance-category
+          operator: In
+          values: ["c", "m", "r"]
+        - key: karpenter.k8s.aws/instance-generation
+          operator: Gt
+          values: ["2"]
+      nodeClassRef:
+        name: default
+  limits:
+    cpu: 1000
+  disruption:
+    consolidationPolicy: WhenUnderutilized
+    expireAfter: {{ .ExpireAfter }}
+---
+apiVersion: karpenter.k8s.aws/v1beta1
+kind: EC2NodeClass
+metadata:
+  name: default
+spec:
+  amiFamily: {{ .AMIFamily }}
+  role: KarpenterNodeRole-{{ .ClusterName }}
+  subnetSelectorTerms:
+    - tags:
+        Name: eksctl-{{ .ClusterName }}-cluster/SubnetPrivate*
+  securityGroupSelectorTerms:
+    - tags:
+        aws:eks:cluster-name: {{ .ClusterName }}
+`
diff --git a/pkg/application/autoscaling/karpenter/default_provisioner.go b/pkg/application/autoscaling/karpenter/default_provisioner.go
diff --git a/pkg/application/autoscaling/karpenter/karpenter.go b/pkg/application/autoscaling/karpenter/karpenter.go
@@ -15,7 +15,7 @@ import (
 // GitHub:  https://github.com/awslabs/karpenter
 // Helm:    https://github.com/awslabs/karpenter/tree/main/charts/karpenter
 // Repo:    https://gallery.ecr.aws/karpenter/controller
-// Version: Latest is v0.31.0 (as of 10/3/23)
+// Version: Latest is v0.32.1 (as of 11/1/23)
 
 func NewApp() *application.Application {
 	options, flags := newOptions()
@@ -69,7 +69,7 @@ func NewApp() *application.Application {
 		},
 
 		PostInstallResources: []*resource.Resource{
-			karpenterDefaultProvisioner(options),
+			karpenterDefaultNodePool(options),
 		},
 	}
 	app.Options = options
@@ -109,7 +109,7 @@ Statement:
     StringEquals:
       aws:RequestTag/kubernetes.io/cluster/{{ .ClusterName }}: owned
     StringLike:
-      aws:RequestTag/karpenter.sh/provisioner-name: "*"
+      aws:RequestTag/karpenter.sh/nodepool: "*"
 - Sid: AllowScopedResourceCreationTagging
   Effect: Allow
   Resource:
@@ -127,21 +127,20 @@ Statement:
       - CreateFleet
       - CreateLaunchTemplate
     StringLike:
-      aws:RequestTag/karpenter.sh/provisioner-name: "*"
-- Sid: AllowMachineMigrationTagging
+      aws:RequestTag/karpenter.sh/nodepool: "*"
+- Sid: AllowScopedResourceTagging
   Effect: Allow
   Resource: arn:{{ .Partition }}:ec2:{{ .Region }}:*:instance/*
   Action: ec2:CreateTags
   Condition:
     StringEquals:
       aws:ResourceTag/kubernetes.io/cluster/{{ .ClusterName }}: owned
-      aws:RequestTag/karpenter.sh/managed-by: "{{ .ClusterName }}"
     StringLike:
-      aws:RequestTag/karpenter.sh/provisioner-name: "*"
+      aws:ResourceTag/karpenter.sh/nodepool: "*"
     ForAllValues:StringEquals:
       aws:TagKeys:
-      - karpenter.sh/provisioner-name
-      - karpenter.sh/managed-by
+      - karpenter.sh/nodeclaim
+      - Name
 - Sid: AllowScopedDeletion
   Effect: Allow
   Resource:
@@ -154,7 +153,7 @@ Statement:
     StringEquals:
       aws:ResourceTag/kubernetes.io/cluster/{{ .ClusterName }}: owned
     StringLike:
-      aws:ResourceTag/karpenter.sh/provisioner-name: "*"
+      aws:ResourceTag/karpenter.sh/nodepool: "*"
 - Sid: AllowRegionalReadActions
   Effect: Allow
   Resource: "*"
@@ -196,6 +195,48 @@ Statement:
   Condition:
     StringEquals:
       iam:PassedToService: ec2.amazonaws.com
+- Sid: AllowScopedInstanceProfileCreationActions
+  Effect: Allow
+  Resource: "*"
+  Action:
+  - iam:CreateInstanceProfile
+  Condition:
+    StringEquals:
+      aws:RequestTag/kubernetes.io/cluster/{{ .ClusterName }}: owned
+      aws:RequestTag/topology.kubernetes.io/region: "{{ .Region }}"
+    StringLike:
+      aws:RequestTag/karpenter.k8s.aws/ec2nodeclass: "*"
+- Sid: AllowScopedInstanceProfileTagActions
+  Effect: Allow
+  Resource: "*"
+  Action:
+  - iam:TagInstanceProfile
+  Condition:
+    StringEquals:
+      aws:ResourceTag/kubernetes.io/cluster/{{ .ClusterName }}: owned
+      aws:ResourceTag/topology.kubernetes.io/region: "{{ .Region }}"
+      aws:RequestTag/kubernetes.io/cluster/{{ .ClusterName }}: owned
+      aws:RequestTag/topology.kubernetes.io/region: "{{ .Region }}"
+    StringLike:
+      aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass: "*"
+      aws:RequestTag/karpenter.k8s.aws/ec2nodeclass: "*"
+- Sid: AllowScopedInstanceProfileActions
+  Effect: Allow
+  Resource: "*"
+  Action:
+  - iam:AddRoleToInstanceProfile
+  - iam:RemoveRoleFromInstanceProfile
+  - iam:DeleteInstanceProfile
+  Condition:
+    StringEquals:
+      aws:ResourceTag/kubernetes.io/cluster/{{ .ClusterName }}: owned
+      aws:ResourceTag/topology.kubernetes.io/region: "{{ .Region }}"
+    StringLike:
+      aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass: "*"
+- Sid: AllowInstanceProfileReadActions
+  Effect: Allow
+  Resource: "*"
+  Action: iam:GetInstanceProfile
 - Sid: AllowAPIServerEndpointDiscovery
   Effect: Allow
   Resource: arn:{{ .Partition }}:eks:{{ .Region }}:{{ .Account }}:cluster/{{ .ClusterName }}
@@ -212,14 +253,14 @@ replicas: {{ .Replicas }}
 controller:
   image:
     tag: {{ .Version }}
+  resources:
+    requests:
+      cpu: "1"
+      memory: "1Gi"
 settings:
-  aws:
-    clusterName: {{ .ClusterName }}
-    defaultInstanceProfile: KarpenterNodeInstanceProfile-{{ .ClusterName }}
-    interruptionQueueName: karpenter-{{ .ClusterName }}
+  clusterName: {{ .ClusterName }}
+  interruptionQueue: karpenter-{{ .ClusterName }}
   featureGates:
-    # -- driftEnabled is in ALPHA and is disabled by default. eksdemo enables it by default
-    # Setting driftEnabled to true enables the drift deprovisioner to watch for drift between currently deployed nodes
-    # and the desired state of nodes set in provisioners and node templates
-    driftEnabled: {{ not .DisableDrift }}
+    # -- drift is in ALPHA and is disabled by default. eksdemo enables it by default.
+    drift: {{ not .DisableDrift }}
 `
diff --git a/pkg/application/autoscaling/karpenter/node_role.go b/pkg/application/autoscaling/karpenter/node_role.go
@@ -34,13 +34,6 @@ Parameters:
     Description: "EKS cluster name"
     Default: "{{ .ClusterName }}"
 Resources:
-  KarpenterNodeInstanceProfile:
-    Type: "AWS::IAM::InstanceProfile"
-    Properties:
-      InstanceProfileName: !Sub "KarpenterNodeInstanceProfile-${ClusterName}"
-      Path: "/"
-      Roles:
-        - Ref: "KarpenterNodeRole"
   KarpenterNodeRole:
     Type: "AWS::IAM::Role"
     Properties:

diff --git a/pkg/application/autoscaling/karpenter/options.go b/pkg/application/autoscaling/karpenter/options.go
@@ -12,10 +12,10 @@ import (
 type KarpenterOptions struct {
 	application.ApplicationOptions
 
-	AMIFamily            string
-	DisableDrift         bool
-	Replicas             int
-	TTLSecondsAfterEmpty int
+	AMIFamily    string
+	DisableDrift bool
+	ExpireAfter  string
+	Replicas     int
 }
 
 func newOptions() (options *KarpenterOptions, flags cmd.Flags) {
@@ -24,14 +24,15 @@ func newOptions() (options *KarpenterOptions, flags cmd.Flags) {
 			Namespace:      "karpenter",
 			ServiceAccount: "karpenter",
 			DefaultVersion: &application.LatestPrevious{
-				LatestChart:   "v0.31.0",
-				Latest:        "v0.31.0",
-				PreviousChart: "v0.29.2",
-				Previous:      "v0.29.2",
+				LatestChart:   "v0.32.1",
+				Latest:        "v0.32.1",
+				PreviousChart: "v0.31.0",
+				Previous:      "v0.31.0",
 			},
 		},
-		AMIFamily: "AL2",
-		Replicas:  1,
+		AMIFamily:   "AL2",
+		ExpireAfter: "720h",
+		Replicas:    1,
 	}
 
 	flags = cmd.Flags{
@@ -66,20 +67,19 @@ func newOptions() (options *KarpenterOptions, flags cmd.Flags) {
 			},
 			Option: &options.DisableDrift,
 		},
-		&cmd.IntFlag{
+		&cmd.StringFlag{
 			CommandFlag: cmd.CommandFlag{
-				Name:        "replicas",
-				Description: "number of replicas for the controller deployment",
+				Name:        "expire-after",
+				Description: "duration the controller will wait before terminating a node",
 			},
-			Option: &options.Replicas,
+			Option: &options.ExpireAfter,
 		},
 		&cmd.IntFlag{
 			CommandFlag: cmd.CommandFlag{
-				Name:        "ttl-after-empty",
-				Description: "provisioner ttl seconds after empty (disables consolidation)",
-				Shorthand:   "T",
+				Name:        "replicas",
+				Description: "number of replicas for the controller deployment",
 			},
-			Option: &options.TTLSecondsAfterEmpty,
+			Option: &options.Replicas,
 		},
 	}
 	return