diff --git a/pkg/application/autoscaling/karpenter/default_nodepool.go b/pkg/application/autoscaling/karpenter/default_nodepool.go new file mode 100644 index 0000000..d23c31d --- /dev/null +++ b/pkg/application/autoscaling/karpenter/default_nodepool.go @@ -0,0 +1,77 @@ +package karpenter + +import ( + "github.com/awslabs/eksdemo/pkg/manifest" + "github.com/awslabs/eksdemo/pkg/resource" + "github.com/awslabs/eksdemo/pkg/template" +) + +type NodePoolOptions struct { + resource.CommonOptions + *KarpenterOptions +} + +func karpenterDefaultNodePool(o *KarpenterOptions) *resource.Resource { + res := &resource.Resource{ + Options: &NodePoolOptions{ + CommonOptions: resource.CommonOptions{ + Name: "karpenter-default-nodepool", + }, + KarpenterOptions: o, + }, + + Manager: &manifest.ResourceManager{ + Template: &template.TextTemplate{ + Template: yamlTemplate, + }, + }, + } + return res +} + +const yamlTemplate = `--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand", "spot"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] + nodeClassRef: + name: default + limits: + cpu: 1000 + disruption: + consolidationPolicy: WhenUnderutilized + expireAfter: {{ .ExpireAfter }} +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: default +spec: + amiFamily: {{ .AMIFamily }} + role: KarpenterNodeRole-{{ .ClusterName }} + subnetSelectorTerms: + - tags: + Name: eksctl-{{ .ClusterName }}-cluster/SubnetPrivate* + securityGroupSelectorTerms: + - tags: + aws:eks:cluster-name: {{ .ClusterName }} +` diff --git a/pkg/application/autoscaling/karpenter/default_provisioner.go b/pkg/application/autoscaling/karpenter/default_provisioner.go deleted file mode 100644 index bd33444..0000000 --- a/pkg/application/autoscaling/karpenter/default_provisioner.go +++ /dev/null @@ -1,64 +0,0 @@ -package karpenter - -import ( - "github.com/awslabs/eksdemo/pkg/manifest" - "github.com/awslabs/eksdemo/pkg/resource" - "github.com/awslabs/eksdemo/pkg/template" -) - -type ProvisionerOptions struct { - resource.CommonOptions - *KarpenterOptions -} - -func karpenterDefaultProvisioner(o *KarpenterOptions) *resource.Resource { - res := &resource.Resource{ - Options: &ProvisionerOptions{ - CommonOptions: resource.CommonOptions{ - Name: "karpenter-default-provisioner", - }, - KarpenterOptions: o, - }, - - Manager: &manifest.ResourceManager{ - Template: &template.TextTemplate{ - Template: yamlTemplate, - }, - }, - } - return res -} - -const yamlTemplate = `--- -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner -metadata: - name: default -spec: - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["on-demand", "spot"] - limits: - resources: - cpu: 1000 - providerRef: - name: default -{{- if .TTLSecondsAfterEmpty }} - ttlSecondsAfterEmpty: {{ .TTLSecondsAfterEmpty }} -{{- else }} - consolidation: - enabled: true -{{- end }} ---- -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -metadata: - name: default -spec: - amiFamily: {{ .AMIFamily }} - subnetSelector: - Name: eksctl-{{ .ClusterName }}-cluster/SubnetPrivate* - securityGroupSelector: - aws:eks:cluster-name: {{ .ClusterName }} -` diff --git a/pkg/application/autoscaling/karpenter/karpenter.go b/pkg/application/autoscaling/karpenter/karpenter.go index c1b6766..7bde83d 100644 --- a/pkg/application/autoscaling/karpenter/karpenter.go +++ b/pkg/application/autoscaling/karpenter/karpenter.go @@ -15,7 +15,7 @@ import ( // GitHub: https://github.com/awslabs/karpenter // Helm: https://github.com/awslabs/karpenter/tree/main/charts/karpenter // Repo: https://gallery.ecr.aws/karpenter/controller -// Version: Latest is v0.31.0 (as of 10/3/23) +// Version: Latest is v0.32.1 (as of 11/1/23) func NewApp() *application.Application { options, flags := newOptions() @@ -69,7 +69,7 @@ func NewApp() *application.Application { }, PostInstallResources: []*resource.Resource{ - karpenterDefaultProvisioner(options), + karpenterDefaultNodePool(options), }, } app.Options = options @@ -109,7 +109,7 @@ Statement: StringEquals: aws:RequestTag/kubernetes.io/cluster/{{ .ClusterName }}: owned StringLike: - aws:RequestTag/karpenter.sh/provisioner-name: "*" + aws:RequestTag/karpenter.sh/nodepool: "*" - Sid: AllowScopedResourceCreationTagging Effect: Allow Resource: @@ -127,21 +127,20 @@ Statement: - CreateFleet - CreateLaunchTemplate StringLike: - aws:RequestTag/karpenter.sh/provisioner-name: "*" -- Sid: AllowMachineMigrationTagging + aws:RequestTag/karpenter.sh/nodepool: "*" +- Sid: AllowScopedResourceTagging Effect: Allow Resource: arn:{{ .Partition }}:ec2:{{ .Region }}:*:instance/* Action: ec2:CreateTags Condition: StringEquals: aws:ResourceTag/kubernetes.io/cluster/{{ .ClusterName }}: owned - aws:RequestTag/karpenter.sh/managed-by: "{{ .ClusterName }}" StringLike: - aws:RequestTag/karpenter.sh/provisioner-name: "*" + aws:ResourceTag/karpenter.sh/nodepool: "*" ForAllValues:StringEquals: aws:TagKeys: - - karpenter.sh/provisioner-name - - karpenter.sh/managed-by + - karpenter.sh/nodeclaim + - Name - Sid: AllowScopedDeletion Effect: Allow Resource: @@ -154,7 +153,7 @@ Statement: StringEquals: aws:ResourceTag/kubernetes.io/cluster/{{ .ClusterName }}: owned StringLike: - aws:ResourceTag/karpenter.sh/provisioner-name: "*" + aws:ResourceTag/karpenter.sh/nodepool: "*" - Sid: AllowRegionalReadActions Effect: Allow Resource: "*" @@ -196,6 +195,48 @@ Statement: Condition: StringEquals: iam:PassedToService: ec2.amazonaws.com +- Sid: AllowScopedInstanceProfileCreationActions + Effect: Allow + Resource: "*" + Action: + - iam:CreateInstanceProfile + Condition: + StringEquals: + aws:RequestTag/kubernetes.io/cluster/{{ .ClusterName }}: owned + aws:RequestTag/topology.kubernetes.io/region: "{{ .Region }}" + StringLike: + aws:RequestTag/karpenter.k8s.aws/ec2nodeclass: "*" +- Sid: AllowScopedInstanceProfileTagActions + Effect: Allow + Resource: "*" + Action: + - iam:TagInstanceProfile + Condition: + StringEquals: + aws:ResourceTag/kubernetes.io/cluster/{{ .ClusterName }}: owned + aws:ResourceTag/topology.kubernetes.io/region: "{{ .Region }}" + aws:RequestTag/kubernetes.io/cluster/{{ .ClusterName }}: owned + aws:RequestTag/topology.kubernetes.io/region: "{{ .Region }}" + StringLike: + aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass: "*" + aws:RequestTag/karpenter.k8s.aws/ec2nodeclass: "*" +- Sid: AllowScopedInstanceProfileActions + Effect: Allow + Resource: "*" + Action: + - iam:AddRoleToInstanceProfile + - iam:RemoveRoleFromInstanceProfile + - iam:DeleteInstanceProfile + Condition: + StringEquals: + aws:ResourceTag/kubernetes.io/cluster/{{ .ClusterName }}: owned + aws:ResourceTag/topology.kubernetes.io/region: "{{ .Region }}" + StringLike: + aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass: "*" +- Sid: AllowInstanceProfileReadActions + Effect: Allow + Resource: "*" + Action: iam:GetInstanceProfile - Sid: AllowAPIServerEndpointDiscovery Effect: Allow Resource: arn:{{ .Partition }}:eks:{{ .Region }}:{{ .Account }}:cluster/{{ .ClusterName }} @@ -212,14 +253,14 @@ replicas: {{ .Replicas }} controller: image: tag: {{ .Version }} + resources: + requests: + cpu: "1" + memory: "1Gi" settings: - aws: - clusterName: {{ .ClusterName }} - defaultInstanceProfile: KarpenterNodeInstanceProfile-{{ .ClusterName }} - interruptionQueueName: karpenter-{{ .ClusterName }} + clusterName: {{ .ClusterName }} + interruptionQueue: karpenter-{{ .ClusterName }} featureGates: - # -- driftEnabled is in ALPHA and is disabled by default. eksdemo enables it by default - # Setting driftEnabled to true enables the drift deprovisioner to watch for drift between currently deployed nodes - # and the desired state of nodes set in provisioners and node templates - driftEnabled: {{ not .DisableDrift }} + # -- drift is in ALPHA and is disabled by default. eksdemo enables it by default. + drift: {{ not .DisableDrift }} ` diff --git a/pkg/application/autoscaling/karpenter/node_role.go b/pkg/application/autoscaling/karpenter/node_role.go index 84adc0a..7149320 100644 --- a/pkg/application/autoscaling/karpenter/node_role.go +++ b/pkg/application/autoscaling/karpenter/node_role.go @@ -34,13 +34,6 @@ Parameters: Description: "EKS cluster name" Default: "{{ .ClusterName }}" Resources: - KarpenterNodeInstanceProfile: - Type: "AWS::IAM::InstanceProfile" - Properties: - InstanceProfileName: !Sub "KarpenterNodeInstanceProfile-${ClusterName}" - Path: "/" - Roles: - - Ref: "KarpenterNodeRole" KarpenterNodeRole: Type: "AWS::IAM::Role" Properties: diff --git a/pkg/application/autoscaling/karpenter/options.go b/pkg/application/autoscaling/karpenter/options.go index 6bce4bf..f72180f 100644 --- a/pkg/application/autoscaling/karpenter/options.go +++ b/pkg/application/autoscaling/karpenter/options.go @@ -12,10 +12,10 @@ import ( type KarpenterOptions struct { application.ApplicationOptions - AMIFamily string - DisableDrift bool - Replicas int - TTLSecondsAfterEmpty int + AMIFamily string + DisableDrift bool + ExpireAfter string + Replicas int } func newOptions() (options *KarpenterOptions, flags cmd.Flags) { @@ -24,14 +24,15 @@ func newOptions() (options *KarpenterOptions, flags cmd.Flags) { Namespace: "karpenter", ServiceAccount: "karpenter", DefaultVersion: &application.LatestPrevious{ - LatestChart: "v0.31.0", - Latest: "v0.31.0", - PreviousChart: "v0.29.2", - Previous: "v0.29.2", + LatestChart: "v0.32.1", + Latest: "v0.32.1", + PreviousChart: "v0.31.0", + Previous: "v0.31.0", }, }, - AMIFamily: "AL2", - Replicas: 1, + AMIFamily: "AL2", + ExpireAfter: "720h", + Replicas: 1, } flags = cmd.Flags{ @@ -66,20 +67,19 @@ func newOptions() (options *KarpenterOptions, flags cmd.Flags) { }, Option: &options.DisableDrift, }, - &cmd.IntFlag{ + &cmd.StringFlag{ CommandFlag: cmd.CommandFlag{ - Name: "replicas", - Description: "number of replicas for the controller deployment", + Name: "expire-after", + Description: "duration the controller will wait before terminating a node", }, - Option: &options.Replicas, + Option: &options.ExpireAfter, }, &cmd.IntFlag{ CommandFlag: cmd.CommandFlag{ - Name: "ttl-after-empty", - Description: "provisioner ttl seconds after empty (disables consolidation)", - Shorthand: "T", + Name: "replicas", + Description: "number of replicas for the controller deployment", }, - Option: &options.TTLSecondsAfterEmpty, + Option: &options.Replicas, }, } return