CAPI e2e tests (#13)

.github/workflows/e2e.yaml

@@ -0,0 +1,86 @@
+name: E2E Tests
+
+on:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  build-e2e-images:
+    name: Build & Run E2E Images
+    runs-on: [self-hosted, linux, X64, jammy, large]
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v4
+      - name: Install requirements
+        run: |
+          sudo apt update
+          sudo snap install go --classic --channel=1.22/stable
+          sudo apt install make
+          sudo apt install docker-buildx
+          sudo snap install kubectl --classic --channel=1.30/stable
+      - name: Build provider images
+        run: sudo make docker-build-e2e
+      - name: Build k8s-snap image
+        run: |
+          cd templates/docker
+          sudo docker build . -t k8s-snap:dev
+      - name: Save provider image
+        run: |
+          sudo docker save -o provider-images.tar ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev
+          sudo chmod 775 provider-images.tar
+      - name: Save k8s-snap image
+        run: |
+          sudo docker save -o k8s-snap-image.tar k8s-snap:dev
+          sudo chmod 775 k8s-snap-image.tar
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-images
+          path: |
+            provider-images.tar
+            k8s-snap-image.tar
+
+  run-e2e-tests:
+    name: Run E2E Tests
+    runs-on: [self-hosted, linux, X64, jammy, large]
+    needs: build-e2e-images
+    strategy:
+      matrix:
+        ginkgo_focus:
+          - "KCP remediation"
+          - "MachineDeployment remediation"
+          - "Workload cluster creation"
+          - "Workload cluster scaling"
+          - "Workload cluster upgrade"
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v4
+      - name: Install requirements
+        run: |
+          sudo apt update
+          sudo snap install go --classic --channel=1.22/stable
+          sudo apt install make
+          sudo apt install docker-buildx
+          sudo snap install kubectl --classic --channel=1.30/stable
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: e2e-images
+          path: .
+      - name: Load provider image
+        run: sudo docker load -i provider-images.tar
+      - name: Load k8s-snap image
+        run: sudo docker load -i k8s-snap-image.tar
+      - name: Create docker network
+        run: |
+          sudo docker network create kind --driver=bridge -o com.docker.network.bridge.enable_ip_masquerade=true
+      - name: Increase inotify watches
+        run: |
+          # Prevents https://cluster-api.sigs.k8s.io/user/troubleshooting#cluster-api-with-docker----too-many-open-files
+          sudo sysctl fs.inotify.max_user_watches=1048576
+          sudo sysctl fs.inotify.max_user_instances=8192
+      - name: Run e2e tests
+        run: |
+          sudo GINKGO_FOCUS="${{ matrix.ginkgo_focus }}" SKIP_RESOURCE_CLEANUP=true make test-e2e

.golangci.yml

@@ -90,6 +90,7 @@ linters-settings:
 
           - github.com/google/uuid
           - github.com/pkg/errors
+          - sigs.k8s.io/kind/pkg/errors
   gci:
     sections:
       - standard

Makefile

@@ -242,6 +242,9 @@ docker-build-bootstrap-%:
 	DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$* --build-arg package=./bootstrap/main.go --build-arg ldflags="$(LDFLAGS)" . -t ${BOOTSTRAP_IMG}:${BOOTSTRAP_IMG_TAG}-$*
 docker-build-bootstrap: manager-bootstrap docker-build-bootstrap-amd64 docker-build-bootstrap-arm64
 
+docker-build-bootstrap-e2e: manager-bootstrap
+	DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg package=./bootstrap/main.go --build-arg ldflags="$(LDFLAGS)" . -t ${BOOTSTRAP_IMG}:${BOOTSTRAP_IMG_TAG}
+
 # Push the bootstrap multiarch image
 .PHONY: docker-push-bootstrap
 docker-push-bootstrap-%: docker-build-bootstrap-%
@@ -266,8 +269,8 @@ test-controlplane: envtest generate-controlplane generate-controlplane-conversio
 docker-build-e2e: ## Run docker-build-* targets for all the images with settings to be used for the e2e tests
     # please ensure the generated image name matches image names used in the E2E_CONF_FILE
     # and it also match the image tags in bootstrap/config/default and controlplane/config/default
-	$(MAKE) BOOTSTRAP_IMG_TAG=dev docker-build-bootstrap
-	$(MAKE) CONTROLPLANE_IMG_TAG=dev docker-build-controlplane
+	$(MAKE) BOOTSTRAP_IMG_TAG=dev docker-build-bootstrap-e2e
+	$(MAKE) CONTROLPLANE_IMG_TAG=dev docker-build-controlplane-e2e
 
 .PHONY: test-e2e
 test-e2e: $(GINKGO) $(KUSTOMIZE) ## Run the end-to-end tests
@@ -326,6 +329,9 @@ docker-build-controlplane-%:
 	DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$* --build-arg package=./controlplane/main.go --build-arg ldflags="$(LDFLAGS)" . -t ${CONTROLPLANE_IMG}:${CONTROLPLANE_IMG_TAG}-$*
 docker-build-controlplane: manager-controlplane docker-build-controlplane-amd64 docker-build-controlplane-arm64
 
+docker-build-controlplane-e2e: manager-controlplane
+	DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=${ARCH} --build-arg package=./controlplane/main.go --build-arg ldflags="$(LDFLAGS)" . -t ${CONTROLPLANE_IMG}:${CONTROLPLANE_IMG_TAG}
+
 # Push the controlplane multiarch image
 .PHONY: docker-push-controlplane
 docker-push-controlplane-%: docker-build-controlplane-%

bootstrap/config/samples/bootstrap_v1alpha1_ck8sconfig.yaml

@@ -1,4 +1,4 @@
-apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
 kind: CK8sConfig
 metadata:
   name: ck8sconfig-sample

bootstrap/config/samples/bootstrap_v1alpha3_ck8sconfigtemplate.yaml

@@ -1,4 +1,4 @@
-apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
 kind: CK8sConfigTemplate
 metadata:
   name: ck8sconfigtemplate-sample

controlplane/config/default/kustomization.yaml

@@ -28,7 +28,7 @@ patchesStrategicMerge:
   # Protect the /metrics endpoint by putting it behind auth.
   # If you want your controller-manager to expose the /metrics
   # endpoint w/o any authn/z, please comment the following line.
-- manager_auth_proxy_patch.yaml
+#- manager_auth_proxy_patch.yaml
 
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml

controlplane/config/samples/controlplane_v1alpha3_ck8scontrolplane.yaml

@@ -1,4 +1,4 @@
-apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+apiVersion: controlplane.cluster.x-k8s.io/v1beta2
 kind: CK8sControlPlane
 metadata:
   name: ck8scontrolplane-sample

docs/main.md

@@ -14,7 +14,7 @@ The default behaviour is to `snap install k8s` using the matching track (e.g. in
 You can override this behaviour by changing the default installation script by setting the following fields on the config template:
 
 ```yaml
-apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+apiVersion: controlplane.cluster.x-k8s.io/v1beta2
 kind: CK8sControlPlane
 metadata:
   name: ${CLUSTER_NAME}-control-plane
@@ -47,7 +47,7 @@ For airgap deployments, or environment you can specify `airGapped: true` to prev
 Any extra yaml files placed in `/capi/manifests` will be applied once on the cluster after bootstrapping. Files are applied in alphabetical order, so you can use this in case of dependencies. Example:
 
 ```yaml
-apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+apiVersion: controlplane.cluster.x-k8s.io/v1beta2
 kind: CK8sControlPlane
 metadata:
   name: ${CLUSTER_NAME}-control-plane

go.mod

@@ -115,6 +115,7 @@ require (
 	go.uber.org/zap v1.26.0 // indirect
 	golang.org/x/crypto v0.21.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
+	golang.org/x/mod v0.19.0
 	golang.org/x/net v0.23.0 // indirect
 	golang.org/x/oauth2 v0.18.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect

go.sum

@@ -337,6 +337,8 @@ golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqR
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8=
+golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=

test/e2e/README.md

@@ -1,27 +1,29 @@
 # e2e test
+
 The e2e test use the [Cluster API test framework](https://pkg.go.dev/sigs.k8s.io/cluster-api/test/framework?tab=doc) and use the [CAPD](https://github.com/kubernetes-sigs/cluster-api/tree/main/test/infrastructure/docker) as the infrastructure provider. Please make sure you have [Docker](https://docs.docker.com/install/) and [kind](https://kind.sigs.k8s.io/) installed.
 
-You could refer to the [Testing Cluster API](https://cluster-api.sigs.k8s.io/developer/testing) for more information.
+Refer to the [Testing Cluster API](https://cluster-api.sigs.k8s.io/developer/testing) for more information.
 
 ## Run the e2e test
+
 The e2e image will be built with tag `dev`. You should build the image first before running the test.
+
 ```shell
 make docker-build-e2e   # should be run everytime you change the controller code
 make test-e2e   # run all e2e tests
 ```
+
 ### Run a specific e2e test
+
 To run a specific e2e test, such as `[PR-Blocking]`, use the `GINKGO_FOCUS` environment variable as shown below:
+
 ```shell
 make GINKGO_FOCUS="\\[PR-Blocking\\]" test-e2e  # only run e2e test with `[PR-Blocking]` in its spec name
 ```
-### Run the e2e test with tilt
-It is quite useful to run the e2e test with [tilt](https://cluster-api.sigs.k8s.io/developer/tilt), so that you will not need to rebuild docker image with `make docker-build-e2e` everytime. Also you will not need to wait a new cluster creation and setup. If you have set up your tilt cluster and made the current context points to this cluster, you could run:
-```shell
-# running e2e for the cluster pointed by the current context
-make USE_EXISTING_CLUSTER=true test-e2e
-```
+
 ## Develop an e2e test
-You could refer to [Developing E2E tests](https://cluster-api.sigs.k8s.io/developer/e2e) for a complete guide for developing e2e tests.
+
+Refer to [Developing E2E tests](https://cluster-api.sigs.k8s.io/developer/e2e) for a complete guide for developing e2e tests.
 
 A guide for developing a ck8s e2e test:
 
@@ -31,4 +33,5 @@ A guide for developing a ck8s e2e test:
 * If reusing a [cluster-api test spec](https://github.com/kubernetes-sigs/cluster-api/tree/main/test/e2e), note that they assume the use of `KubeadmControlPlane`. For customization, copy code into `test/e2e/helpers.go`.
 
 ## Troubleshooting
+
 * [Cluster API with Docker - "too many open files".](https://cluster-api.sigs.k8s.io/user/troubleshooting.html?highlight=too%20many#cluster-api-with-docker----too-many-open-files)

test/e2e/cluster_upgrade_test.go

@@ -25,6 +25,11 @@ import (
 )
 
 var _ = Describe("Workload cluster upgrade [CK8s-Upgrade]", func() {
+	BeforeEach(func() {
+		// TODO(bschimke): Remove once we find a way to run e2e tests with other infrastructure providers that support snap.
+		Skip("Skipping the upgrade tests as snap does not work on CAPD.")
+	})
+
 	Context("Upgrading a cluster with 1 control plane", func() {
 		ClusterUpgradeSpec(ctx, func() ClusterUpgradeSpecInput {
 			return ClusterUpgradeSpecInput{

test/e2e/config/ck8s-docker.yaml

@@ -64,6 +64,9 @@ providers:
       # is modified
       - name: v0.1.99 # next; use manifest from source files
         value: "../../../bootstrap/config/default"
+        replacements:
+          - old: "ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:latest"
+            new: "ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev"
     files:
       - sourcePath: "../../../metadata.yaml"
         targetName: "metadata.yaml"
@@ -72,6 +75,9 @@ providers:
     versions:
       - name: v0.1.99 # next; use manifest from source files
         value: "../../../controlplane/config/default"
+        replacements:
+          - old: "ghcr.io/canonical/cluster-api-k8s/controlplane-controller:latest"
+            new: "ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev"
     files:
       - sourcePath: "../../../metadata.yaml"
         targetName: "metadata.yaml"

test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml

@@ -13,7 +13,7 @@ spec:
       - 10.46.0.0/16
     serviceDomain: cluster.local
   controlPlaneRef:
-    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+    apiVersion: controlplane.cluster.x-k8s.io/v1beta2
     kind: CK8sControlPlane
     name: ${CLUSTER_NAME}-control-plane
   infrastructureRef:
@@ -27,19 +27,24 @@ metadata:
   name: ${CLUSTER_NAME}
 spec: {}
 ---
-apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+apiVersion: controlplane.cluster.x-k8s.io/v1beta2
 kind: CK8sControlPlane
 metadata:
   name: ${CLUSTER_NAME}-control-plane
   namespace: ${NAMESPACE}
 spec:
-  infrastructureTemplate:
-    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
-    kind: DockerMachineTemplate
-    name: ${CLUSTER_NAME}-control-plane
   replicas: ${CONTROL_PLANE_MACHINE_COUNT}
   version: ${KUBERNETES_VERSION}
+  machineTemplate:
+    infrastructureTemplate:
+      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+      kind: DockerMachineTemplate
+      name: ${CLUSTER_NAME}-control-plane
   spec:
+    airGapped: true
+    controlPlane:
+      extraKubeAPIServerArgs:
+        --anonymous-auth: "true"
     files:
     - path: /wait-signal.sh
       content: |
@@ -62,19 +67,14 @@ spec:
           echo "signal $signal"
 
           if [ "$signal" == "pass" ]; then
-             curl -k -s --header "Authorization: Bearer $TOKEN" -XPATCH -H "Content-Type: application/strategic-merge-patch+json" --data '{"data": {"signal": "ack-pass"}}' $SERVER/api/v1/namespaces/$NAMESPACE/configmaps/mhc-test
-             exit 0
+            curl -k -s --header "Authorization: Bearer $TOKEN" -XPATCH -H "Content-Type: application/strategic-merge-patch+json" --data '{"data": {"signal": "ack-pass"}}' $SERVER/api/v1/namespaces/$NAMESPACE/configmaps/mhc-test
+            exit 0
           fi
         done
       permissions: "0777"
-    preK3sCommands:
+      owner: root:root
+    preRunCommands:
     - ./wait-signal.sh "${TOKEN}" "${SERVER}" "${NAMESPACE}"
-    serverConfig:
-      tlsSan:
-        - localhost
-        - 127.0.0.1
-        - 0.0.0.0
-        - host.docker.internal
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
 kind: DockerMachineTemplate
@@ -84,7 +84,7 @@ metadata:
 spec:
   template:
     spec:
-      customImage: kindest/node:${KIND_IMAGE_VERSION}
+      customImage: k8s-snap:dev
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: MachineDeployment
@@ -111,7 +111,7 @@ spec:
       clusterName: ${CLUSTER_NAME}
       bootstrap:
         configRef:
-          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
           kind: CK8sConfigTemplate
           name: ${CLUSTER_NAME}-md-0
       infrastructureRef:
@@ -127,9 +127,9 @@ metadata:
 spec:
   template:
     spec:
-      customImage: kindest/node:${KIND_IMAGE_VERSION}
+      customImage: k8s-snap:dev
 ---
-apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
 kind: CK8sConfigTemplate
 metadata:
   name: ${CLUSTER_NAME}-md-0