diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index d054c4e1..e2bce00e 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -11,6 +11,14 @@ jobs: name: Build & Run E2E Images runs-on: [self-hosted, linux, X64, jammy, large] steps: + - + name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + # We run into rate limiting issues if we don't authenticate + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Check out repo uses: actions/checkout@v4 - name: Install requirements @@ -22,29 +30,32 @@ jobs: sudo snap install kubectl --classic --channel=1.30/stable - name: Build provider images run: sudo make docker-build-e2e - - name: Build k8s-snap image + - name: Build k8s-snap images + working-directory: hack/ run: | - cd templates/docker - sudo docker build . -t k8s-snap:dev + ./build-e2e-images.sh - name: Save provider image run: | sudo docker save -o provider-images.tar ghcr.io/canonical/cluster-api-k8s/controlplane-controller:dev ghcr.io/canonical/cluster-api-k8s/bootstrap-controller:dev sudo chmod 775 provider-images.tar - name: Save k8s-snap image run: | - sudo docker save -o k8s-snap-image.tar k8s-snap:dev - sudo chmod 775 k8s-snap-image.tar + sudo docker save -o k8s-snap-image-old.tar k8s-snap:dev-old + sudo docker save -o k8s-snap-image-new.tar k8s-snap:dev-new + sudo chmod 775 k8s-snap-image-old.tar + sudo chmod 775 k8s-snap-image-new.tar - name: Upload artifacts uses: actions/upload-artifact@v4 with: name: e2e-images path: | provider-images.tar - k8s-snap-image.tar + k8s-snap-image-old.tar + k8s-snap-image-new.tar run-e2e-tests: name: Run E2E Tests - runs-on: [self-hosted, linux, X64, jammy, large] + runs-on: [self-hosted, linux, X64, jammy, xlarge] needs: build-e2e-images strategy: matrix: @@ -54,7 +65,17 @@ jobs: - "Workload cluster creation" - "Workload cluster scaling" - "Workload cluster upgrade" + # TODO(ben): Remove once all tests are running stable. + fail-fast: false steps: + - + name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + # We run into rate limiting issues if we don't authenticate + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Check out repo uses: actions/checkout@v4 - name: Install requirements @@ -71,8 +92,13 @@ jobs: path: . - name: Load provider image run: sudo docker load -i provider-images.tar - - name: Load k8s-snap image - run: sudo docker load -i k8s-snap-image.tar + - name: Load k8s-snap old image + if: matrix.ginkgo_focus == 'Workload cluster upgrade' + run: | + sudo docker load -i k8s-snap-image-old.tar + - name: Load k8s-snap new image + run: | + sudo docker load -i k8s-snap-image-new.tar - name: Create docker network run: | sudo docker network create kind --driver=bridge -o com.docker.network.bridge.enable_ip_masquerade=true diff --git a/hack/build-e2e-images.sh b/hack/build-e2e-images.sh new file mode 100755 index 00000000..85f12de0 --- /dev/null +++ b/hack/build-e2e-images.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# Description: +# Build k8s-snap docker images required for e2e tests. +# +# Usage: +# ./build-e2e-images.sh + +DIR="$(realpath "$(dirname "${0}")")" + +cd "${DIR}/../templates/docker" +sudo docker build . -t k8s-snap:dev-old --build-arg BRANCH=main --build-arg KUBERNETES_VERSION=v1.29.6 +sudo docker build . -t k8s-snap:dev-new --build-arg BRANCH=main --build-arg KUBERNETES_VERSION=v1.30.4 +cd - diff --git a/pkg/ck8s/workload_cluster.go b/pkg/ck8s/workload_cluster.go index 93668cc2..ebce977e 100644 --- a/pkg/ck8s/workload_cluster.go +++ b/pkg/ck8s/workload_cluster.go @@ -290,7 +290,7 @@ func (w *Workload) doK8sdRequest(ctx context.Context, method, endpoint string, r return fmt.Errorf("k8sd request failed: %s", responseBody.Error) } if responseBody.Metadata == nil || response == nil { - // Nothing to decode + // No response expected. return nil } if err := json.Unmarshal(responseBody.Metadata, response); err != nil { diff --git a/templates/docker/Dockerfile b/templates/docker/Dockerfile index 1a4c6e38..b6a3b08f 100644 --- a/templates/docker/Dockerfile +++ b/templates/docker/Dockerfile @@ -36,6 +36,8 @@ FROM $BUILD_BASE AS builder ARG REPO=https://github.com/canonical/k8s-snap ARG BRANCH=main +ARG KUBERNETES_VERSION="" + ## NOTE(neoaggelos): install dependencies needed to build the tools ## !!!IMPORTANT!!! Keep up to date with "snapcraft.yaml:parts.build-deps.build-packages" RUN apt-get update \ @@ -86,7 +88,12 @@ RUN /src/k8s-snap/build-scripts/build-component.sh helm ## kubernetes build FROM builder AS build-kubernetes -RUN /src/k8s-snap/build-scripts/build-component.sh kubernetes +ENV KUBERNETES_VERSION=${KUBERNETES_VERSION} +RUN if [ -n "$KUBERNETES_VERSION" ]; then \ + echo "Overwriting Kubernetes version with $KUBERNETES_VERSION"; \ + echo "$KUBERNETES_VERSION" > /src/k8s-snap/build-scripts/components/kubernetes/version; \ + fi +RUN /src/k8s-snap/build-scripts/build-component.sh kubernetes ## runc build FROM builder AS build-runc @@ -156,3 +163,7 @@ ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/k8s/ ## NOTE(neoaggelos): Required for containerd to properly set up overlayfs for pods VOLUME ["/var/snap/k8s/common/var/lib/containerd"] + +## NOTE(ben): Remove existing kind image kubectl and kubelet binaries +# to avoid version confusion. +RUN rm -f /usr/bin/kubectl /usr/bin/kubelet diff --git a/test/e2e/cluster_upgrade.go b/test/e2e/cluster_upgrade.go index 0a9ce02b..05ffd068 100644 --- a/test/e2e/cluster_upgrade.go +++ b/test/e2e/cluster_upgrade.go @@ -139,7 +139,7 @@ func ClusterUpgradeSpec(ctx context.Context, inputGetter func() ClusterUpgradeSp ClusterctlConfigPath: input.ClusterctlConfigPath, KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), InfrastructureProvider: *input.InfrastructureProvider, - Flavor: ptr.Deref(input.Flavor, ""), + Flavor: ptr.Deref(input.Flavor, "upgrades"), Namespace: namespace.Name, ClusterName: clusterName, KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion), @@ -158,6 +158,7 @@ func ClusterUpgradeSpec(ctx context.Context, inputGetter func() ClusterUpgradeSp Cluster: result.Cluster, ControlPlane: result.ControlPlane, KubernetesUpgradeVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo), + UpgradeMachineTemplate: ptr.To(fmt.Sprintf("%s-control-plane-old", clusterName)), WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"), }) @@ -167,13 +168,14 @@ func ClusterUpgradeSpec(ctx context.Context, inputGetter func() ClusterUpgradeSp Cluster: result.Cluster, UpgradeVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo), MachineDeployments: result.MachineDeployments, + UpgradeMachineTemplate: ptr.To(fmt.Sprintf("%s-md-new-0", clusterName)), WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), }) By("Waiting until nodes are ready") workloadProxy := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, result.Cluster.Name) workloadClient := workloadProxy.GetClient() - framework.WaitForNodesReady(ctx, framework.WaitForNodesReadyInput{ + WaitForNodesReady(ctx, WaitForNodesReadyInput{ Lister: workloadClient, KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo), Count: int(result.ExpectedTotalNodes()), diff --git a/test/e2e/cluster_upgrade_test.go b/test/e2e/cluster_upgrade_test.go index a058b65c..f0f1984d 100644 --- a/test/e2e/cluster_upgrade_test.go +++ b/test/e2e/cluster_upgrade_test.go @@ -25,13 +25,10 @@ import ( ) var _ = Describe("Workload cluster upgrade [CK8s-Upgrade]", func() { - BeforeEach(func() { - // TODO(bschimke): Remove once we find a way to run e2e tests with other infrastructure providers that support snap. - Skip("Skipping the upgrade tests as snap does not work on CAPD.") - }) - - Context("Upgrading a cluster with 1 control plane", func() { - ClusterUpgradeSpec(ctx, func() ClusterUpgradeSpecInput { + // Skipping this test as in-place upgrades are not supported yet. + // TODO(ben): Remove this skip when in-place upgrades are supported. + //Context("Upgrading a cluster with 1 control plane", func() { + /* ClusterUpgradeSpec(ctx, func() ClusterUpgradeSpecInput { return ClusterUpgradeSpecInput{ E2EConfig: e2eConfig, ClusterctlConfigPath: clusterctlConfigPath, @@ -42,8 +39,8 @@ var _ = Describe("Workload cluster upgrade [CK8s-Upgrade]", func() { ControlPlaneMachineCount: ptr.To[int64](1), WorkerMachineCount: ptr.To[int64](2), } - }) - }) + }) */ + //}) Context("Upgrading a cluster with HA control plane", func() { ClusterUpgradeSpec(ctx, func() ClusterUpgradeSpecInput { diff --git a/test/e2e/config/ck8s-docker.yaml b/test/e2e/config/ck8s-docker.yaml index a622c8c1..06134e12 100644 --- a/test/e2e/config/ck8s-docker.yaml +++ b/test/e2e/config/ck8s-docker.yaml @@ -51,9 +51,10 @@ providers: - old: "imagePullPolicy: Always" new: "imagePullPolicy: IfNotPresent" files: - - sourcePath: "../data/infrastructure-docker/cluster-template.yaml" - sourcePath: "../data/infrastructure-docker/cluster-template-kcp-remediation.yaml" - sourcePath: "../data/infrastructure-docker/cluster-template-md-remediation.yaml" + - sourcePath: "../data/infrastructure-docker/cluster-template-upgrades.yaml" + - sourcePath: "../data/infrastructure-docker/cluster-template.yaml" - name: ck8s type: BootstrapProvider versions: @@ -84,8 +85,8 @@ providers: variables: KUBERNETES_VERSION_MANAGEMENT: "v1.28.0" - KUBERNETES_VERSION: "v1.30.0" - KUBERNETES_VERSION_UPGRADE_TO: "v1.30.1" + KUBERNETES_VERSION: "v1.29.6" + KUBERNETES_VERSION_UPGRADE_TO: "v1.30.3" IP_FAMILY: "IPv4" KIND_IMAGE_VERSION: "v1.28.0" diff --git a/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml b/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml index 8c8cb43b..cbd88799 100644 --- a/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml +++ b/test/e2e/data/infrastructure-docker/cluster-template-kcp-remediation.yaml @@ -84,7 +84,7 @@ metadata: spec: template: spec: - customImage: k8s-snap:dev + customImage: k8s-snap:dev-old --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment @@ -127,7 +127,7 @@ metadata: spec: template: spec: - customImage: k8s-snap:dev + customImage: k8s-snap:dev-old --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 kind: CK8sConfigTemplate diff --git a/test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml b/test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml index 7b2bfe64..b2bf8e7e 100644 --- a/test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml +++ b/test/e2e/data/infrastructure-docker/cluster-template-md-remediation.yaml @@ -57,7 +57,7 @@ metadata: spec: template: spec: - customImage: k8s-snap:dev + customImage: k8s-snap:dev-old --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment @@ -101,7 +101,7 @@ metadata: spec: template: spec: - customImage: k8s-snap:dev + customImage: k8s-snap:dev-old --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 kind: CK8sConfigTemplate diff --git a/test/e2e/data/infrastructure-docker/cluster-template-upgrades.yaml b/test/e2e/data/infrastructure-docker/cluster-template-upgrades.yaml new file mode 100644 index 00000000..4a028727 --- /dev/null +++ b/test/e2e/data/infrastructure-docker/cluster-template-upgrades.yaml @@ -0,0 +1,137 @@ +# TODO: copied and modified from https://github.com/k3s-io/cluster-api-k3s/pull/93/files#diff-c4a336ec56832a2ff7aed26c94d0d67ae3a0e6139d30701cc53c0f0962fe8cca +# should be the same as samples/docker/quickstart.yaml in the future +# for testing the quickstart scenario +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +spec: + clusterNetwork: + pods: + cidrBlocks: + - 10.1.0.0/16 + services: + cidrBlocks: + - 10.152.0.0/16 + serviceDomain: cluster.local + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: CK8sControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerCluster +metadata: + name: ${CLUSTER_NAME} +spec: {} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: CK8sControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: ${NAMESPACE} +spec: + machineTemplate: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: ${CLUSTER_NAME}-control-plane-old + spec: + airGapped: true + controlPlane: + extraKubeAPIServerArgs: + --anonymous-auth: "true" + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +# Initial template for the control-plane deployment +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane-old + namespace: ${NAMESPACE} +spec: + template: + spec: + customImage: k8s-snap:dev-old +--- +# After upgrade template for the control plane deployment +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane-new + namespace: ${NAMESPACE} +spec: + template: + spec: + customImage: k8s-snap:dev-new +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: worker-md-0 + namespace: ${NAMESPACE} +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + + # This label will be needed for upgrade test + # it will be used as a selector for only selecting + # machines belonging to this machine deployment + cluster.x-k8s.io/deployment-name: worker-md-0 + template: + metadata: + labels: + cluster.x-k8s.io/deployment-name: worker-md-0 + spec: + version: ${KUBERNETES_VERSION} + clusterName: ${CLUSTER_NAME} + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: CK8sConfigTemplate + name: ${CLUSTER_NAME}-md-0 + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: ${CLUSTER_NAME}-md-old-0 +--- +# Initial template for the machine deployment +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-old-0 + namespace: ${NAMESPACE} +spec: + template: + spec: + customImage: k8s-snap:dev-old +--- +# After upgrade template for the machine deployment +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-new-0 + namespace: ${NAMESPACE} +spec: + template: + spec: + customImage: k8s-snap:dev-new +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: CK8sConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: ${NAMESPACE} +spec: + template: + spec: + airGapped: true diff --git a/test/e2e/data/infrastructure-docker/cluster-template.yaml b/test/e2e/data/infrastructure-docker/cluster-template.yaml index b33ac49d..dbb3136e 100644 --- a/test/e2e/data/infrastructure-docker/cluster-template.yaml +++ b/test/e2e/data/infrastructure-docker/cluster-template.yaml @@ -57,8 +57,7 @@ metadata: spec: template: spec: - # TODO: make this customable - customImage: k8s-snap:dev + customImage: k8s-snap:dev-old --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment @@ -101,7 +100,7 @@ metadata: spec: template: spec: - customImage: k8s-snap:dev + customImage: k8s-snap:dev-old --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 kind: CK8sConfigTemplate diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index 5c0cc56a..8c64ad84 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -558,6 +558,7 @@ type UpgradeControlPlaneAndWaitForUpgradeInput struct { Cluster *clusterv1.Cluster ControlPlane *controlplanev1.CK8sControlPlane KubernetesUpgradeVersion string + UpgradeMachineTemplate *string WaitForMachinesToBeUpgraded []interface{} } @@ -577,6 +578,17 @@ func UpgradeControlPlaneAndWaitForUpgrade(ctx context.Context, input UpgradeCont input.ControlPlane.Spec.Version = input.KubernetesUpgradeVersion + // Create a new ObjectReference for the infrastructure provider + newInfrastructureRef := corev1.ObjectReference{ + APIVersion: "infrastructure.cluster.x-k8s.io/v1beta1", + Kind: "DockerMachineTemplate", + Name: fmt.Sprintf("%s-control-plane-new", input.Cluster.Name), + Namespace: input.ControlPlane.Spec.MachineTemplate.InfrastructureRef.Namespace, + } + + // Update the infrastructureRef + input.ControlPlane.Spec.MachineTemplate.InfrastructureRef = newInfrastructureRef + Eventually(func() error { return patchHelper.Patch(ctx, input.ControlPlane) }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to patch the new kubernetes version to KCP %s", klog.KObj(input.ControlPlane)) @@ -590,6 +602,48 @@ func UpgradeControlPlaneAndWaitForUpgrade(ctx context.Context, input UpgradeCont }, input.WaitForMachinesToBeUpgraded...) } +// UpgradeMachineDeploymentsAndWait upgrades a machine deployment and waits for its machines to be upgraded. +func UpgradeMachineDeploymentsAndWait(ctx context.Context, input framework.UpgradeMachineDeploymentsAndWaitInput) { + Expect(ctx).NotTo(BeNil(), "ctx is required for UpgradeMachineDeploymentsAndWait") + Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.ClusterProxy can't be nil when calling UpgradeMachineDeploymentsAndWait") + Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling UpgradeMachineDeploymentsAndWait") + Expect(input.UpgradeVersion).ToNot(BeNil(), "Invalid argument. input.UpgradeVersion can't be nil when calling UpgradeMachineDeploymentsAndWait") + Expect(input.MachineDeployments).ToNot(BeEmpty(), "Invalid argument. input.MachineDeployments can't be empty when calling UpgradeMachineDeploymentsAndWait") + + mgmtClient := input.ClusterProxy.GetClient() + + for _, deployment := range input.MachineDeployments { + patchHelper, err := patch.NewHelper(deployment, mgmtClient) + Expect(err).ToNot(HaveOccurred()) + + oldVersion := deployment.Spec.Template.Spec.Version + deployment.Spec.Template.Spec.Version = &input.UpgradeVersion + // Create a new ObjectReference for the infrastructure provider + newInfrastructureRef := corev1.ObjectReference{ + APIVersion: "infrastructure.cluster.x-k8s.io/v1beta1", + Kind: "DockerMachineTemplate", + Name: fmt.Sprintf("%s-md-new-0", input.Cluster.Name), + Namespace: deployment.Spec.Template.Spec.InfrastructureRef.Namespace, + } + + // Update the infrastructureRef + deployment.Spec.Template.Spec.InfrastructureRef = newInfrastructureRef + Eventually(func() error { + return patchHelper.Patch(ctx, deployment) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to patch Kubernetes version on MachineDeployment %s", klog.KObj(deployment)) + + Byf("Waiting for Kubernetes versions of machines in MachineDeployment %s to be upgraded from %s to %s", + klog.KObj(deployment), *oldVersion, input.UpgradeVersion) + framework.WaitForMachineDeploymentMachinesToBeUpgraded(ctx, framework.WaitForMachineDeploymentMachinesToBeUpgradedInput{ + Lister: mgmtClient, + Cluster: input.Cluster, + MachineCount: int(*deployment.Spec.Replicas), + KubernetesUpgradeVersion: input.UpgradeVersion, + MachineDeployment: *deployment, + }, input.WaitForMachinesToBeUpgraded...) + } +} + type WaitForNodesReadyInput struct { Lister framework.Lister KubernetesVersion string @@ -607,14 +661,17 @@ func WaitForNodesReady(ctx context.Context, input WaitForNodesReadyInput) { } nodeReadyCount := 0 for _, node := range nodeList.Items { + fmt.Fprintf(GinkgoWriter, "KubeletVersions: %s, KubernetesVersion: %s\n", semver.MajorMinor(node.Status.NodeInfo.KubeletVersion), semver.MajorMinor(input.KubernetesVersion)) if !(semver.MajorMinor(node.Status.NodeInfo.KubeletVersion) == semver.MajorMinor(input.KubernetesVersion)) { return false, nil } + fmt.Fprintf(GinkgoWriter, "node %s is ready: %t\n", node.Name, noderefutil.IsNodeReady(&node)) if !noderefutil.IsNodeReady(&node) { return false, nil } nodeReadyCount++ } + fmt.Fprintf(GinkgoWriter, "nodeReadyCount: %d, expected count: %d\n", nodeReadyCount, input.Count) return input.Count == nodeReadyCount, nil }, input.WaitForNodesReady...).Should(BeTrue()) }