[kwokctl] Integrate metrics-server

.github/workflows/test.yaml

@@ -177,22 +177,58 @@ jobs:
         run: |
           export GOBIN="$(go env GOPATH)/bin"
           cd "${GITHUB_WORKSPACE}/kubernetes" && go install ./cmd/{kube-apiserver,kube-controller-manager,kube-scheduler}
-          mkdir -p "${GITHUB_WORKSPACE//\\//}/workdir" && cat >"${GITHUB_WORKSPACE//\\//}/workdir/kwok.yaml" << EOF
+          mkdir -p "${GITHUB_WORKSPACE//\\//}/workdir" && cat >>"${GITHUB_WORKSPACE//\\//}/workdir/kwok.yaml" << EOF
           kind: KwokctlConfiguration
           apiVersion: config.kwok.x-k8s.io/v1alpha1
           options:
             kubeBinaryPrefix: "${GOBIN//\\//}"
+          ---
           EOF
       - name: Build Kubernetes Binary for MacOS
         if: ${{ matrix.kwokctl-runtime == 'binary' && matrix.os == 'macos-latest' }}
         shell: bash
         run: |
           cd "${GITHUB_WORKSPACE}/kubernetes" && make WHAT="cmd/kube-apiserver cmd/kube-controller-manager cmd/kube-scheduler"
-          mkdir -p "${GITHUB_WORKSPACE}/workdir" && cat >"${GITHUB_WORKSPACE}/workdir/kwok.yaml" << EOF
+          mkdir -p "${GITHUB_WORKSPACE}/workdir" && cat >>"${GITHUB_WORKSPACE}/workdir/kwok.yaml" << EOF
           kind: KwokctlConfiguration
           apiVersion: config.kwok.x-k8s.io/v1alpha1
           options:
             kubeBinaryPrefix: "${GITHUB_WORKSPACE}/kubernetes/_output/bin"
+          ---
+          EOF
+
+        # Build metrics-server binary
+      - name: Download Metrics Server Source Code
+        uses: actions/checkout@v3
+        if: ${{ matrix.kwokctl-runtime == 'binary' }}
+        with:
+          repository: kubernetes-sigs/metrics-server
+          path: metrics-server
+          ref: v0.6.3
+      - name: Build Metrics Server Binary for Linux and MacOS
+        if: ${{ matrix.kwokctl-runtime == 'binary' && matrix.os != 'windows-latest' }}
+        shell: bash
+        run: |
+          cd "${GITHUB_WORKSPACE}/metrics-server" && make metrics-server
+          mkdir -p "${GITHUB_WORKSPACE}/workdir" && cat >>"${GITHUB_WORKSPACE}/workdir/kwok.yaml" << EOF
+          kind: KwokctlConfiguration
+          apiVersion: config.kwok.x-k8s.io/v1alpha1
+          options:
+            metricsServerBinary: "${GITHUB_WORKSPACE}/metrics-server/metrics-server"
+          ---
+          EOF
+      - name: Build Metrics Server Binary for Windows
+        if: ${{ matrix.kwokctl-runtime == 'binary' && matrix.os == 'windows-latest' }}
+        shell: bash
+        run: |
+          export GOBIN="$(go env GOPATH)/bin"
+          cd "${GITHUB_WORKSPACE}/metrics-server" && go install ./cmd/metrics-server
+          mkdir -p "${GITHUB_WORKSPACE//\\//}/workdir" && cat >>"${GITHUB_WORKSPACE//\\//}/workdir/kwok.yaml" << EOF
+          kind: KwokctlConfiguration
+          apiVersion: config.kwok.x-k8s.io/v1alpha1
+          options:
+            metricsServerBinary: "${GOBIN//\\//}/metrics-server.exe"
+          ---
           EOF
 
         # TODO: workaround for https://github.com/actions/runner-images/issues/7753 (caused by https://bugs.launchpad.net/ubuntu/+source/libpod/+bug/2024394).

kustomize/kwokctl/resource/node.yaml

@@ -20,6 +20,7 @@ template: |-
     annotations:
       kwok.x-k8s.io/node: fake
       node.alpha.kubernetes.io/ttl: "0"
+      metrics.k8s.io/resource-metrics-path: "/metrics/nodes/{{ Name }}/metrics/resource"
     labels:
       beta.kubernetes.io/arch: {{ .nodeInfo.architecture }}
       beta.kubernetes.io/os: {{ .nodeInfo.operatingSystem }}

pkg/apis/config/v1alpha1/kwokctl_configuration_types.go

@@ -123,6 +123,9 @@ type KwokctlConfigurationOptions struct {
 	// is the default value for env KWOK_JAEGER_VERSION
 	JaegerVersion string `json:"jaegerVersion,omitempty"`
 
+	// MetricsServerVersion is the version of metrics-server to use.
+	MetricsServerVersion string `json:"metricsServerVersion,omitempty"`
+
 	// DockerComposeVersion is the version of docker-compose to use.
 	// is the default value for env KWOK_DOCKER_COMPOSE_VERSION
 	// Deprecated: docker compose will be removed in a future release
@@ -156,6 +159,10 @@ type KwokctlConfigurationOptions struct {
 	// +default=false
 	DisableKubeControllerManager *bool `json:"disableKubeControllerManager,omitempty"`
 
+	// EnableMetricsServer is the flag to enable metrics-server.
+	// +default=false
+	EnableMetricsServer *bool `json:"enableMetricsServer,omitempty"`
+
 	// KubeImagePrefix is the prefix of the kubernetes image.
 	// is the default value for env KWOK_KUBE_IMAGE_PREFIX
 	//+k8s:conversion-gen=false
@@ -185,6 +192,10 @@ type KwokctlConfigurationOptions struct {
 	//+k8s:conversion-gen=false
 	JaegerImagePrefix string `json:"jaegerImagePrefix,omitempty"`
 
+	// MetricsServerImagePrefix is the prefix of the metrics-server image.
+	//+k8s:conversion-gen=false
+	MetricsServerImagePrefix string `json:"metricsServerImagePrefix,omitempty"`
+
 	// EtcdImage is the image of etcd.
 	// is the default value for flag --etcd-image and env KWOK_ETCD_IMAGE
 	EtcdImage string `json:"etcdImage,omitempty"`
@@ -216,6 +227,9 @@ type KwokctlConfigurationOptions struct {
 	// is the default value for flag --jaeger-image and env KWOK_JAEGER_IMAGE
 	JaegerImage string `json:"jaegerImage,omitempty"`
 
+	// MetricsServerImage is the image of metrics-server.
+	MetricsServerImage string `json:"metricsServerImage,omitempty"`
+
 	// KindNodeImagePrefix is the prefix of the kind node image.
 	// is the default value for env KWOK_KIND_NODE_IMAGE_PREFIX
 	//+k8s:conversion-gen=false
@@ -306,6 +320,13 @@ type KwokctlConfigurationOptions struct {
 	// is the default value for env KWOK_JAEGER_TAR
 	JaegerBinaryTar string `json:"jaegerBinaryTar,omitempty"`
 
+	// MetricsServerBinaryPrefix is the prefix of the metrics-server binary.
+	//+k8s:conversion-gen=false
+	MetricsServerBinaryPrefix string `json:"metricsServerBinaryPrefix,omitempty"`
+
+	// MetricsServerBinary is the binary of metrics-server.
+	MetricsServerBinary string `json:"metricsServerBinary,omitempty"`
+
 	// DockerComposeBinaryPrefix is the binary of docker-compose.
 	// is the default value for env KWOK_DOCKER_COMPOSE_BINARY_PREFIX
 	// Deprecated: docker compose will be removed in a future release
@@ -369,6 +390,9 @@ type KwokctlConfigurationOptions struct {
 	// is the default value for flag --controller-port and env KWOK_CONTROLLER_PORT
 	KwokControllerPort uint32 `json:"kwokControllerPort,omitempty"`
 
+	// MetricsServerPort is metrics-server port that is exposed to the host.
+	MetricsServerPort uint32 `json:"metricsServerPort,omitempty"`
+
 	// CacheDir is the directory of the cache.
 	CacheDir string `json:"cacheDir,omitempty"`
 

pkg/apis/internalversion/kwokctl_configuration_types.go

@@ -101,6 +101,9 @@ type KwokctlConfigurationOptions struct {
 	// JaegerVersion is the version of Jaeger to use.
 	JaegerVersion string
 
+	// MetricsServerVersion is the version of metrics-server to use.
+	MetricsServerVersion string
+
 	// DockerComposeVersion is the version of docker-compose to use.
 	DockerComposeVersion string
 
@@ -123,6 +126,9 @@ type KwokctlConfigurationOptions struct {
 	// DisableKubeControllerManager is the flag to disable kube-controller-manager.
 	DisableKubeControllerManager bool
 
+	// EnableMetricsServer is the flag to enable metrics-server.
+	EnableMetricsServer bool
+
 	// EtcdImage is the image of etcd.
 	EtcdImage string
 
@@ -147,6 +153,9 @@ type KwokctlConfigurationOptions struct {
 	// JaegerImage is the image of Jaeger
 	JaegerImage string
 
+	// MetricsServerImage is the image of metrics-server.
+	MetricsServerImage string
+
 	// KindNodeImage is the image of kind node.
 	KindNodeImage string
 
@@ -191,6 +200,9 @@ type KwokctlConfigurationOptions struct {
 	// JaegerBinaryTar is the tar of binary of Jaeger.
 	JaegerBinaryTar string
 
+	// MetricsServerBinary is the binary of metrics-server.
+	MetricsServerBinary string
+
 	// DockerComposeBinary is the binary of Docker compose.
 	DockerComposeBinary string
 
@@ -233,6 +245,9 @@ type KwokctlConfigurationOptions struct {
 	// KwokControllerPort is kwok-controller port that is exposed to the host.
 	KwokControllerPort uint32
 
+	// MetricsServerPort is metrics-server port that is exposed to the host.
+	MetricsServerPort uint32
+
 	// CacheDir is the directory of the cache.
 	CacheDir string
 

pkg/config/vars.go

@@ -246,6 +246,8 @@ func setKwokctlConfigurationDefaults(config *configv1alpha1.KwokctlConfiguration
 
 	setKwokctlJaegerConfig(conf)
 
+	setMetricsServerConfig(conf)
+
 	return config
 }
 
@@ -527,6 +529,35 @@ func setKwokctlJaegerConfig(conf *configv1alpha1.KwokctlConfigurationOptions) {
 	conf.JaegerBinaryTar = envs.GetEnvWithPrefix("JAEGER_BINARY_TAR", conf.JaegerBinaryTar)
 }
 
+func setMetricsServerConfig(conf *configv1alpha1.KwokctlConfigurationOptions) {
+	if conf.MetricsServerVersion == "" {
+		conf.MetricsServerVersion = consts.MetricsServerVersion
+	}
+	conf.MetricsServerVersion = version.AddPrefixV(envs.GetEnvWithPrefix("METRICS_SERVER_VERSION", conf.MetricsServerVersion))
+
+	if conf.MetricsServerImagePrefix == "" {
+		conf.MetricsServerImagePrefix = consts.MetricsServerImagePrefix
+	}
+	conf.MetricsServerImagePrefix = envs.GetEnvWithPrefix("METRICS_SERVER_IMAGE_PREFIX", conf.MetricsServerImagePrefix)
+
+	if conf.MetricsServerImage == "" {
+		conf.MetricsServerImage = joinImageURI(conf.MetricsServerImagePrefix, "metrics-server", version.AddPrefixV(conf.MetricsServerVersion))
+	}
+	conf.MetricsServerImage = envs.GetEnvWithPrefix("METRICS_SERVER_IMAGE", conf.MetricsServerImage)
+
+	// TODO: add metrics server binary prefix
+	// if conf.MetricsServerBinaryPrefix == "" {
+	// 	conf.MetricsServerBinaryPrefix = consts.MetricsServerBinaryPrefix + "/" + conf.MetricsServerVersion
+	// }
+	conf.MetricsServerBinaryPrefix = envs.GetEnvWithPrefix("METRICS_SERVER_BINARY_PREFIX", conf.MetricsServerBinaryPrefix)
+
+	if conf.MetricsServerBinaryPrefix != "" &&
+		conf.MetricsServerBinary == "" {
+		conf.MetricsServerBinary = conf.MetricsServerBinaryPrefix + "/metrics-server-" + GOOS + "-" + GOARCH + conf.BinSuffix
+	}
+	conf.MetricsServerBinary = envs.GetEnvWithPrefix("METRICS_SERVER_BINARY", conf.MetricsServerBinary)
+}
+
 // joinImageURI joins the image URI.
 func joinImageURI(prefix, name, version string) string {
 	return prefix + "/" + name + ":" + version

pkg/consts/consts.go

@@ -60,6 +60,10 @@ var (
 	JaegerBinaryPrefix = "https://github.com/jaegertracing/jaeger/releases/download"
 	JaegerImagePrefix  = "docker.io/jaegertracing"
 
+	MetricsServerVersion      = "0.6.3"
+	MetricsServerBinaryPrefix = ""
+	MetricsServerImagePrefix  = "registry.k8s.io/metrics-server"
+
 	KindNodeImagePrefix = "docker.io/kindest"
 
 	DefaultUnlimitedQPS   = 5000.0