feat(disk, diskutil): Add new metric type disk and diskutil

api/shared_definitions.yaml

@@ -28,6 +28,6 @@ schemas:
   metric_type:
     description: |
       One user-defined custom metric type or one of the system-default metric types, which are:
-      "memoryused", "memoryutil", "responsetime", "throughput", "cpu" and "cpuutil"
+      "memoryused", "memoryutil", "responsetime", "throughput", "cpu", "cpuutil", "disk" and "diskutil"
     type: string
     example: memoryused

jobs/golangapiserver/spec

@@ -156,6 +156,18 @@ properties:
   autoscaler.apiserver.scaling_rules.cpuutil.upper_threshold:
     description: "Allowable upper threshold of the cpuutil scaling range"
     default: 100
+  autoscaler.apiserver.scaling_rules.diskutil.lower_threshold:
+    description: "Allowable lower threshold of the diskutil scaling range"
+    default: 1
+  autoscaler.apiserver.scaling_rules.diskutil.upper_threshold:
+    description: "Allowable upper threshold of the diskutil scaling range"
+    default: 100
+  autoscaler.apiserver.scaling_rules.disk.lower_threshold:
+    description: "Allowable lower threshold of the disk scaling range"
+    default: 1
+  autoscaler.apiserver.scaling_rules.disk.upper_threshold:
+    description: "Allowable upper threshold of the disk scaling range"
+    default: 2048 # same default as the maximum app disk size maintained in the cloud controller API release https://github.com/cloudfoundry/capi-release/blob/dd94bda54387eb68a73a01dcb1c1f0102ebcf7b3/jobs/cc_deployment_updater/spec#L200-L201
   autoscaler.policy_db.address:
     description: "IP address on which the policydb server will listen"
     default: "autoscalerpostgres.service.cf.internal"

jobs/golangapiserver/templates/apiserver.yml.erb

@@ -158,6 +158,12 @@ scaling_rules:
   cpuutil:
     lower_threshold: <%= p("autoscaler.apiserver.scaling_rules.cpuutil.lower_threshold") %>
     upper_threshold: <%= p("autoscaler.apiserver.scaling_rules.cpuutil.upper_threshold") %>
+  diskutil:
+    lower_threshold: <%= p("autoscaler.apiserver.scaling_rules.diskutil.lower_threshold") %>
+    upper_threshold: <%= p("autoscaler.apiserver.scaling_rules.diskutil.upper_threshold") %>
+  disk:
+    lower_threshold: <%= p("autoscaler.apiserver.scaling_rules.disk.lower_threshold") %>
+    upper_threshold: <%= p("autoscaler.apiserver.scaling_rules.disk.upper_threshold") %>
 
 rate_limit:
   valid_duration: <%= p("autoscaler.apiserver.rate_limit.valid_duration") %>

src/acceptance/app/dynamic_policy_test.go

@@ -244,12 +244,12 @@ var _ = Describe("AutoScaler dynamic policy", func() {
 
 		It("when cpu is greater than scaling out threshold", func() {
 			By("should scale out to 2 instances")
-			AppSetCpuUsage(cfg, appName, int(float64(cfg.CPUUpperThreshold)*0.9), 5)
+			StartCPUUsage(cfg, appName, int(float64(cfg.CPUUpperThreshold)*0.9), 5)
 			WaitForNInstancesRunning(appGUID, 2, 5*time.Minute)
 
 			By("should scale in to 1 instance after cpu usage is reduced")
 			//only hit the one instance that was asked to run hot.
-			AppEndCpuTest(cfg, appName, 0)
+			StopCPUUsage(cfg, appName, 0)
 
 			WaitForNInstancesRunning(appGUID, 1, 10*time.Minute)
 		})
@@ -275,15 +275,51 @@ var _ = Describe("AutoScaler dynamic policy", func() {
 			//   - app memory = 1GB
 			//   - app CPU entitlement = 4096[total shares] / (32[GB host ram] * 1024) * (1[app memory in GB] * 1024) * 0,1953 ~= 25%
 
-			SetAppMemory(cfg, appName, cfg.CPUUtilScalingPolicyTest.AppMemory)
+			ScaleMemory(cfg, appName, cfg.CPUUtilScalingPolicyTest.AppMemory)
 
 			// cpuutil will be 100% if cpu usage is reaching the value of cpu entitlement
 			maxCPUUsage := cfg.CPUUtilScalingPolicyTest.AppCPUEntitlement
-			AppSetCpuUsage(cfg, appName, maxCPUUsage, 5)
+			StartCPUUsage(cfg, appName, maxCPUUsage, 5)
 			WaitForNInstancesRunning(appGUID, 2, 5*time.Minute)
 
-			//only hit the one instance that was asked to run hot
-			AppEndCpuTest(cfg, appName, 0)
+			// only hit the one instance that was asked to run hot
+			StopCPUUsage(cfg, appName, 0)
+			WaitForNInstancesRunning(appGUID, 1, 5*time.Minute)
+		})
+	})
+
+	Context("when there is a scaling policy for diskutil", func() {
+		BeforeEach(func() {
+			policy = GenerateDynamicScaleOutAndInPolicy(1, 2, "diskutil", 30, 60)
+			initialInstanceCount = 1
+		})
+
+		It("should scale out and in", func() {
+			ScaleDisk(cfg, appName, "1GB")
+
+			StartDiskUsage(cfg, appName, 800, 5)
+			WaitForNInstancesRunning(appGUID, 2, 5*time.Minute)
+
+			// only hit the one instance that was asked to occupy disk space
+			StopDiskUsage(cfg, appName, 0)
+			WaitForNInstancesRunning(appGUID, 1, 5*time.Minute)
+		})
+	})
+
+	Context("when there is a scaling policy for disk", func() {
+		BeforeEach(func() {
+			policy = GenerateDynamicScaleOutAndInPolicy(1, 2, "disk", 300, 600)
+			initialInstanceCount = 1
+		})
+
+		It("should scale out and in", func() {
+			ScaleDisk(cfg, appName, "1GB")
+
+			StartDiskUsage(cfg, appName, 800, 5)
+			WaitForNInstancesRunning(appGUID, 2, 5*time.Minute)
+
+			// only hit the one instance that was asked to occupy disk space
+			StopDiskUsage(cfg, appName, 0)
 			WaitForNInstancesRunning(appGUID, 1, 5*time.Minute)
 		})
 	})

src/acceptance/assets/app/go_app/internal/app/app.go

@@ -16,7 +16,7 @@ import (
 	"go.uber.org/zap/zapcore"
 )
 
-func Router(logger *zap.Logger, timewaster TimeWaster, memoryTest MemoryGobbler, cpuTest CPUWaster, customMetricTest CustomMetricClient) *gin.Engine {
+func Router(logger *zap.Logger, timewaster TimeWaster, memoryTest MemoryGobbler, cpuTest CPUWaster, diskOccupier DiskOccupier, customMetricTest CustomMetricClient) *gin.Engine {
 	r := gin.New()
 
 	otel.SetTracerProvider(sdktrace.NewTracerProvider())
@@ -51,15 +51,23 @@ func Router(logger *zap.Logger, timewaster TimeWaster, memoryTest MemoryGobbler,
 	MemoryTests(logr, r.Group("/memory"), memoryTest)
 	ResponseTimeTests(logr, r.Group("/responsetime"), timewaster)
 	CPUTests(logr, r.Group("/cpu"), cpuTest)
+	DiskTest(r.Group("/disk"), diskOccupier)
 	CustomMetricsTests(logr, r.Group("/custom-metrics"), customMetricTest)
 	return r
 }
 
 func New(logger *zap.Logger, address string) *http.Server {
 	errorLog, _ := zap.NewStdLogAt(logger, zapcore.ErrorLevel)
 	return &http.Server{
-		Addr:         address,
-		Handler:      Router(logger, &Sleeper{}, &ListBasedMemoryGobbler{}, &ConcurrentBusyLoopCPUWaster{}, &CustomMetricAPIClient{}),
+		Addr: address,
+		Handler: Router(
+			logger,
+			&Sleeper{},
+			&ListBasedMemoryGobbler{},
+			&ConcurrentBusyLoopCPUWaster{},
+			NewDefaultDiskOccupier("this-file-is-being-used-during-disk-occupation"),
+			&CustomMetricAPIClient{},
+		),
 		ReadTimeout:  5 * time.Second,
 		IdleTimeout:  2 * time.Second,
 		WriteTimeout: 30 * time.Second,

src/acceptance/assets/app/go_app/internal/app/app_test.go

@@ -80,5 +80,5 @@ func apiTest(timeWaster app.TimeWaster, memoryGobbler app.MemoryGobbler, cpuWast
 	logger := zaptest.LoggerWriter(GinkgoWriter)
 
 	return apitest.New().
-		Handler(app.Router(logger, timeWaster, memoryGobbler, cpuWaster, customMetricClient))
+		Handler(app.Router(logger, timeWaster, memoryGobbler, cpuWaster, nil, customMetricClient))
 }

src/acceptance/assets/app/go_app/internal/app/disk.go

@@ -0,0 +1,124 @@
+package app
+
+import (
+	"crypto/rand"
+	"errors"
+	"io"
+	"net/http"
+	"os"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/gin-gonic/gin"
+)
+
+func DiskTest(r *gin.RouterGroup, diskOccupier DiskOccupier) *gin.RouterGroup {
+	r.GET("/:utilization/:minutes", func(c *gin.Context) {
+		var utilisation int64
+		var minutes int64
+		var err error
+
+		utilisation, err = strconv.ParseInt(c.Param("utilization"), 10, 64)
+		if err != nil {
+			Error(c, http.StatusBadRequest, "invalid utilization: %s", err.Error())
+			return
+		}
+		if minutes, err = strconv.ParseInt(c.Param("minutes"), 10, 64); err != nil {
+			Error(c, http.StatusBadRequest, "invalid minutes: %s", err.Error())
+			return
+		}
+		duration := time.Duration(minutes) * time.Minute
+		spaceInMB := utilisation * 1000 * 1000
+		if err = diskOccupier.Occupy(spaceInMB, duration); err != nil {
+			Error(c, http.StatusInternalServerError, "error invoking occupation: %s", err.Error())
+			return
+		}
+		c.JSON(http.StatusOK, gin.H{"utilization": utilisation, "minutes": minutes})
+	})
+
+	r.GET("/close", func(c *gin.Context) {
+		diskOccupier.Stop()
+		c.String(http.StatusOK, "close disk test")
+	})
+
+	return r
+}
+
+//counterfeiter:generate . DiskOccupier
+type DiskOccupier interface {
+	Occupy(space int64, duration time.Duration) error
+	Stop()
+}
+
+type defaultDiskOccupier struct {
+	mu        sync.RWMutex
+	isRunning bool
+	filePath  string
+}
+
+func NewDefaultDiskOccupier(filePath string) *defaultDiskOccupier {
+	return &defaultDiskOccupier{
+		filePath: filePath,
+	}
+}
+
+func (d *defaultDiskOccupier) Occupy(space int64, duration time.Duration) error {
+	if err := d.checkAlreadyRunning(); err != nil {
+		return err
+	}
+
+	if err := d.occupy(space); err != nil {
+		return err
+	}
+
+	d.stopAfter(duration)
+
+	return nil
+}
+
+func (d *defaultDiskOccupier) checkAlreadyRunning() error {
+	d.mu.RLock()
+	defer d.mu.RUnlock()
+
+	if d.isRunning {
+		return errors.New("disk space is already being occupied")
+	}
+
+	return nil
+}
+
+func (d *defaultDiskOccupier) occupy(space int64) error {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	file, err := os.Create(d.filePath)
+	if err != nil {
+		return err
+	}
+	if _, err := io.CopyN(file, io.LimitReader(rand.Reader, space), space); err != nil {
+		return err
+	}
+	if err := file.Close(); err != nil {
+		return err
+	}
+	d.isRunning = true
+
+	return nil
+}
+
+func (d *defaultDiskOccupier) stopAfter(duration time.Duration) {
+	go func() {
+		time.Sleep(duration)
+		d.Stop()
+	}()
+}
+
+func (d *defaultDiskOccupier) Stop() {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	if err := os.Remove(d.filePath); err == nil {
+		d.isRunning = false
+	}
+}