Merge pull request #1291 from cloudfoundry/fixes-panic-calling-cf-com…

…mands

Improve performance test and make it run for larger app count

Makefile

@@ -253,7 +253,8 @@ lint-actions:
 
 $(addprefix lint_,$(go_modules)): lint_%:
 	@echo " - linting: $(patsubst lint_%,%,$@)"
-	@pushd src/$(patsubst lint_%,%,$@) >/dev/null && golangci-lint run --path-prefix=src/$(patsubst lint_%,%,$@) --config ${lint_config} ${OPTS}
+	@echo " receiving OPTS ${OPTS}"
+	@pushd src/$(patsubst lint_%,%,$@) >/dev/null && golangci-lint run --config ${lint_config} ${OPTS}
 
 .PHONY: spec-test
 spec-test:

ci/autoscaler/scripts/vars.source.sh

@@ -104,13 +104,13 @@ debug "SERVICE_OFFERING_ENABLED: ${SERVICE_OFFERING_ENABLED}"
 
 export GINKGO_OPTS=${GINKGO_OPTS:-"--fail-fast"}
 
-export PERFORMANCE_APP_COUNT="${PERFORMANCE_APP_COUNT:-20}"
+export PERFORMANCE_APP_COUNT="${PERFORMANCE_APP_COUNT:-50}"
 debug "PERFORMANCE_APP_COUNT: ${PERFORMANCE_APP_COUNT}"
 
 export PERFORMANCE_APP_PERCENTAGE_TO_SCALE="${PERFORMANCE_APP_PERCENTAGE_TO_SCALE:-30}"
 debug "PERFORMANCE_APP_PERCENTAGE_TO_SCALE: ${PERFORMANCE_APP_PERCENTAGE_TO_SCALE}"
 
-export PERFORMANCE_SETUP_WORKERS="${PERFORMANCE_SETUP_WORKERS:-50}"
+export PERFORMANCE_SETUP_WORKERS="${PERFORMANCE_SETUP_WORKERS:-20}"
 debug "PERFORMANCE_SETUP_WORKERS: ${PERFORMANCE_SETUP_WORKERS}"
 
 export PERFORMANCE_TEARDOWN=${PERFORMANCE_TEARDOWN:-true}

src/acceptance/helpers/org.go

@@ -43,7 +43,6 @@ func DeleteOrgs(orgs []string, timeout time.Duration) {
 	}
 
 	fmt.Printf("\nDeleting orgs: %s ", strings.Join(orgs, ", "))
-
 	for _, org := range orgs {
 		deleteOrg := cf.Cf("delete-org", org, "-f").Wait(timeout)
 		Expect(deleteOrg).To(Exit(0), fmt.Sprintf("unable to delete org %s", org))

src/acceptance/helpers/spaces.go

@@ -38,7 +38,7 @@ func GetTestSpaces(orgGuid string, cfg *config.Config) []string {
 			spaceNames = append(spaceNames, space.Name)
 		}
 	}
-	ginkgo.GinkgoWriter.Printf("\nGot orgs: %s\n", spaceNames)
+	ginkgo.GinkgoWriter.Printf("\nGot spaces: %s\n", spaceNames)
 	return spaceNames
 }
 
@@ -56,9 +56,7 @@ func DeleteSpaces(orgName string, spaces []string, timeout time.Duration) {
 	if len(spaces) == 0 {
 		return
 	}
-
-	fmt.Printf("\nDeleting spaces: %s ", strings.Join(spaces, ", "))
-
+	fmt.Printf("\nDeleting spaces: %s \n", strings.Join(spaces, ", "))
 	for _, spaceName := range spaces {
 		if timeout.Seconds() != 0 {
 			deleteSpace := cf.Cf("delete-space", "-f", "-o", orgName, spaceName).Wait(timeout)

src/acceptance/run_performance/run_performance_test.go

@@ -13,14 +13,17 @@ import (
 	"github.com/onsi/gomega/gmeasure"
 )
 
+const pollTime = 10 * time.Second
+const desiredScalingTime = 300 * time.Minute
+
 var _ = Describe("Scale in and out (eg: 30%) percentage of apps", func() {
 	var (
 		appsToScaleCount       int
 		percentageToScale      int
 		appCount               int
 		samplingConfig         gmeasure.SamplingConfig
 		experiment             *gmeasure.Experiment
-		doneAppsCount          int32
+		scaledInAppsCount      int32
 		scaledOutAppsCount     int32
 		errors                 sync.Map
 		startedApps            []helpers.AppInfo
@@ -49,12 +52,12 @@ var _ = Describe("Scale in and out (eg: 30%) percentage of apps", func() {
 		actualAppsToScaleCount = int(math.RoundToEven(float64(len(startedApps) * percentageToScale / 100)))
 
 		fmt.Printf("\nDesired Scaling %d apps: \n", appsToScaleCount)
-		fmt.Printf("Actual Scaling %d apps (based on sucessuful apps pushed): \n\n", actualAppsToScaleCount)
+		fmt.Printf("Actual Scaling %d apps (based on sucessuful apps pushed) \n\n", actualAppsToScaleCount)
 
 		samplingConfig = gmeasure.SamplingConfig{
 			N:           actualAppsToScaleCount,
-			NumParallel: actualAppsToScaleCount,
-			Duration:    60 * time.Minute,
+			NumParallel: 100, // number of sample to execute at a time
+			Duration:    300 * time.Minute,
 		}
 		experiment = gmeasure.NewExperiment("Scaling Benchmark")
 	})
@@ -66,55 +69,56 @@ var _ = Describe("Scale in and out (eg: 30%) percentage of apps", func() {
 			experiment.Sample(func(i int) {
 				defer GinkgoRecover()
 				appName := startedApps[i].Name
-				appGUID, err := helpers.GetAppGuid(cfg, appName)
-				if err != nil {
-					errors.Store(appName, err)
-				}
-				pollTime := 10 * time.Second
+				appGUID := startedApps[i].Guid
 
 				wg := sync.WaitGroup{}
 				wg.Add(1)
-				experiment.MeasureDuration("scale-out", func() {
-					scaleOut := func() (int, error) {
-						helpers.SendMetric(cfg, appName, 550)
-						return helpers.RunningInstances(appGUID, 20*time.Second)
-					}
-					Eventually(scaleOut).WithPolling(pollTime).WithTimeout(5*time.Minute).Should(Equal(2),
-						fmt.Sprintf("Failed to scale out app: %s", appName))
-					fmt.Printf("\nfinished scaling-out app: %s at index %d\n", appName, i)
-					wg.Done()
-				})
+				experiment.MeasureDuration("scale-out", scaleOutApp(appName, appGUID, &wg))
 				wg.Wait()
 
 				atomic.AddInt32(&scaledOutAppsCount, 1)
 				fmt.Printf("Scaled-Out apps: %d/%d\n", atomic.LoadInt32(&scaledOutAppsCount), actualAppsToScaleCount)
 
 				wg = sync.WaitGroup{}
 				wg.Add(1)
-				experiment.MeasureDuration("scale-in", func() {
-					scaleIn := func() (int, error) {
-						helpers.SendMetric(cfg, appName, 100)
-						return helpers.RunningInstances(appGUID, 20*time.Second)
-					}
-					Eventually(scaleIn).WithPolling(pollTime).WithTimeout(5*time.Minute).Should(Equal(1),
-						fmt.Sprintf("Failed to scale in app: %s", appName))
-					fmt.Printf("\nfinished scaling-in app: %s at index %d\n", appName, i)
-					wg.Done()
-				})
+				experiment.MeasureDuration("scale-in", scaleInApp(appName, appGUID, &wg))
 				wg.Wait()
-
-				atomic.AddInt32(&doneAppsCount, 1)
-				fmt.Printf("Scaled-in apps: %d/%d\n", atomic.LoadInt32(&doneAppsCount), actualAppsToScaleCount)
+				atomic.AddInt32(&scaledInAppsCount, 1)
+				fmt.Printf("Scaled-in apps: %d/%d\n", atomic.LoadInt32(&scaledInAppsCount), actualAppsToScaleCount)
 
 			}, samplingConfig)
-
-			Eventually(func() int32 { return atomic.LoadInt32(&doneAppsCount) }, 10*time.Minute, 10*time.Second).Should(BeEquivalentTo(actualAppsToScaleCount))
+			fmt.Printf("Waiting %s minutes to finish scaling...\n\n", desiredScalingTime)
+			Eventually(func() int32 { return atomic.LoadInt32(&scaledInAppsCount) }, desiredScalingTime, 10*time.Second).Should(BeEquivalentTo(actualAppsToScaleCount))
 			checkMedianDurationFor(experiment, "scale-out")
 			checkMedianDurationFor(experiment, "scale-in")
 		})
 	})
 })
 
+func scaleInApp(appName string, appGUID string, wg *sync.WaitGroup) func() {
+	return func() {
+		scaleIn := func() (int, error) {
+			helpers.SendMetric(cfg, appName, 100)
+			return helpers.RunningInstances(appGUID, 20*time.Second)
+		}
+		Eventually(scaleIn).WithPolling(pollTime).WithTimeout(5*time.Minute).Should(Equal(1),
+			fmt.Sprintf("Failed to scale in app: %s", appName))
+		wg.Done()
+	}
+}
+
+func scaleOutApp(appName string, appGUID string, wg *sync.WaitGroup) func() {
+	return func() {
+		scaleOut := func() (int, error) {
+			helpers.SendMetric(cfg, appName, 550)
+			return helpers.RunningInstances(appGUID, 20*time.Second)
+		}
+		Eventually(scaleOut).WithPolling(pollTime).WithTimeout(5*time.Minute).Should(Equal(2),
+			fmt.Sprintf("Failed to scale out app: %s", appName))
+		wg.Done()
+	}
+}
+
 func checkMedianDurationFor(experiment *gmeasure.Experiment, statName string) {
 	scaleOutStats := experiment.GetStats(statName)
 	medianScaleOutDuration := scaleOutStats.DurationFor(gmeasure.StatMedian)

src/acceptance/setup_performance/setup_performance_suite_test.go

@@ -10,6 +10,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/KevinJCross/cf-test-helpers/v2/cf"
+	"github.com/onsi/gomega/gexec"
+
 	"github.com/KevinJCross/cf-test-helpers/v2/workflowhelpers"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -32,7 +35,6 @@ func TestSetup(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-	var spaceGuid, orgGuid string
 
 	if os.Getenv("SKIP_TEARDOWN") == "true" {
 		fmt.Println("Skipping Teardown...")
@@ -45,12 +47,11 @@ var _ = BeforeSuite(func() {
 
 	EnableServiceAccess(setup, cfg, setup.GetOrganizationName())
 	workflowhelpers.AsUser(setup.AdminUserContext(), cfg.DefaultTimeoutDuration(), func() {
-		_, orgGuid, _, spaceGuid = GetOrgSpaceNamesAndGuids(cfg, setup.GetOrganizationName())
+		_, orgGuid, _, spaceGuid := GetOrgSpaceNamesAndGuids(cfg, setup.GetOrganizationName())
+		Expect(spaceGuid).NotTo(BeEmpty())
 		updateOrgQuotaForPerformanceTest(orgGuid)
 	})
 
-	cleanUpServiceInstanceInParallel(setup, orgGuid, spaceGuid)
-
 	if cfg.IsServiceOfferingEnabled() {
 		CheckServiceExists(cfg, setup.TestSpace.SpaceName(), cfg.ServiceName)
 	}
@@ -77,45 +78,98 @@ func updateOrgQuotaForPerformanceTest(orgGuid string) {
 	}
 }
 
-func cleanUpServiceInstanceInParallel(setup *workflowhelpers.ReproducibleTestSuiteSetup, orgGuid string, spaceGuid string) {
+func cleanup() {
+	setup = workflowhelpers.NewTestSuiteSetup(cfg)
+	workflowhelpers.AsUser(setup.AdminUserContext(), cfg.DefaultTimeoutDuration(), func() {
+		fmt.Println("\nCleaning up test leftovers...")
+		if cfg.UseExistingOrganization {
+			targetOrg(setup)
+			orgGuid := GetOrgGuid(cfg, cfg.ExistingOrganization)
+			spaceNames := GetTestSpaces(orgGuid, cfg)
+			if len(spaceNames) == 0 {
+				return
+			}
+			waitGroup := sync.WaitGroup{}
+			waitGroup.Add(2)
+
+			deleteAllServices(cfg.Performance.SetupWorkers, orgGuid, GetSpaceGuid(cfg, orgGuid), &waitGroup)
+			// delete all apps in a test space - only from first space - what if two spaces are present
+			deleteAllApps(cfg.Performance.SetupWorkers, orgGuid, GetSpaceGuid(cfg, orgGuid), &waitGroup)
+			fmt.Println("\nWaiting for services and apps to be deleted...")
+			waitGroup.Wait()
+			DeleteSpaces(cfg.ExistingOrganization, GetTestSpaces(orgGuid, cfg), cfg.DefaultTimeoutDuration())
+		} else {
+			DeleteOrgs(GetTestOrgs(cfg), time.Duration(120)*time.Second)
+		}
+	})
+}
+
+func deleteAllServices(workersCount int, orgGuid string, spaceGuid string, parentWaitGroup *sync.WaitGroup) {
+	defer parentWaitGroup.Done()
+	defer GinkgoRecover()
 	waitGroup := sync.WaitGroup{}
 	servicesChan := make(chan string)
 
-	serviceInstances := GetServices(cfg, orgGuid, spaceGuid)
-	if len(serviceInstances) != 0 {
-		fmt.Printf("\ndeleting existing service instances: %d\n", len(serviceInstances))
-		for i := 0; i < len(serviceInstances); i++ {
-			waitGroup.Add(1)
-			i := i
-			go deleteExistingServiceInstances(i, servicesChan, setup, orgGuid, spaceGuid, &waitGroup)
-		}
-		for _, serviceInstanceName := range serviceInstances {
-			servicesChan <- serviceInstanceName
-		}
-		close(servicesChan)
-		waitGroup.Wait()
+	services := GetServices(cfg, orgGuid, spaceGuid)
+	if len(services) == 0 {
+		fmt.Printf("- deleting existing service instances: %d\n", len(services))
+		return
+	}
+	fmt.Printf("- deleting existing service instances: %d\n", len(services))
+	for i := 1; i <= workersCount; i++ {
+		waitGroup.Add(1)
+		go deleteExistingServiceInstances(i, servicesChan, setup, &waitGroup)
 	}
+	for _, serviceInstanceName := range services {
+		servicesChan <- serviceInstanceName
+	}
+	close(servicesChan)
+	waitGroup.Wait()
 }
 
-func deleteExistingServiceInstances(workerId int, servicesChan chan string, setup *workflowhelpers.ReproducibleTestSuiteSetup, orgGuid string, spaceGuid string, wg *sync.WaitGroup) {
-	fmt.Printf("Worker %d  - Delete Service Instance starting...\n", workerId)
+func deleteAllApps(workersCount int, orgGuid string, spaceName string, parentWaitGroup *sync.WaitGroup) {
+	defer parentWaitGroup.Done()
+	defer GinkgoRecover()
+	waitGroup := sync.WaitGroup{}
+	appsChan := make(chan string)
+
+	apps := GetApps(cfg, orgGuid, spaceName, "node-custom-metric-benchmark-")
+	fmt.Printf("\n- deleting existing app instances: %d\n", len(apps))
+	if len(apps) == 0 {
+		return
+	}
+	for i := 1; i <= workersCount; i++ {
+		waitGroup.Add(1)
+		go deleteExistingApps(i, appsChan, &waitGroup)
+	}
+	for _, app := range apps {
+		appsChan <- app
+	}
+	close(appsChan)
+	waitGroup.Wait()
+}
+
+func deleteExistingServiceInstances(workerId int, servicesChan chan string, setup *workflowhelpers.ReproducibleTestSuiteSetup, wg *sync.WaitGroup) {
 	defer wg.Done()
 	defer GinkgoRecover()
+
 	for instanceName := range servicesChan {
 		fmt.Printf("worker %d  - deleting service instance - %s\n", workerId, instanceName)
 		DeleteServiceInstance(cfg, setup, instanceName)
 	}
-	fmt.Printf("worker %d  - Delete Service Instance finished...\n", workerId)
 }
 
-func cleanup() {
-	setup = workflowhelpers.NewTestSuiteSetup(cfg)
-	workflowhelpers.AsUser(setup.AdminUserContext(), cfg.DefaultTimeoutDuration(), func() {
-		DeleteOrgs(GetTestOrgs(cfg), time.Duration(120)*time.Second)
+func deleteExistingApps(workerId int, appsChan chan string, wg *sync.WaitGroup) {
+	defer wg.Done()
+	defer GinkgoRecover()
 
-		if cfg.UseExistingOrganization {
-			orgGuid := GetOrgGuid(cfg, cfg.ExistingOrganization)
-			DeleteSpaces(cfg.ExistingOrganization, GetTestSpaces(orgGuid, cfg), 0*time.Second)
-		}
-	})
+	for appName := range appsChan {
+		fmt.Printf("worker %d  - deleting app instance - %s\n", workerId, appName)
+		DeleteTestApp(appName, cfg.DefaultTimeoutDuration())
+	}
+}
+
+func targetOrg(setup *workflowhelpers.ReproducibleTestSuiteSetup) {
+	cmd := cf.Cf("target", "-o", setup.GetOrganizationName()).Wait(cfg.DefaultTimeoutDuration())
+	Expect(cmd).To(gexec.Exit(0), fmt.Sprintf("failed cf target org  %s : %s", setup.GetOrganizationName(), string(cmd.Err.Contents())))
 }