Merge pull request #12100 from izaac/jenkins_debug

[Automation] - Adjust cypress execution to generate html report on Jenkins

cypress/jenkins/Jenkinsfile

@@ -66,8 +66,7 @@ node {
             try {
               stage('Grab Results') {
                   sh "cypress/jenkins/scpget.sh dashboard/results.xml"
-                  sh "cypress/jenkins/scpget.sh dashboard/results.html"
-                  sh "cypress/jenkins/scpget.sh dashboard/assets"
+                  sh "cypress/jenkins/scpget.sh dashboard/cypress/reports/html/* html"
               }
             } catch(err) {
                 echo "Error: " + err
@@ -87,6 +86,15 @@ node {
             try {
               stage('Test Report') {
                   step([$class: 'JUnitResultArchiver', testResults: '**/results.xml'])
+                  sh "pwd && ls -al && ls -al cypress"
+                  publishHTML(target : [allowMissing: false,
+                    alwaysLinkToLastBuild: true,
+                    keepAll: true,
+                    // The path is relative to ${WORKSPACE}
+                    reportDir: 'html',
+                    reportFiles: 'index.html',
+                    reportName: 'html-report',
+                    reportTitles: 'Report'])
               }
             }
             catch(err) {

cypress/jenkins/cypress.config.jenkins.ts

@@ -0,0 +1,117 @@
+/* eslint-disable no-console */
+import { defineConfig } from 'cypress';
+import { removeDirectory } from 'cypress-delete-downloads-folder';
+// Required for env vars to be available in cypress
+require('dotenv').config();
+
+/**
+ * VARIABLES
+ */
+
+const testDirs = [
+  'cypress/e2e/tests/priority/**/*.spec.ts',
+  'cypress/e2e/tests/components/**/*.spec.ts',
+  'cypress/e2e/tests/setup/**/*.spec.ts',
+  'cypress/e2e/tests/pages/**/*.spec.ts',
+  'cypress/e2e/tests/navigation/**/*.spec.ts',
+  'cypress/e2e/tests/global-ui/**/*.spec.ts',
+  'cypress/e2e/tests/features/**/*.spec.ts',
+  'cypress/e2e/tests/extensions/**/*.spec.ts'
+];
+const skipSetup = process.env.TEST_SKIP?.includes('setup');
+const baseUrl = (process.env.TEST_BASE_URL || 'https://localhost:8005').replace(/\/$/, '');
+const DEFAULT_USERNAME = 'admin';
+const username = process.env.TEST_USERNAME || DEFAULT_USERNAME;
+const apiUrl = process.env.API || (baseUrl.endsWith('/dashboard') ? baseUrl.split('/').slice(0, -1).join('/') : baseUrl);
+
+/**
+ * LOGS:
+ * Summary of the environment variables that we have detected (or are going ot use)
+ * We won't show any passwords
+ */
+console.log('E2E Test Configuration');
+console.log('');
+console.log(`    Username: ${ username }`);
+
+if (!process.env.CATTLE_BOOTSTRAP_PASSWORD && !process.env.TEST_PASSWORD) {
+  console.log(' ❌ You must provide either CATTLE_BOOTSTRAP_PASSWORD or TEST_PASSWORD');
+}
+if (process.env.CATTLE_BOOTSTRAP_PASSWORD && process.env.TEST_PASSWORD) {
+  console.log(' ❗ If both CATTLE_BOOTSTRAP_PASSWORD and TEST_PASSWORD are provided, the first will be used');
+}
+if (!skipSetup && !process.env.CATTLE_BOOTSTRAP_PASSWORD) {
+  console.log(' ❌ You must provide CATTLE_BOOTSTRAP_PASSWORD when running setup tests');
+}
+if (skipSetup && !process.env.TEST_PASSWORD) {
+  console.log(' ❌ You must provide TEST_PASSWORD when running the tests without the setup tests');
+}
+
+console.log(`    Setup tests will ${ skipSetup ? 'NOT' : '' } be run`);
+console.log(`    Dashboard URL: ${ baseUrl }`);
+console.log(`    Rancher API URL: ${ apiUrl }`);
+
+// Check API - sometimes in dev, you might have API set to a different system to the base url - this won't work
+// as the login cookie will be for the base url and any API requests will fail as not authenticated
+if (apiUrl && !baseUrl.startsWith(apiUrl)) {
+  console.log('\n ❗ API variable is different to TEST_BASE_URL - tests may fail due to authentication issues');
+}
+
+console.log('');
+
+/**
+ * CONFIGURATION
+ */
+export default defineConfig({
+  projectId:             process.env.TEST_PROJECT_ID,
+  defaultCommandTimeout: process.env.TEST_TIMEOUT ? +process.env.TEST_TIMEOUT : 10000,
+  trashAssetsBeforeRuns: true,
+  chromeWebSecurity:     false,
+  retries:               {
+    runMode:  2,
+    openMode: 0
+  },
+  env: {
+    grepFilterSpecs:     true,
+    grepOmitFiltered:    true,
+    baseUrl,
+    api:                 apiUrl,
+    username,
+    password:            process.env.CATTLE_BOOTSTRAP_PASSWORD || process.env.TEST_PASSWORD,
+    bootstrapPassword:   process.env.CATTLE_BOOTSTRAP_PASSWORD,
+    grepTags:            process.env.GREP_TAGS,
+    // the below env vars are only available to tests that run in Jenkins
+    awsAccessKey:        process.env.AWS_ACCESS_KEY_ID,
+    awsSecretKey:        process.env.AWS_SECRET_ACCESS_KEY,
+    azureSubscriptionId: process.env.AZURE_AKS_SUBSCRIPTION_ID,
+    azureClientId:       process.env.AZURE_CLIENT_ID,
+    azureClientSecret:   process.env.AZURE_CLIENT_SECRET,
+    customNodeIp:        process.env.CUSTOM_NODE_IP,
+    customNodeKey:       process.env.CUSTOM_NODE_KEY
+  },
+  // Jenkins reporters configuration jUnit and HTML
+  reporter:        'cypress-multi-reporters',
+  reporterOptions: {
+    reporterEnabled:                   'cypress-mochawesome-reporter, mocha-junit-reporter',
+    mochaJunitReporterReporterOptions: {
+      mochaFile:      'cypress/jenkins/reports/junit/junit-[hash].xml',
+      toConsole:      true,
+      jenkinsMode:    true,
+      includePending: true
+    },
+    cypressMochawesomeReporterReporterOptions: { charts: false },
+  },
+  e2e: {
+    setupNodeEvents(on, config) {
+      require('cypress-mochawesome-reporter/plugin')(on);
+      require('@cypress/grep/src/plugin')(config);
+      on('task', { removeDirectory });
+    },
+    fixturesFolder:               'cypress/e2e/blueprints',
+    experimentalSessionAndOrigin: true,
+    specPattern:                  testDirs,
+    baseUrl
+  },
+  video:               false,
+  videoCompression:    25,
+  videoUploadOnPasses: false,
+});

cypress/jenkins/cypress.sh

@@ -11,13 +11,14 @@ kubectl version --client=true
 kubectl get nodes
 
 node -v
-yarn add --dev -W mocha mochawesome mochawesome-merge \
-  mochawesome-report-generator cypress-multi-reporters \
-  mocha-junit-reporter
-
-NO_COLOR=1 CYPRESS_grepTags="CYPRESSTAGS" cypress run --browser chrome \
-  --reporter cypress-multi-reporters \
-  --reporter-options \
-    configFile=cypress/jenkins/reporter-options.json
+
+yarn config set ignore-engines true
+
+yarn add -W mocha cypress-mochawesome-reporter cypress-multi-reporters cypress-commands \
+  mochawesome-report-generator mochawesome-merge mocha-junit-reporter junit-report-merger
+
+yarn add -W https://github.com/elaichenkov/cypress-delete-downloads-folder
+
+NO_COLOR=1 CYPRESS_grepTags="CYPRESSTAGS" cypress run --browser chrome --config-file cypress/jenkins/cypress.config.jenkins.ts
 
 echo "CYPRESS EXIT CODE: $?"

cypress/jenkins/scpget.sh

@@ -16,5 +16,13 @@ chmod 400 "${PRIV_KEY}"
 
 NODE_EXTERNAL_IP="$(corral vars ci first_node_ip)"
 
+
+REPORT_DIR="."
+
+if [[ $# -gt 1 ]]; then
+  REPORT_DIR="$2"
+  mkdir -p "$2"
+fi
+
 scp -r -i ${PRIV_KEY} -o StrictHostKeyChecking=no \
-  -o UserKnownHostsFile=/dev/null "root@${NODE_EXTERNAL_IP}:$1" .
+  -o UserKnownHostsFile=/dev/null "root@${NODE_EXTERNAL_IP}:$1" "${REPORT_DIR}"

cypress/support/e2e.ts

@@ -4,6 +4,7 @@ import './commands/chainable';
 import './commands/rancher-api-commands';
 import registerCypressGrep from '@cypress/grep/src/support';
 import { addCustomCommand } from 'cypress-delete-downloads-folder';
+import 'cypress-mochawesome-reporter/register';
 
 registerCypressGrep();
 addCustomCommand();

package.json

@@ -172,6 +172,7 @@
     "cy2": "4.0.9",
     "cypress": "11.1.0",
     "cypress-delete-downloads-folder": "0.0.4",
+    "cypress-mochawesome-reporter": "^3.8.2",
     "eslint": "7.32.0",
     "eslint-config-standard": "16.0.3",
     "eslint-import-resolver-node": "0.3.4",