test(core): Log amazonq/toolkit tests at the bottom of the logs

buildspec/linuxE2ETests.yml

@@ -37,7 +37,7 @@ phases:
         commands:
             - export HOME=/home/codebuild-user
             # Ignore failure until throttling issues are fixed.
-            - xvfb-run npm run testE2E
+            - xvfb-run npm run testE2E; npm run createTestReport -- "$?"
             - VCS_COMMIT_ID="${CODEBUILD_RESOLVED_SOURCE_VERSION}"
             - CI_BUILD_URL=$(echo $CODEBUILD_BUILD_URL | sed 's/#/%23/g')
             - CI_BUILD_ID="${CODEBUILD_BUILD_ID}"

buildspec/linuxIntegrationTests.yml

@@ -92,7 +92,7 @@ phases:
     build:
         commands:
             - export HOME=/home/codebuild-user
-            - xvfb-run npm run testInteg
+            - xvfb-run npm run testInteg; npm run createTestReport -- "$?"
             - VCS_COMMIT_ID="${CODEBUILD_RESOLVED_SOURCE_VERSION}"
             - CI_BUILD_URL=$(echo $CODEBUILD_BUILD_URL | sed 's/#/%23/g')
             - CI_BUILD_ID="${CODEBUILD_BUILD_ID}"

buildspec/linuxTests.yml

@@ -41,7 +41,7 @@ phases:
                     # Ensure that "foo | run_and_report" fails correctly.
                     set -o pipefail
                     . buildspec/shared/common.sh
-                    2>&1 xvfb-run npm test --silent | run_and_report 2 \
+                    2>&1 xvfb-run npm test --silent; npm run createTestReport -- "$?" | run_and_report 2 \
                         'rejected promise not handled' \
                         'This typically indicates a bug. Read https://developer.mozilla.org/docs/Web/JavaScript/Guide/Using_promises#error_handling'
                 }

package.json

@@ -25,18 +25,19 @@
         "buildCustomLintPlugin": "npm run build -w plugins/eslint-plugin-aws-toolkits",
         "compile": "npm run compile -w packages/",
         "testCompile": "npm run testCompile -w packages/ --if-present",
-        "test": "npm run test -w packages/ --if-present",
-        "testWeb": "npm run testWeb -w packages/ --if-present",
-        "testE2E": "npm run testE2E -w packages/ --if-present",
-        "testInteg": "npm run testInteg -w packages/ --if-present",
+        "test": "npm run test -w packages/ --if-present; npm run createTestReport",
+        "testWeb": "npm run testWeb -w packages/ --if-present; npm run createTestReport",
+        "testE2E": "npm run testE2E -w packages/ --if-present; npm run createTestReport",
+        "testInteg": "npm run testInteg -w packages/ --if-present; npm run createTestReport",
         "package": "npm run package -w packages/toolkit -w packages/amazonq",
         "newChange": "echo 'Must specify subproject/workspace with -w packages/<subproject>' && false",
         "createRelease": "echo 'Must specify subproject/workspace with -w packages/<subproject>' && false",
         "lint": "npm run lint -w packages/ --if-present",
         "lintfix": "eslint -c .eslintrc.js --ignore-path .gitignore --ignore-pattern '**/*.json' --ignore-pattern '**/*.gen.ts' --ignore-pattern '**/types/*.d.ts' --ignore-pattern '**/src/testFixtures/**' --ignore-pattern '**/resources/js/graphStateMachine.js' --fix --ext .ts packages plugins",
         "clean": "npm run clean -w packages/ -w plugins/",
         "reset": "npm run clean && ts-node ./scripts/clean.ts node_modules && npm install",
-        "generateNonCodeFiles": "npm run generateNonCodeFiles -w packages/ --if-present"
+        "generateNonCodeFiles": "npm run generateNonCodeFiles -w packages/ --if-present",
+        "createTestReport": "ts-node ./scripts/createTestReport.ts"
     },
     "devDependencies": {
         "@aws-toolkits/telemetry": "^1.0.289",

packages/amazonq/test/unit/validation.test.ts

@@ -27,4 +27,32 @@ describe('package validations', function () {
         )
         assert.deepStrictEqual(packageJson.contributes.icons, corePackageJson.contributes.icons)
     })
+
+    describe('foo', () => {
+        it('bar1', () => {
+            assert.ok(true)
+        })
+        describe('fi', () => {
+            it('bar2', () => {
+                assert.ok(true)
+            })
+            describe('fo234', () => {
+                it('bar3', () => {
+                    throw new Error('foo')
+                })
+            })
+        })
+    })
+
+    describe('package validations', () => {
+        it('wee', () => {
+            assert.ok(true)
+        })
+
+        describe('foo', () => {
+            it('wee 2', () => {
+                assert.ok(true)
+            })
+        })
+    })
 })

packages/core/src/test/shared/applicationBuilder/explorer/nodes/appNode.test.ts

@@ -169,6 +169,7 @@ describe('AppNode', () => {
 
     describe('getTreeItem', () => {
         it('should return a TreeItem with the correct properties', () => {
+            assert.fail('failed')
             const treeItem = appNode.getTreeItem()
             const expextedLabel = path.join('VSCode Example Workspace', 'Project One Root Folder')
 

packages/core/src/test/testRunner.ts

@@ -65,7 +65,8 @@ export async function runTests(
     }
 
     const root = getRoot()
-    const outputFile = path.resolve(root, '../../', '.test-reports', 'report.xml')
+    // output the report to the individual package
+    const outputFile = path.resolve(root, '.test-reports', 'report.xml')
     const colorOutput = !process.env['AWS_TOOLKIT_TEST_NO_COLOR']
 
     // Create the mocha test

scripts/createTestReport.ts

@@ -0,0 +1,243 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import * as fs from 'fs'
+import * as path from 'path'
+import * as xml2js from 'xml2js'
+
+interface TestFailure {
+    $: {
+        message: string
+    }
+    _: string
+}
+
+interface TestCase {
+    $: {
+        classname: string
+        name: string
+        time: string
+    }
+    failure?: TestFailure[]
+}
+
+interface TestSuite {
+    $: {
+        name: string
+        tests: string
+        failures: string
+        errors: string
+        time: string
+        file: string
+    }
+    testcase: TestCase[] | undefined
+}
+
+interface TestReport {
+    testsuites: {
+        testsuite: TestSuite[]
+    }
+}
+
+interface TestSummary {
+    totalTests: number
+    totalFailures: number
+    totalTime: number
+    failedTests: FailedTest[]
+}
+
+interface FailedTest {
+    suite: string
+    test: string
+    message: string
+    contents: string
+    path: string[]
+}
+
+/**
+ * Merge all of the packages/ test reports into a single directory
+ */
+async function createTestReport() {
+    console.log('Merging test reports')
+
+    const packagesDir = path.join(__dirname, '..', 'packages')
+
+    // Get all packages/* directories
+    const packageDirs = fs.readdirSync(packagesDir).map((dir) => path.join(packagesDir, dir))
+
+    // Find report.xml files in .test-reports subdirectories
+    const testReports = packageDirs
+        .map((dir) => path.join(dir, '.test-reports', 'report.xml'))
+        .filter((file) => fs.existsSync(file))
+
+    const mergedReport: TestReport = {
+        testsuites: {
+            testsuite: [],
+        },
+    }
+
+    const failedTests: FailedTest[] = []
+    let totalTests = 0
+    let totalFailures = 0
+    let totalTime = 0
+
+    let filePath = ''
+    let suites = new Set<string>()
+
+    /**
+     * Collect all test reports into a single merged test report object.
+     * Also keeps track of test count, test failures, and test run time
+     */
+    for (const file of testReports) {
+        const content = fs.readFileSync(file)
+        const result: { testsuites: { testsuite: TestSuite[] } } = await xml2js.parseStringPromise(content)
+        if (result.testsuites && result.testsuites.testsuite) {
+            for (const suite of result.testsuites.testsuite) {
+                if (suite.$.file !== filePath) {
+                    filePath = suite.$.file
+                    suites = new Set<string>()
+                }
+
+                for (const testcase of suite.testcase ?? []) {
+                    if (testcase.failure) {
+                        const testPath = parseTestHierarchy(suites, testcase.$.classname, suite.$.name, testcase.$.name)
+                        failedTests.push({
+                            suite: suite.$.name,
+                            test: testcase.$.name,
+                            message: testcase.failure[0].$.message,
+                            contents: testcase.failure[0]._,
+                            path: testPath,
+                        })
+                    }
+                }
+
+                totalTests += parseInt(suite.$.tests, 10)
+                totalFailures += parseInt(suite.$.failures, 10)
+                totalTime += parseFloat(suite.$.time)
+
+                suites.add(suite.$.name)
+            }
+
+            mergedReport.testsuites.testsuite.push(...result.testsuites.testsuite)
+        }
+    }
+
+    printTestSummary({
+        totalTests,
+        totalFailures,
+        totalTime,
+        failedTests,
+    })
+
+    writeReport(mergedReport)
+}
+
+/**
+ * Extracts and constructs a hierarchical test path from a test case identifier
+ *
+ * @param suites - Set of known test suite names
+ * @param className - Name of the test class
+ * @param suiteName - Name of the test suite
+ * @param testcaseName - Full name of the test case
+ * @example
+ * parseTestHierarchy(new Set(["package validations"]), 'bar1', 'foo', 'package validations foo bar1') -> ["package validations", "bar1", "foo"]
+ * @returns An array of path components representing the test hierarchy
+ */
+function parseTestHierarchy(suites: Set<string>, className: string, suiteName: string, testcaseName: string) {
+    let remainingPath = testcaseName
+    remainingPath = remainingPath.substring(0, remainingPath.lastIndexOf(className))
+    remainingPath = remainingPath.substring(0, remainingPath.lastIndexOf(suiteName))
+
+    const pathComponents = remainingPath.trim().split(' ')
+    let index = 0
+    let currentComponent = pathComponents[0]
+    const path = []
+    while (remainingPath.length > 0) {
+        index++
+        if (!suites.has(currentComponent)) {
+            currentComponent = currentComponent + ' ' + pathComponents[index]
+        } else {
+            path.push(currentComponent)
+            remainingPath = remainingPath.substring(currentComponent.length).trim()
+            currentComponent = pathComponents[index]
+        }
+    }
+
+    path.push(suiteName)
+    path.push(className)
+
+    return path
+}
+
+function printTestSummary({ totalTests, totalFailures, totalTime, failedTests }: TestSummary) {
+    const passingTests = totalTests - totalFailures
+    const pendingTests = 0
+
+    console.log(`${passingTests} passing (${Math.round(totalTime)}s)`)
+    if (pendingTests > 0) {
+        console.log(`${pendingTests} pending`)
+    }
+    if (totalFailures > 0) {
+        console.log(`${totalFailures} failing`)
+
+        failedTests.forEach((test, index) => {
+            let indent = '  '
+
+            for (let x = 0; x < test.path.length; x++) {
+                if (x == 0) {
+                    console.log(`${indent}${index + 1}) ${test.path[x]}`)
+                    indent += '   '
+                } else {
+                    console.log(`${indent}${test.path[x]}`)
+                }
+                indent += '  '
+            }
+
+            if (test.contents) {
+                // Indent the stack trace
+                console.log(
+                    test.contents
+                        .split('\n')
+                        .map((line) => `${indent}${line}`)
+                        .join('\n')
+                )
+            }
+            console.log() // Add empty line between failures
+        })
+    }
+}
+
+function writeReport(mergedReport: TestReport) {
+    const builder = new xml2js.Builder()
+    const xml = builder.buildObject(mergedReport)
+
+    /**
+     * Create the new test reports directory and write the test report
+     */
+    const reportsDir = path.join(__dirname, '..', '.test-reports')
+
+    // Create reports directory if it doesn't exist
+    if (!fs.existsSync(reportsDir)) {
+        fs.mkdirSync(reportsDir, { recursive: true })
+    }
+
+    fs.writeFileSync(path.join(reportsDir, 'report.xml'), xml)
+
+    const exitCodeArg = process.argv[2]
+    if (exitCodeArg) {
+        /**
+         * Retrieves the exit code from the previous test run execution.
+         *
+         * This allows us to:
+         * 1. Merge and upload test reports regardless of the test execution status
+         * 2. Preserve the original test run exit code
+         * 3. Report the test status back to CI
+         */
+        const exitCode = parseInt(exitCodeArg || '0', 10)
+        process.exit(exitCode)
+    }
+}
+
+createTestReport()