diff --git a/.github/workflows/java-reachables-test.yml b/.github/workflows/java-reachables-test.yml new file mode 100644 index 000000000..17d746115 --- /dev/null +++ b/.github/workflows/java-reachables-test.yml @@ -0,0 +1,46 @@ +name: Reachables tests + +on: + pull_request: + workflow_dispatch: +jobs: + build: + strategy: + fail-fast: false + matrix: + node-version: ['21.x'] + os: ['ubuntu-latest'] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Set up JDK + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '19' + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + - name: npm install, build + run: | + npm install + npm run build --if-present + mkdir -p repotests + mkdir -p bomresults + - uses: actions/checkout@v4 + with: + repository: 'DependencyTrack/dependency-track' + path: 'repotests/dependency-track' + - name: compile + run: | + cd repotests/dependency-track + mvn clean compile -DskipTests -Dmaven.test.skip=true + - name: repotests + run: | + bin/cdxgen.js -p -t java --profile research -o repotests/dependency-track/bom.json repotests/dependency-track + cp -rf repotests/dependency-track/*.json *.slices.json bomresults/ + - uses: actions/upload-artifact@v3 + with: + name: bomresults + path: bomresults diff --git a/bin/cdxgen.js b/bin/cdxgen.js index dd8646e8c..8990de589 100755 --- a/bin/cdxgen.js +++ b/bin/cdxgen.js @@ -150,6 +150,7 @@ const args = yargs(hideBin(process.argv)) }) .option("install-deps", { type: "boolean", + hidden: true, default: true, description: "Install dependencies automatically for some projects. Defaults to true but disabled for containers and oci scans. Use --no-install-deps to disable this feature." @@ -215,10 +216,15 @@ const args = yargs(hideBin(process.argv)) "generic" ] }) + .option("exclude", { + description: "Additional glob pattern(s) to ignore", + hidden: true + }) .completion("completion", "Generate bash/zsh completion") .array("filter") .array("only") .array("author") + .array("exclude") .option("auto-compositions", { type: "boolean", default: true, diff --git a/docs/LESSON1.md b/docs/LESSON1.md new file mode 100644 index 000000000..16db7b2cd --- /dev/null +++ b/docs/LESSON1.md @@ -0,0 +1,41 @@ +# Create an SBOM with reachable evidence + +## Learning Objective + +In this lesson, we will learn about generating an SBOM with reachable evidence for Dependency-Track, a Java application. + +## Pre-requisites + +Ensure the following tools are installed. + +``` +Java >= 17 +Maven +Node.js > 18 +``` + +## Getting started + +Install cdxgen + +```shell +sudo npm install -g @cyclonedx/cdxgen +``` + +Clone and compile dependency track + +```shell +git clone https://github.com/DependencyTrack/dependency-track +cd dependency-track +mvn clean compile -P clean-exclude-wars -P enhance -P embedded-jetty -DskipTests +``` + +Create SBOM with the research profile + +```shell +cd dependency-track +# Takes around 5 mins +cdxgen -o bom.json -t java --profile research . -p +``` + +The resulting BOM file would include components with the occurrence and call stack evidence. diff --git a/docs/LESSON2.md b/docs/LESSON2.md new file mode 100644 index 000000000..e144d6ea7 --- /dev/null +++ b/docs/LESSON2.md @@ -0,0 +1,38 @@ +# Create an SBOM with reachable evidence + +## Learning Objective + +In this lesson, we will learn about generating an SBOM with reachable evidence for Dependency-Track frontend, a JavaScript application. + +## Pre-requisites + +Ensure the following tools are installed. + +``` +Java >= 17 +Node.js > 18 +``` + +## Getting started + +Install cdxgen + +```shell +sudo npm install -g @cyclonedx/cdxgen +``` + +Clone + +```shell +git clone https://github.com/DependencyTrack/frontend +``` + +Create SBOM with the research profile + +```shell +cd frontend +# Takes around 5 mins +cdxgen -o bom.json -t js --profile research . -p +``` + +The resulting BOM file would include components with the occurrence and call stack evidence. diff --git a/docs/_sidebar.md b/docs/_sidebar.md index b1bc7f3e0..71ba09846 100644 --- a/docs/_sidebar.md +++ b/docs/_sidebar.md @@ -3,4 +3,6 @@ - [Server Usage](SERVER.md) - [Configuration](ENV.md) - [Advanced Usage](ADVANCED.md) +- [Tutorials - Java](LESSON1.md) +- [Tutorials - JavaScript](LESSON2.md) - [Enterprise Support](SUPPORT.md) diff --git a/evinser.js b/evinser.js index 946d547ba..5cbafb143 100644 --- a/evinser.js +++ b/evinser.js @@ -7,7 +7,7 @@ import { collectMvnDependencies } from "./utils.js"; import { tmpdir } from "node:os"; -import path, { basename } from "node:path"; +import path from "node:path"; import fs from "node:fs"; import * as db from "./db.js"; import { PackageURL } from "packageurl-js"; @@ -94,15 +94,30 @@ export const catalogMavenDeps = async ( Namespaces, options = {} ) => { - console.log("About to collect jar dependencies for the path", dirPath); - const mavenCmd = getMavenCommand(dirPath, dirPath); - // collect all jars including from the cache if data-flow mode is enabled - const jarNSMapping = collectMvnDependencies( - mavenCmd, - dirPath, - false, - options.withDeepJarCollector - ); + let jarNSMapping = undefined; + if (fs.existsSync(path.join(dirPath, "bom.json.map"))) { + try { + const mapData = JSON.parse( + fs.readFileSync(path.join(dirPath, "bom.json.map")) + ); + if (mapData && Object.keys(mapData).length) { + jarNSMapping = mapData; + } + } catch (err) { + // ignore + } + } + if (!jarNSMapping) { + console.log("About to collect jar dependencies for the path", dirPath); + const mavenCmd = getMavenCommand(dirPath, dirPath); + // collect all jars including from the cache if data-flow mode is enabled + jarNSMapping = collectMvnDependencies( + mavenCmd, + dirPath, + false, + options.withDeepJarCollector + ); + } if (jarNSMapping) { for (const purl of Object.keys(jarNSMapping)) { purlsJars[purl] = jarNSMapping[purl].jarFile; @@ -317,9 +332,6 @@ export const analyzeProject = async (dbObjMap, options) => { if (retMap && retMap.slicesFile && fs.existsSync(retMap.slicesFile)) { usageSlice = JSON.parse(fs.readFileSync(retMap.slicesFile, "utf-8")); usagesSlicesFile = retMap.slicesFile; - console.log( - `To speed up this step, cache ${usagesSlicesFile} and invoke evinse with the --usages-slices-file argument.` - ); } } if (usageSlice && Object.keys(usageSlice).length) { @@ -349,9 +361,6 @@ export const analyzeProject = async (dbObjMap, options) => { if (retMap && retMap.slicesFile && fs.existsSync(retMap.slicesFile)) { dataFlowSlicesFile = retMap.slicesFile; dataFlowSlice = JSON.parse(fs.readFileSync(retMap.slicesFile, "utf-8")); - console.log( - `To speed up this step, cache ${dataFlowSlicesFile} and invoke evinse with the --data-flow-slices-file argument.` - ); } } } @@ -381,9 +390,6 @@ export const analyzeProject = async (dbObjMap, options) => { reachablesSlice = JSON.parse( fs.readFileSync(retMap.slicesFile, "utf-8") ); - console.log( - `To speed up this step, cache ${reachablesSlicesFile} and invoke evinse with the --reachables-slices-file argument.` - ); } } } @@ -783,7 +789,7 @@ export const detectServicesFromUDT = ( const endpoints = extractEndpoints(language, fields[0].name); let serviceName = "service"; if (audt.fileName) { - serviceName = `${basename( + serviceName = `${path.basename( audt.fileName.replace(".py", "") )}-service`; } diff --git a/index.js b/index.js index 8d20f219c..f89a044d4 100644 --- a/index.js +++ b/index.js @@ -1063,12 +1063,14 @@ export const createJarBom = async (path, options) => { } else { jarFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "*.[jw]ar" + (options.multiProject ? "**/" : "") + "*.[jw]ar", + options ); // Jenkins plugins const hpiFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "*.hpi" + (options.multiProject ? "**/" : "") + "*.hpi", + options ); if (hpiFiles.length) { jarFiles = jarFiles.concat(hpiFiles); @@ -1143,7 +1145,8 @@ export const createJavaBom = async (path, options) => { // maven - pom.xml const pomFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "pom.xml" + (options.multiProject ? "**/" : "") + "pom.xml", + options ); let bomJsonFiles = []; if ( @@ -1179,15 +1182,15 @@ export const createJavaBom = async (path, options) => { const mavenCmd = getMavenCommand(basePath, path); // Should we attempt to resolve class names if (options.resolveClass || options.deep) { - console.log( - "Creating class names list based on available jars. This might take a few mins ..." - ); - jarNSMapping = collectMvnDependencies( + const tmpjarNSMapping = collectMvnDependencies( mavenCmd, basePath, true, false ); + if (tmpjarNSMapping && Object.keys(tmpjarNSMapping).length) { + jarNSMapping = { ...jarNSMapping, ...tmpjarNSMapping }; + } } console.log( `Executing '${mavenCmd} ${mvnArgs.join(" ")}' in`, @@ -1202,10 +1205,10 @@ export const createJavaBom = async (path, options) => { }); // Check if the cyclonedx plugin created the required bom.xml file // Sometimes the plugin fails silently for complex maven projects - bomJsonFiles = getAllFiles(path, "**/target/*.json"); + bomJsonFiles = getAllFiles(path, "**/target/*.json", options); // Check if the bom json files got created in a directory other than target if (!bomJsonFiles.length) { - bomJsonFiles = getAllFiles(path, "**/bom*.json"); + bomJsonFiles = getAllFiles(path, "**/bom*.json", options); } const bomGenerated = bomJsonFiles.length; if (!bomGenerated || result.status !== 0 || result.error) { @@ -1296,7 +1299,7 @@ export const createJavaBom = async (path, options) => { } } } // for - const bomFiles = getAllFiles(path, "**/target/bom.xml"); + const bomFiles = getAllFiles(path, "**/target/bom.xml", options); for (const abjson of bomJsonFiles) { let bomJsonObj = undefined; try { @@ -1359,7 +1362,8 @@ export const createJavaBom = async (path, options) => { // gradle const gradleFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "build.gradle*" + (options.multiProject ? "**/" : "") + "build.gradle*", + options ); const allProjects = []; const allProjectsAddedPurls = []; @@ -1541,9 +1545,6 @@ export const createJavaBom = async (path, options) => { } // Should we attempt to resolve class names if (options.resolveClass || options.deep) { - console.log( - "Creating class names list based on available jars. This might take a few mins ..." - ); jarNSMapping = collectJarNS(GRADLE_CACHE_DIR); } pkgList = await getMvnMetadata(pkgList, jarNSMapping); @@ -1558,7 +1559,7 @@ export const createJavaBom = async (path, options) => { // Bazel // Look for the BUILD file only in the root directory - const bazelFiles = getAllFiles(path, "BUILD"); + const bazelFiles = getAllFiles(path, "BUILD", options); if ( bazelFiles && bazelFiles.length && @@ -1665,7 +1666,8 @@ export const createJavaBom = async (path, options) => { let sbtProjectFiles = getAllFiles( path, (options.multiProject ? "**/" : "") + - "project/{build.properties,*.sbt,*.scala}" + "project/{build.properties,*.sbt,*.scala}", + options ); let sbtProjects = []; @@ -1680,7 +1682,8 @@ export const createJavaBom = async (path, options) => { if (!sbtProjects.length) { sbtProjectFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "*.sbt" + (options.multiProject ? "**/" : "") + "*.sbt", + options ); for (const i in sbtProjectFiles) { const baseDir = dirname(sbtProjectFiles[i]); @@ -1693,7 +1696,8 @@ export const createJavaBom = async (path, options) => { ); const sbtLockFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "build.sbt.lock" + (options.multiProject ? "**/" : "") + "build.sbt.lock", + options ); if (sbtProjects && sbtProjects.length) { @@ -1827,9 +1831,6 @@ export const createJavaBom = async (path, options) => { } // Should we attempt to resolve class names if (options.resolveClass || options.deep) { - console.log( - "Creating class names list based on available jars. This might take a few mins ..." - ); jarNSMapping = collectJarNS(SBT_CACHE_DIR); } pkgList = await getMvnMetadata(pkgList, jarNSMapping); @@ -1859,7 +1860,7 @@ export const createNodejsBom = async (path, options) => { let ppurl = ""; // Docker mode requires special handling if (["docker", "oci", "os"].includes(options.projectType)) { - const pkgJsonFiles = getAllFiles(path, "**/package.json"); + const pkgJsonFiles = getAllFiles(path, "**/package.json", options); // Are there any package.json files in the container? if (pkgJsonFiles.length) { for (const pj of pkgJsonFiles) { @@ -1890,30 +1891,36 @@ export const createNodejsBom = async (path, options) => { } const yarnLockFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "yarn.lock" + (options.multiProject ? "**/" : "") + "yarn.lock", + options ); const shrinkwrapFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "npm-shrinkwrap.json" + (options.multiProject ? "**/" : "") + "npm-shrinkwrap.json", + options ); let pkgLockFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "package-lock.json" + (options.multiProject ? "**/" : "") + "package-lock.json", + options ); if (shrinkwrapFiles.length) { pkgLockFiles = pkgLockFiles.concat(shrinkwrapFiles); } const pnpmLockFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "pnpm-lock.yaml" + (options.multiProject ? "**/" : "") + "pnpm-lock.yaml", + options ); const minJsFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "*min.js" + (options.multiProject ? "**/" : "") + "*min.js", + options ); const bowerFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "bower.json" + (options.multiProject ? "**/" : "") + "bower.json", + options ); // Parse min js files if (minJsFiles && minJsFiles.length) { @@ -2179,7 +2186,8 @@ export const createNodejsBom = async (path, options) => { if (!pkgList.length && existsSync(join(path, "node_modules"))) { const pkgJsonFiles = getAllFiles( join(path, "node_modules"), - "**/package.json" + "**/package.json", + options ); manifestFiles = manifestFiles.concat(pkgJsonFiles); for (const pkgjf of pkgJsonFiles) { @@ -2241,37 +2249,44 @@ export const createPythonBom = async (path, options) => { const pipenvMode = existsSync(join(path, "Pipfile")); let poetryFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "poetry.lock" + (options.multiProject ? "**/" : "") + "poetry.lock", + options ); const pdmLockFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "pdm.lock" + (options.multiProject ? "**/" : "") + "pdm.lock", + options ); if (pdmLockFiles && pdmLockFiles.length) { poetryFiles = poetryFiles.concat(pdmLockFiles); } let reqFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "*requirements*.txt" + (options.multiProject ? "**/" : "") + "*requirements*.txt", + options ); reqFiles = reqFiles.filter( (f) => !f.includes(join("mercurial", "helptext", "internals")) ); const reqDirFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "requirements/*.txt" + (options.multiProject ? "**/" : "") + "requirements/*.txt", + options ); const metadataFiles = getAllFiles( path, - (options.multiProject ? "**/site-packages/**/" : "") + "METADATA" + (options.multiProject ? "**/site-packages/**/" : "") + "METADATA", + options ); const whlFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "*.whl" + (options.multiProject ? "**/" : "") + "*.whl", + options ); const eggInfoFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "*.egg-info" + (options.multiProject ? "**/" : "") + "*.egg-info", + options ); const setupPy = join(path, "setup.py"); const pyProjectFile = join(path, "pyproject.toml"); @@ -2609,7 +2624,8 @@ export const createGoBom = async (path, options) => { // Read in go.sum and merge all go.sum files. const gosumFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "go.sum" + (options.multiProject ? "**/" : "") + "go.sum", + options ); // If USE_GOSUM is true|1, generate BOM components only using go.sum. @@ -2723,13 +2739,15 @@ export const createGoBom = async (path, options) => { // Read in data from Gopkg.lock files if they exist const gopkgLockFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "Gopkg.lock" + (options.multiProject ? "**/" : "") + "Gopkg.lock", + options ); // Read in go.mod files and parse BOM components with checksums from gosumData const gomodFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "go.mod" + (options.multiProject ? "**/" : "") + "go.mod", + options ); if (gomodFiles.length) { let shouldManuallyParse = false; @@ -2925,11 +2943,13 @@ export const createRustBom = async (path, options) => { } let cargoLockFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "Cargo.lock" + (options.multiProject ? "**/" : "") + "Cargo.lock", + options ); const cargoFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "Cargo.toml" + (options.multiProject ? "**/" : "") + "Cargo.toml", + options ); const cargoMode = cargoFiles.length; const cargoLockMode = cargoLockFiles.length; @@ -2952,7 +2972,8 @@ export const createRustBom = async (path, options) => { // Get the new lock files cargoLockFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "Cargo.lock" + (options.multiProject ? "**/" : "") + "Cargo.lock", + options ); if (cargoLockFiles.length) { for (const f of cargoLockFiles) { @@ -2982,11 +3003,13 @@ export const createRustBom = async (path, options) => { export const createDartBom = async (path, options) => { const pubFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "pubspec.lock" + (options.multiProject ? "**/" : "") + "pubspec.lock", + options ); const pubSpecYamlFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "pubspec.yaml" + (options.multiProject ? "**/" : "") + "pubspec.yaml", + options ); let pkgList = []; if (pubFiles.length) { @@ -3036,26 +3059,34 @@ export const createCppBom = (path, options) => { const addedParentComponentsMap = {}; const conanLockFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "conan.lock" + (options.multiProject ? "**/" : "") + "conan.lock", + options ); const conanFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "conanfile.txt" + (options.multiProject ? "**/" : "") + "conanfile.txt", + options ); let cmakeLikeFiles = []; const mesonBuildFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "meson.build" + (options.multiProject ? "**/" : "") + "meson.build", + options ); if (mesonBuildFiles && mesonBuildFiles.length) { cmakeLikeFiles = cmakeLikeFiles.concat(mesonBuildFiles); } cmakeLikeFiles = cmakeLikeFiles.concat( - getAllFiles(path, (options.multiProject ? "**/" : "") + "CMakeLists.txt") + getAllFiles( + path, + (options.multiProject ? "**/" : "") + "CMakeLists.txt", + options + ) ); const cmakeFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "*.cmake" + (options.multiProject ? "**/" : "") + "*.cmake", + options ); if (cmakeFiles && cmakeFiles.length) { cmakeLikeFiles = cmakeLikeFiles.concat(cmakeFiles); @@ -3193,11 +3224,13 @@ export const createCppBom = (path, options) => { export const createClojureBom = (path, options) => { const ednFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "deps.edn" + (options.multiProject ? "**/" : "") + "deps.edn", + options ); const leinFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "project.clj" + (options.multiProject ? "**/" : "") + "project.clj", + options ); let pkgList = []; if (leinFiles.length) { @@ -3313,7 +3346,8 @@ export const createClojureBom = (path, options) => { export const createHaskellBom = (path, options) => { const cabalFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "cabal.project.freeze" + (options.multiProject ? "**/" : "") + "cabal.project.freeze", + options ); let pkgList = []; if (cabalFiles.length) { @@ -3344,7 +3378,8 @@ export const createHaskellBom = (path, options) => { export const createElixirBom = (path, options) => { const mixFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "mix.lock" + (options.multiProject ? "**/" : "") + "mix.lock", + options ); let pkgList = []; if (mixFiles.length) { @@ -3373,7 +3408,11 @@ export const createElixirBom = (path, options) => { * @param options Parse options from the cli */ export const createGitHubBom = (path, options) => { - const ghactionFiles = getAllFiles(path, ".github/workflows/" + "*.yml"); + const ghactionFiles = getAllFiles( + path, + ".github/workflows/" + "*.yml", + options + ); let pkgList = []; if (ghactionFiles.length) { for (const f of ghactionFiles) { @@ -3401,7 +3440,7 @@ export const createGitHubBom = (path, options) => { * @param options Parse options from the cli */ export const createCloudBuildBom = (path, options) => { - const cbFiles = getAllFiles(path, "cloudbuild.yml"); + const cbFiles = getAllFiles(path, "cloudbuild.yml", options); let pkgList = []; if (cbFiles.length) { for (const f of cbFiles) { @@ -3492,7 +3531,8 @@ export const createJenkinsBom = async (path, options) => { let pkgList = []; const hpiFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "*.hpi" + (options.multiProject ? "**/" : "") + "*.hpi", + options ); const tempDir = mkdtempSync(join(tmpdir(), "hpi-deps-")); if (hpiFiles.length) { @@ -3506,7 +3546,7 @@ export const createJenkinsBom = async (path, options) => { } } } - const jsFiles = getAllFiles(tempDir, "**/*.js"); + const jsFiles = getAllFiles(tempDir, "**/*.js", options); if (jsFiles.length) { for (const f of jsFiles) { if (DEBUG_MODE) { @@ -3540,7 +3580,8 @@ export const createHelmBom = (path, options) => { let pkgList = []; const yamlFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "*.yaml" + (options.multiProject ? "**/" : "") + "*.yaml", + options ); if (yamlFiles.length) { for (const f of yamlFiles) { @@ -3570,11 +3611,13 @@ export const createHelmBom = (path, options) => { export const createSwiftBom = (path, options) => { const swiftFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "Package*.swift" + (options.multiProject ? "**/" : "") + "Package*.swift", + options ); const pkgResolvedFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "Package.resolved" + (options.multiProject ? "**/" : "") + "Package.resolved", + options ); let pkgList = []; let dependencies = []; @@ -3667,19 +3710,23 @@ export const createContainerSpecLikeBom = async (path, options) => { const origProjectType = options.projectType; let dcFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "*.yml" + (options.multiProject ? "**/" : "") + "*.yml", + options ); const yamlFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "*.yaml" + (options.multiProject ? "**/" : "") + "*.yaml", + options ); let oapiFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "open*.json" + (options.multiProject ? "**/" : "") + "open*.json", + options ); const oapiYamlFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "open*.yaml" + (options.multiProject ? "**/" : "") + "open*.yaml", + options ); if (oapiYamlFiles && oapiYamlFiles.length) { oapiFiles = oapiFiles.concat(oapiYamlFiles); @@ -3688,7 +3735,7 @@ export const createContainerSpecLikeBom = async (path, options) => { dcFiles = dcFiles.concat(yamlFiles); } // Privado.ai json files - const privadoFiles = getAllFiles(path, ".privado/" + "*.json"); + const privadoFiles = getAllFiles(path, ".privado/" + "*.json", options); // parse yaml manifest files if (dcFiles.length) { for (const f of dcFiles) { @@ -3942,11 +3989,13 @@ export const createContainerSpecLikeBom = async (path, options) => { export const createPHPBom = (path, options) => { const composerJsonFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "composer.json" + (options.multiProject ? "**/" : "") + "composer.json", + options ); let composerLockFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "composer.lock" + (options.multiProject ? "**/" : "") + "composer.lock", + options ); let pkgList = []; const composerJsonMode = composerJsonFiles.length; @@ -4002,7 +4051,8 @@ export const createPHPBom = (path, options) => { } composerLockFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "composer.lock" + (options.multiProject ? "**/" : "") + "composer.lock", + options ); if (composerLockFiles.length) { for (const f of composerLockFiles) { @@ -4031,11 +4081,13 @@ export const createPHPBom = (path, options) => { export const createRubyBom = async (path, options) => { const gemFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "Gemfile" + (options.multiProject ? "**/" : "") + "Gemfile", + options ); let gemLockFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "Gemfile.lock" + (options.multiProject ? "**/" : "") + "Gemfile.lock", + options ); let pkgList = []; const gemFileMode = gemFiles.length; @@ -4059,7 +4111,8 @@ export const createRubyBom = async (path, options) => { } gemLockFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "Gemfile.lock" + (options.multiProject ? "**/" : "") + "Gemfile.lock", + options ); if (gemLockFiles.length) { for (const f of gemLockFiles) { @@ -4096,27 +4149,33 @@ export const createCsharpBom = async ( let dependencies = []; const csProjFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "*.csproj" + (options.multiProject ? "**/" : "") + "*.csproj", + options ); const pkgConfigFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "packages.config" + (options.multiProject ? "**/" : "") + "packages.config", + options ); const projAssetsFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "project.assets.json" + (options.multiProject ? "**/" : "") + "project.assets.json", + options ); const pkgLockFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "packages.lock.json" + (options.multiProject ? "**/" : "") + "packages.lock.json", + options ); const paketLockFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "paket.lock" + (options.multiProject ? "**/" : "") + "paket.lock", + options ); const nupkgFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "*.nupkg" + (options.multiProject ? "**/" : "") + "*.nupkg", + options ); let pkgList = []; if (nupkgFiles.length && projAssetsFiles.length === 0) { @@ -4891,17 +4950,20 @@ export const createXBom = async (path, options) => { // maven - pom.xml const pomFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "pom.xml" + (options.multiProject ? "**/" : "") + "pom.xml", + options ); // gradle const gradleFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "build.gradle*" + (options.multiProject ? "**/" : "") + "build.gradle*", + options ); // scala sbt const sbtFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "{build.sbt,Build.scala}*" + (options.multiProject ? "**/" : "") + "{build.sbt,Build.scala}*", + options ); if (pomFiles.length || gradleFiles.length || sbtFiles.length) { return await createJavaBom(path, options); @@ -4916,17 +4978,20 @@ export const createXBom = async (path, options) => { } const reqFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "*requirements*.txt" + (options.multiProject ? "**/" : "") + "*requirements*.txt", + options ); const reqDirFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "requirements/*.txt" + (options.multiProject ? "**/" : "") + "requirements/*.txt", + options ); const requirementsMode = (reqFiles && reqFiles.length) || (reqDirFiles && reqDirFiles.length); const whlFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "*.whl" + (options.multiProject ? "**/" : "") + "*.whl", + options ); if (requirementsMode || whlFiles.length) { return await createPythonBom(path, options); @@ -4934,15 +4999,18 @@ export const createXBom = async (path, options) => { // go const gosumFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "go.sum" + (options.multiProject ? "**/" : "") + "go.sum", + options ); const gomodFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "go.mod" + (options.multiProject ? "**/" : "") + "go.mod", + options ); const gopkgLockFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "Gopkg.lock" + (options.multiProject ? "**/" : "") + "Gopkg.lock", + options ); if (gomodFiles.length || gosumFiles.length || gopkgLockFiles.length) { return await createGoBom(path, options); @@ -4951,11 +5019,13 @@ export const createXBom = async (path, options) => { // rust const cargoLockFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "Cargo.lock" + (options.multiProject ? "**/" : "") + "Cargo.lock", + options ); const cargoFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "Cargo.toml" + (options.multiProject ? "**/" : "") + "Cargo.toml", + options ); if (cargoLockFiles.length || cargoFiles.length) { return await createRustBom(path, options); @@ -4964,11 +5034,13 @@ export const createXBom = async (path, options) => { // php const composerJsonFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "composer.json" + (options.multiProject ? "**/" : "") + "composer.json", + options ); const composerLockFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "composer.lock" + (options.multiProject ? "**/" : "") + "composer.lock", + options ); if (composerJsonFiles.length || composerLockFiles.length) { return createPHPBom(path, options); @@ -4977,11 +5049,13 @@ export const createXBom = async (path, options) => { // Ruby const gemFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "Gemfile" + (options.multiProject ? "**/" : "") + "Gemfile", + options ); const gemLockFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "Gemfile.lock" + (options.multiProject ? "**/" : "") + "Gemfile.lock", + options ); if (gemFiles.length || gemLockFiles.length) { return await createRubyBom(path, options); @@ -4990,7 +5064,8 @@ export const createXBom = async (path, options) => { // .Net const csProjFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "*.csproj" + (options.multiProject ? "**/" : "") + "*.csproj", + options ); if (csProjFiles.length) { return await createCsharpBom(path, options); @@ -4999,11 +5074,13 @@ export const createXBom = async (path, options) => { // Dart const pubFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "pubspec.lock" + (options.multiProject ? "**/" : "") + "pubspec.lock", + options ); const pubSpecFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "pubspec.yaml" + (options.multiProject ? "**/" : "") + "pubspec.yaml", + options ); if (pubFiles.length || pubSpecFiles.length) { return await createDartBom(path, options); @@ -5012,7 +5089,8 @@ export const createXBom = async (path, options) => { // Haskell const hackageFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "cabal.project.freeze" + (options.multiProject ? "**/" : "") + "cabal.project.freeze", + options ); if (hackageFiles.length) { return createHaskellBom(path, options); @@ -5021,7 +5099,8 @@ export const createXBom = async (path, options) => { // Elixir const mixFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "mix.lock" + (options.multiProject ? "**/" : "") + "mix.lock", + options ); if (mixFiles.length) { return createElixirBom(path, options); @@ -5030,19 +5109,23 @@ export const createXBom = async (path, options) => { // cpp const conanLockFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "conan.lock" + (options.multiProject ? "**/" : "") + "conan.lock", + options ); const conanFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "conanfile.txt" + (options.multiProject ? "**/" : "") + "conanfile.txt", + options ); const cmakeListFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "CMakeLists.txt" + (options.multiProject ? "**/" : "") + "CMakeLists.txt", + options ); const mesonBuildFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "meson.build" + (options.multiProject ? "**/" : "") + "meson.build", + options ); if ( conanLockFiles.length || @@ -5056,18 +5139,24 @@ export const createXBom = async (path, options) => { // clojure const ednFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "deps.edn" + (options.multiProject ? "**/" : "") + "deps.edn", + options ); const leinFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "project.clj" + (options.multiProject ? "**/" : "") + "project.clj", + options ); if (ednFiles.length || leinFiles.length) { return createClojureBom(path, options); } // GitHub actions - const ghactionFiles = getAllFiles(path, ".github/workflows/" + "*.yml"); + const ghactionFiles = getAllFiles( + path, + ".github/workflows/" + "*.yml", + options + ); if (ghactionFiles.length) { return createGitHubBom(path, options); } @@ -5075,7 +5164,8 @@ export const createXBom = async (path, options) => { // Jenkins plugins const hpiFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "*.hpi" + (options.multiProject ? "**/" : "") + "*.hpi", + options ); if (hpiFiles.length) { return await createJenkinsBom(path, options); @@ -5084,11 +5174,13 @@ export const createXBom = async (path, options) => { // Helm charts const chartFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "Chart.yaml" + (options.multiProject ? "**/" : "") + "Chart.yaml", + options ); const yamlFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "values.yaml" + (options.multiProject ? "**/" : "") + "values.yaml", + options ); if (chartFiles.length || yamlFiles.length) { return createHelmBom(path, options); @@ -5097,15 +5189,18 @@ export const createXBom = async (path, options) => { // Docker compose, kubernetes and skaffold const dcFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "docker-compose*.yml" + (options.multiProject ? "**/" : "") + "docker-compose*.yml", + options ); const skFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "skaffold.yaml" + (options.multiProject ? "**/" : "") + "skaffold.yaml", + options ); const deplFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "deployment.yaml" + (options.multiProject ? "**/" : "") + "deployment.yaml", + options ); if (dcFiles.length || skFiles.length || deplFiles.length) { return await createContainerSpecLikeBom(path, options); @@ -5114,7 +5209,8 @@ export const createXBom = async (path, options) => { // Google CloudBuild const cbFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "cloudbuild.yaml" + (options.multiProject ? "**/" : "") + "cloudbuild.yaml", + options ); if (cbFiles.length) { return createCloudBuildBom(path, options); @@ -5123,11 +5219,13 @@ export const createXBom = async (path, options) => { // Swift const swiftFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "Package*.swift" + (options.multiProject ? "**/" : "") + "Package*.swift", + options ); const pkgResolvedFiles = getAllFiles( path, - (options.multiProject ? "**/" : "") + "Package.resolved" + (options.multiProject ? "**/" : "") + "Package.resolved", + options ); if (swiftFiles.length || pkgResolvedFiles.length) { return createSwiftBom(path, options); diff --git a/package-lock.json b/package-lock.json index 0a5e09994..e95f46c6e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,17 +1,17 @@ { "name": "@cyclonedx/cdxgen", - "version": "9.9.2", + "version": "9.9.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@cyclonedx/cdxgen", - "version": "9.9.2", + "version": "9.9.3", "license": "Apache-2.0", "dependencies": { "@babel/parser": "^7.23.0", "@babel/traverse": "^7.23.2", - "@npmcli/arborist": "^7.2.0", + "@npmcli/arborist": "7.2.0", "ajv": "^8.12.0", "ajv-formats": "^2.1.1", "cheerio": "^1.0.0-rc.12", diff --git a/package.json b/package.json index c58455185..98ed8d9c4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@cyclonedx/cdxgen", - "version": "9.9.2", + "version": "9.9.3", "description": "Creates CycloneDX Software Bill of Materials (SBOM) from source or container image", "homepage": "http://github.com/cyclonedx/cdxgen", "author": "Prabhu Subramanian ", @@ -57,7 +57,7 @@ "dependencies": { "@babel/parser": "^7.23.0", "@babel/traverse": "^7.23.2", - "@npmcli/arborist": "^7.2.0", + "@npmcli/arborist": "7.2.0", "ajv": "^8.12.0", "ajv-formats": "^2.1.1", "cheerio": "^1.0.0-rc.12", diff --git a/utils.js b/utils.js index 3681aa3e3..c9bc659a3 100644 --- a/utils.js +++ b/utils.js @@ -107,6 +107,8 @@ export const MAX_BUFFER = // Metadata cache export let metadata_cache = {}; +// Speed up lookup namespaces for a given jar +const jarNSMapping_cache = {}; // Whether test scope shall be included for java/maven projects; default, if unset shall be 'true' export const includeMavenTestScope = @@ -140,20 +142,34 @@ export const cdxgenAgent = got.extend({ * @param {string} dirPath Root directory for search * @param {string} pattern Glob pattern (eg: *.gradle) */ -export const getAllFiles = function (dirPath, pattern) { +export const getAllFiles = function (dirPath, pattern, options = {}) { + let ignoreList = [ + "**/.hg/**", + "**/.git/**", + "**/venv/**", + "**/docs/**", + "**/examples/**", + "**/site-packages/**" + ]; + // Only ignore node_modules if the caller is not looking for package.json + if (!pattern.includes("package.json")) { + ignoreList.push("**/node_modules/**"); + } + if (options && options.exclude && Array.isArray(options.exclude)) { + ignoreList = ignoreList.concat(options.exclude); + } + return getAllFilesWithIgnore(dirPath, pattern, ignoreList); +}; + +/** + * Method to get files matching a pattern + * + * @param {string} dirPath Root directory for search + * @param {string} pattern Glob pattern (eg: *.gradle) + * @param {array} ignoreList Directory patterns to ignore + */ +export const getAllFilesWithIgnore = function (dirPath, pattern, ignoreList) { try { - const ignoreList = [ - "**/.hg/**", - "**/.git/**", - "**/venv/**", - "**/docs/**", - "**/examples/**", - "**/site-packages/**" - ]; - // Only ignore node_modules if the caller is not looking for package.json - if (!pattern.includes("package.json")) { - ignoreList.push("**/node_modules/**"); - } return globSync(pattern, { cwd: dirPath, absolute: true, @@ -6072,7 +6088,7 @@ export const collectMvnDependencies = function ( const MAVEN_CACHE_DIR = process.env.MAVEN_CACHE_DIR || join(homedir(), ".m2", "repository"); const tempDir = mkdtempSync(join(tmpdir(), "mvn-deps-")); - const copyArgs = [ + let copyArgs = [ "dependency:copy-dependencies", `-DoutputDirectory=${tempDir}`, "-U", @@ -6082,6 +6098,10 @@ export const collectMvnDependencies = function ( "-Dmdep.prependGroupId=" + (process.env.MAVEN_PREPEND_GROUP || "false"), "-Dmdep.stripVersion=" + (process.env.MAVEN_STRIP_VERSION || "false") ]; + if (process.env.MVN_ARGS) { + const addArgs = process.env.MVN_ARGS.split(" "); + copyArgs = copyArgs.concat(addArgs); + } if (basePath && basePath !== MAVEN_CACHE_DIR) { console.log(`Executing '${mavenCmd} ${copyArgs.join(" ")}' in ${basePath}`); const result = spawnSync(mavenCmd, copyArgs, { @@ -6282,51 +6302,59 @@ export const collectJarNS = function (jarPath, pomPathMap = {}) { purl = purlObj.toString(); } } - if (DEBUG_MODE) { - console.log(`Executing 'jar tf ${jf}'`); - } - - const jarResult = spawnSync("jar", ["-tf", jf], { - encoding: "utf-8", - shell: isWin, - maxBuffer: 50 * 1024 * 1024, - env - }); - if ( - jarResult && - jarResult.stderr && - jarResult.stderr.includes( - "is not recognized as an internal or external command" - ) - ) { - jarCommandAvailable = false; - console.log( - "jar command is not available in PATH. Ensure JDK >= 17 is installed and set the environment variables JAVA_HOME and PATH to the bin directory inside JAVA_HOME." - ); - } - const consolelines = (jarResult.stdout || "").split("\n"); - const nsList = consolelines - .filter((l) => { - return ( - (l.includes(".class") || - l.includes(".java") || - l.includes(".kt")) && - !l.includes("-INF") && - !l.includes("module-info") - ); - }) - .map((e) => { - return e - .replace("\r", "") - .replace(/.(class|java|kt)/, "") - .replace(/\/$/, "") - .replace(/\//g, "."); + // If we have a hit from the cache, use it. + if (purl && jarNSMapping_cache[purl]) { + jarNSMapping[purl] = jarNSMapping_cache[purl]; + } else { + if (DEBUG_MODE) { + console.log(`Executing 'jar tf ${jf}'`); + } + const jarResult = spawnSync("jar", ["-tf", jf], { + encoding: "utf-8", + shell: isWin, + maxBuffer: 50 * 1024 * 1024, + env }); - jarNSMapping[purl || jf] = { - jarFile: jf, - pom: pomData, - namespaces: nsList - }; + if ( + jarResult && + jarResult.stderr && + jarResult.stderr.includes( + "is not recognized as an internal or external command" + ) + ) { + jarCommandAvailable = false; + console.log( + "jar command is not available in PATH. Ensure JDK >= 17 is installed and set the environment variables JAVA_HOME and PATH to the bin directory inside JAVA_HOME." + ); + } + const consolelines = (jarResult.stdout || "").split("\n"); + const nsList = consolelines + .filter((l) => { + return ( + (l.includes(".class") || + l.includes(".java") || + l.includes(".kt")) && + !l.includes("-INF") && + !l.includes("module-info") + ); + }) + .map((e) => { + return e + .replace("\r", "") + .replace(/.(class|java|kt)/, "") + .replace(/\/$/, "") + .replace(/\//g, "."); + }); + jarNSMapping[purl || jf] = { + jarFile: jf, + pom: pomData, + namespaces: nsList + }; + // Retain in the global cache to speed up future lookups + if (purl) { + jarNSMapping_cache[purl] = jarNSMapping[purl]; + } + } } if (!jarNSMapping) { console.log(`Unable to determine class names for the jars in ${jarPath}`);