Support for excluding directories. Tweaks to cache jar namespaces (#701)

* Support for excluding directories. Tweaks to cache jar namespaces Signed-off-by: Prabhu Subramanian <[email protected]> Bump version Signed-off-by: Prabhu Subramanian <[email protected]> New reachables test Signed-off-by: Prabhu Subramanian <[email protected]> --------- Signed-off-by: Prabhu Subramanian <[email protected]>
CycloneDX · Nov 8, 2023 · ee16607 · ee16607
1 parent 1f7794b
commit ee16607
Show file tree

Hide file tree

Showing 10 changed files with 461 additions and 196 deletions.
diff --git a/.github/workflows/java-reachables-test.yml b/.github/workflows/java-reachables-test.yml
@@ -0,0 +1,46 @@
+name: Reachables tests
+
+on:
+  pull_request:
+  workflow_dispatch:
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        node-version: ['21.x']
+        os: ['ubuntu-latest']
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '19'
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: npm install, build
+        run: |
+          npm install
+          npm run build --if-present
+          mkdir -p repotests
+          mkdir -p bomresults
+      - uses: actions/checkout@v4
+        with:
+          repository: 'DependencyTrack/dependency-track'
+          path: 'repotests/dependency-track'
+      - name: compile
+        run: |
+          cd repotests/dependency-track
+          mvn clean compile -DskipTests -Dmaven.test.skip=true
+      - name: repotests
+        run: |         
+          bin/cdxgen.js -p -t java --profile research -o repotests/dependency-track/bom.json repotests/dependency-track
+          cp -rf repotests/dependency-track/*.json *.slices.json bomresults/
+      - uses: actions/upload-artifact@v3
+        with:
+          name: bomresults
+          path: bomresults
diff --git a/bin/cdxgen.js b/bin/cdxgen.js
@@ -150,6 +150,7 @@ const args = yargs(hideBin(process.argv))
   })
   .option("install-deps", {
     type: "boolean",
+    hidden: true,
     default: true,
     description:
       "Install dependencies automatically for some projects. Defaults to true but disabled for containers and oci scans. Use --no-install-deps to disable this feature."
@@ -215,10 +216,15 @@ const args = yargs(hideBin(process.argv))
       "generic"
     ]
   })
+  .option("exclude", {
+    description: "Additional glob pattern(s) to ignore",
+    hidden: true
+  })
   .completion("completion", "Generate bash/zsh completion")
   .array("filter")
   .array("only")
   .array("author")
+  .array("exclude")
   .option("auto-compositions", {
     type: "boolean",
     default: true,

diff --git a/docs/LESSON1.md b/docs/LESSON1.md
@@ -0,0 +1,41 @@
+# Create an SBOM with reachable evidence
+
+## Learning Objective
+
+In this lesson, we will learn about generating an SBOM with reachable evidence for Dependency-Track, a Java application.
+
+## Pre-requisites
+
+Ensure the following tools are installed.
+
+```
+Java >= 17
+Maven
+Node.js > 18
+```
+
+## Getting started
+
+Install cdxgen
+
+```shell
+sudo npm install -g @cyclonedx/cdxgen
+```
+
+Clone and compile dependency track
+
+```shell
+git clone https://github.com/DependencyTrack/dependency-track
+cd dependency-track
+mvn clean compile -P clean-exclude-wars -P enhance -P embedded-jetty -DskipTests
+```
+
+Create SBOM with the research profile
+
+```shell
+cd dependency-track
+# Takes around 5 mins
+cdxgen -o bom.json -t java --profile research . -p
+```
+
+The resulting BOM file would include components with the occurrence and call stack evidence.
diff --git a/docs/LESSON2.md b/docs/LESSON2.md
@@ -0,0 +1,38 @@
+# Create an SBOM with reachable evidence
+
+## Learning Objective
+
+In this lesson, we will learn about generating an SBOM with reachable evidence for Dependency-Track frontend, a JavaScript application.
+
+## Pre-requisites
+
+Ensure the following tools are installed.
+
+```
+Java >= 17
+Node.js > 18
+```
+
+## Getting started
+
+Install cdxgen
+
+```shell
+sudo npm install -g @cyclonedx/cdxgen
+```
+
+Clone
+
+```shell
+git clone https://github.com/DependencyTrack/frontend
+```
+
+Create SBOM with the research profile
+
+```shell
+cd frontend
+# Takes around 5 mins
+cdxgen -o bom.json -t js --profile research . -p
+```
+
+The resulting BOM file would include components with the occurrence and call stack evidence.
diff --git a/docs/_sidebar.md b/docs/_sidebar.md
@@ -3,4 +3,6 @@
 - [Server Usage](SERVER.md)
 - [Configuration](ENV.md)
 - [Advanced Usage](ADVANCED.md)
+- [Tutorials - Java](LESSON1.md)
+- [Tutorials - JavaScript](LESSON2.md)
 - [Enterprise Support](SUPPORT.md)
diff --git a/evinser.js b/evinser.js
@@ -7,7 +7,7 @@ import {
   collectMvnDependencies
 } from "./utils.js";
 import { tmpdir } from "node:os";
-import path, { basename } from "node:path";
+import path from "node:path";
 import fs from "node:fs";
 import * as db from "./db.js";
 import { PackageURL } from "packageurl-js";
@@ -94,15 +94,30 @@ export const catalogMavenDeps = async (
   Namespaces,
   options = {}
 ) => {
-  console.log("About to collect jar dependencies for the path", dirPath);
-  const mavenCmd = getMavenCommand(dirPath, dirPath);
-  // collect all jars including from the cache if data-flow mode is enabled
-  const jarNSMapping = collectMvnDependencies(
-    mavenCmd,
-    dirPath,
-    false,
-    options.withDeepJarCollector
-  );
+  let jarNSMapping = undefined;
+  if (fs.existsSync(path.join(dirPath, "bom.json.map"))) {
+    try {
+      const mapData = JSON.parse(
+        fs.readFileSync(path.join(dirPath, "bom.json.map"))
+      );
+      if (mapData && Object.keys(mapData).length) {
+        jarNSMapping = mapData;
+      }
+    } catch (err) {
+      // ignore
+    }
+  }
+  if (!jarNSMapping) {
+    console.log("About to collect jar dependencies for the path", dirPath);
+    const mavenCmd = getMavenCommand(dirPath, dirPath);
+    // collect all jars including from the cache if data-flow mode is enabled
+    jarNSMapping = collectMvnDependencies(
+      mavenCmd,
+      dirPath,
+      false,
+      options.withDeepJarCollector
+    );
+  }
   if (jarNSMapping) {
     for (const purl of Object.keys(jarNSMapping)) {
       purlsJars[purl] = jarNSMapping[purl].jarFile;
@@ -317,9 +332,6 @@ export const analyzeProject = async (dbObjMap, options) => {
     if (retMap && retMap.slicesFile && fs.existsSync(retMap.slicesFile)) {
       usageSlice = JSON.parse(fs.readFileSync(retMap.slicesFile, "utf-8"));
       usagesSlicesFile = retMap.slicesFile;
-      console.log(
-        `To speed up this step, cache ${usagesSlicesFile} and invoke evinse with the --usages-slices-file argument.`
-      );
     }
   }
   if (usageSlice && Object.keys(usageSlice).length) {
@@ -349,9 +361,6 @@ export const analyzeProject = async (dbObjMap, options) => {
       if (retMap && retMap.slicesFile && fs.existsSync(retMap.slicesFile)) {
         dataFlowSlicesFile = retMap.slicesFile;
         dataFlowSlice = JSON.parse(fs.readFileSync(retMap.slicesFile, "utf-8"));
-        console.log(
-          `To speed up this step, cache ${dataFlowSlicesFile} and invoke evinse with the --data-flow-slices-file argument.`
-        );
       }
     }
   }
@@ -381,9 +390,6 @@ export const analyzeProject = async (dbObjMap, options) => {
         reachablesSlice = JSON.parse(
           fs.readFileSync(retMap.slicesFile, "utf-8")
         );
-        console.log(
-          `To speed up this step, cache ${reachablesSlicesFile} and invoke evinse with the --reachables-slices-file argument.`
-        );
       }
     }
   }
@@ -783,7 +789,7 @@ export const detectServicesFromUDT = (
           const endpoints = extractEndpoints(language, fields[0].name);
           let serviceName = "service";
           if (audt.fileName) {
-            serviceName = `${basename(
+            serviceName = `${path.basename(
               audt.fileName.replace(".py", "")
             )}-service`;
           }