Made multithreaded gradle the default and removed singlethreaded code

Signed-off-by: Roland Asmann <[email protected]>
CycloneDX · Sep 22, 2024 · a6b4c93 · a6b4c93
1 parent 93b4cc6
commit a6b4c93
Show file tree

Hide file tree

Showing 3 changed files with 109 additions and 213 deletions.
diff --git a/.github/workflows/repotests.yml b/.github/workflows/repotests.yml
@@ -455,14 +455,12 @@ jobs:
       - name: repotests expo
         run: |
           cd repotests/expo-test && npm ci && cd ../..
-          GRADLE_ARGS_DEPENDENCIES="--configuration releaseRuntimeClasspath"                                                                                   bin/cdxgen.js -p -t gradle repotests/expo-test -o bomresults/bom-expo.json
-          GRADLE_ARGS_DEPENDENCIES="--configuration releaseRuntimeClasspath" GRADLE_MULTI_THREADED=true GRADLE_SKIP_MODULES=root                               bin/cdxgen.js -p -t gradle repotests/expo-test -o bomresults/bom-expo-multi.json
-          custom-json-diff -i bomresults/bom-expo.json bomresults/bom-expo-multi.json -o bomresults/diff-expo
-          GRADLE_ARGS_DEPENDENCIES="--configuration releaseRuntimeClasspath" GRADLE_MULTI_THREADED=true GRADLE_SKIP_MODULES=root GRADLE_RESOLVE_FROM_NODE=true bin/cdxgen.js -p -t gradle repotests/expo-test -o bomresults/bom-expo-npm.json
+          GRADLE_ARGS_DEPENDENCIES="--configuration releaseRuntimeClasspath" GRADLE_SKIP_MODULES=root                               bin/cdxgen.js -p -t gradle repotests/expo-test -o bomresults/bom-expo.json
+          GRADLE_ARGS_DEPENDENCIES="--configuration releaseRuntimeClasspath" GRADLE_SKIP_MODULES=root GRADLE_RESOLVE_FROM_NODE=true bin/cdxgen.js -p -t gradle repotests/expo-test -o bomresults/bom-expo-npm.json
         shell: bash
       - name: repotests elasticsearch
         run: |
-          GRADLE_MULTI_THREADED=true bin/cdxgen.js -p -t gradle repotests/elasticsearch -o bomresults/bom-elasticsearch.json
+          bin/cdxgen.js -p -t gradle repotests/elasticsearch -o bomresults/bom-elasticsearch.json
         shell: bash
       - name: jenkins plugins
         run: |

diff --git a/lib/cli/index.js b/lib/cli/index.js
@@ -1635,7 +1635,7 @@ export async function createJavaBom(path, options) {
     gradleFiles?.length &&
     isPackageManagerAllowed("gradle", ["maven", "bazel", "sbt"], options)
   ) {
-    let retMap = executeGradleProperties(gradleRootPath, null);
+    const retMap = executeGradleProperties(gradleRootPath, null);
     const allProjectsStr = retMap.projects || [];
     const rootProject = retMap.rootProject;
     if (rootProject) {
@@ -1647,50 +1647,31 @@ export async function createJavaBom(path, options) {
     }
     // Get the sub-project properties and set the root dependencies
     if (allProjectsStr?.length) {
-      if (process.env.GRADLE_MULTI_THREADED) {
-        const parallelPropTaskOut = executeParallelGradleProperties(
-          gradleRootPath,
-          allProjectsStr,
-        );
-        const splitPropTaskOut = splitOutputByGradleProjects(
-          parallelPropTaskOut,
-          ["properties"],
-        );
+      const parallelPropTaskOut = executeParallelGradleProperties(
+        gradleRootPath,
+        allProjectsStr,
+      );
+      const splitPropTaskOut = splitOutputByGradleProjects(
+        parallelPropTaskOut,
+        ["properties"],
+      );
 
-        for (const [key, propTaskOut] of splitPropTaskOut.entries()) {
-          const retMap = parseGradleProperties(propTaskOut);
-          const rootSubProject = retMap.rootProject;
-          if (rootSubProject) {
-            const rootSubProjectObj = await buildObjectForGradleModule(
-              rootSubProject.replace(/^:/, ""),
-              retMap.metadata,
-            );
-            if (!allProjectsAddedPurls.includes(rootSubProjectObj["purl"])) {
-              allProjects.push(rootSubProjectObj);
-              rootDependsOn.push(rootSubProjectObj["bom-ref"]);
-              allProjectsAddedPurls.push(rootSubProjectObj["purl"]);
-            }
-            gradleModules.set(key.replace(/^:/, ""), rootSubProjectObj);
-          }
-        }
-      } else {
-        for (const spstr of allProjectsStr) {
-          retMap = executeGradleProperties(gradleRootPath, spstr);
-          const rootSubProject = retMap.rootProject;
-          if (rootSubProject) {
-            const rootSubProjectObj = await buildObjectForGradleModule(
-              rootSubProject.replace(/^:/, ""),
-              retMap.metadata,
-            );
-            if (!allProjectsAddedPurls.includes(rootSubProjectObj["purl"])) {
-              allProjects.push(rootSubProjectObj);
-              rootDependsOn.push(rootSubProjectObj["bom-ref"]);
-              allProjectsAddedPurls.push(rootSubProjectObj["purl"]);
-            }
-            gradleModules.set(spstr.replace(/^:/, ""), rootSubProjectObj);
+      for (const [key, propTaskOut] of splitPropTaskOut.entries()) {
+        const retMap = parseGradleProperties(propTaskOut);
+        const rootSubProject = retMap.rootProject;
+        if (rootSubProject) {
+          const rootSubProjectObj = await buildObjectForGradleModule(
+            rootSubProject.replace(/^:/, ""),
+            retMap.metadata,
+          );
+          if (!allProjectsAddedPurls.includes(rootSubProjectObj["purl"])) {
+            allProjects.push(rootSubProjectObj);
+            rootDependsOn.push(rootSubProjectObj["bom-ref"]);
+            allProjectsAddedPurls.push(rootSubProjectObj["purl"]);
           }
+          gradleModules.set(key.replace(/^:/, ""), rootSubProjectObj);
         }
-      } //end else
+      }
       // Bug #317 fix
       parentComponent.components = allProjects.flatMap((s) => {
         delete s.qualifiers;
@@ -1714,149 +1695,79 @@ export async function createJavaBom(path, options) {
       ? process.env.GRADLE_DEPENDENCY_TASK
       : "dependencies";
 
-    if (process.env.GRADLE_MULTI_THREADED) {
-      const gradleSubCommands = [];
-      const modulesToSkip = process.env.GRADLE_SKIP_MODULES
-        ? process.env.GRADLE_SKIP_MODULES.split(",")
-        : [];
-      if (!modulesToSkip.includes("root")) {
-        gradleSubCommands.push(gradleDepTask);
-      }
-      for (const [key, sp] of gradleModules) {
-        //create single command for dependencies tasks on all subprojects
-        if (sp.purl !== parentComponent.purl && !modulesToSkip.includes(key)) {
-          gradleSubCommands.push(`:${key}:${gradleDepTask}`);
-        }
-      }
-      const gradleArguments = buildGradleCommandArguments(
-        process.env.GRADLE_ARGS ? process.env.GRADLE_ARGS.split(" ") : [],
-        gradleSubCommands,
-        process.env.GRADLE_ARGS_DEPENDENCIES
-          ? process.env.GRADLE_ARGS_DEPENDENCIES.split(" ")
-          : [],
-      );
-      console.log(
-        "Executing",
-        gradleCmd,
-        gradleArguments.join(" "),
-        "in",
-        gradleRootPath,
-      );
-      const sresult = spawnSync(gradleCmd, gradleArguments, {
-        cwd: gradleRootPath,
-        encoding: "utf-8",
-        timeout: TIMEOUT_MS,
-        maxBuffer: MAX_BUFFER,
-      });
+    const gradleSubCommands = [];
+    const modulesToSkip = process.env.GRADLE_SKIP_MODULES
+      ? process.env.GRADLE_SKIP_MODULES.split(",")
+      : [];
+    if (!modulesToSkip.includes("root")) {
+      gradleSubCommands.push(gradleDepTask);
+    }
+    for (const [key, sp] of gradleModules) {
+      //create single command for dependencies tasks on all subprojects
+      if (sp.purl !== parentComponent.purl && !modulesToSkip.includes(key)) {
+        gradleSubCommands.push(`:${key}:${gradleDepTask}`);
+      }
+    }
+    const gradleArguments = buildGradleCommandArguments(
+      process.env.GRADLE_ARGS ? process.env.GRADLE_ARGS.split(" ") : [],
+      gradleSubCommands,
+      process.env.GRADLE_ARGS_DEPENDENCIES
+        ? process.env.GRADLE_ARGS_DEPENDENCIES.split(" ")
+        : [],
+    );
+    console.log(
+      "Executing",
+      gradleCmd,
+      gradleArguments.join(" "),
+      "in",
+      gradleRootPath,
+    );
+    const sresult = spawnSync(gradleCmd, gradleArguments, {
+      cwd: gradleRootPath,
+      encoding: "utf-8",
+      timeout: TIMEOUT_MS,
+      maxBuffer: MAX_BUFFER,
+    });
 
-      if (sresult.status !== 0 || sresult.error) {
-        if (options.failOnError || DEBUG_MODE) {
-          console.error(sresult.stdout, sresult.stderr);
-        }
-        options.failOnError && process.exit(1);
-      }
-      const sstdout = sresult.stdout;
-      if (sstdout) {
-        const cmdOutput = Buffer.from(sstdout).toString();
-        const perProjectOutput = splitOutputByGradleProjects(cmdOutput, [
-          gradleDepTask,
-        ]);
-        for (const [key, sp] of gradleModules) {
-          const parsedList = await parseGradleDep(
-            perProjectOutput.has(key) ? perProjectOutput.get(key) : "",
-            key,
-            gradleModules,
-            gradleRootPath,
-          );
-          const dlist = parsedList.pkgList;
-          if (parsedList.dependenciesList && parsedList.dependenciesList) {
-            dependencies = mergeDependencies(
-              dependencies,
-              parsedList.dependenciesList,
-              parentComponent,
-            );
-          }
-          if (dlist?.length) {
-            if (DEBUG_MODE) {
-              console.log(
-                "Found",
-                dlist.length,
-                "packages in gradle project",
-                key,
-              );
-            }
-            pkgList = pkgList.concat(dlist);
-          }
-        }
-      }
-    } else {
-      if (DEBUG_MODE) {
-        console.log(
-          "Try the new multi-threaded mode for gradle. Set the environment variable GRADLE_MULTI_THREADED to true to enable this.",
-        );
+    if (sresult.status !== 0 || sresult.error) {
+      if (options.failOnError || DEBUG_MODE) {
+        console.error(sresult.stdout, sresult.stderr);
       }
+      options.failOnError && process.exit(1);
+    }
+    const sstdout = sresult.stdout;
+    if (sstdout) {
+      const cmdOutput = Buffer.from(sstdout).toString();
+      const perProjectOutput = splitOutputByGradleProjects(cmdOutput, [
+        gradleDepTask,
+      ]);
       for (const [key, sp] of gradleModules) {
-        const gradleArguments = buildGradleCommandArguments(
-          process.env.GRADLE_ARGS ? process.env.GRADLE_ARGS.split(" ") : [],
-          [
-            sp.purl === parentComponent.purl
-              ? gradleDepTask
-              : `:${key}:${gradleDepTask}`,
-          ],
-          process.env.GRADLE_ARGS_DEPENDENCIES
-            ? process.env.GRADLE_ARGS_DEPENDENCIES.split(" ")
-            : [],
-        );
-
-        console.log(
-          "Executing",
-          gradleCmd,
-          gradleArguments.join(" "),
-          "in",
+        const parsedList = await parseGradleDep(
+          perProjectOutput.has(key) ? perProjectOutput.get(key) : "",
+          key,
+          gradleModules,
           gradleRootPath,
         );
-        const sresult = spawnSync(gradleCmd, gradleArguments, {
-          cwd: gradleRootPath,
-          encoding: "utf-8",
-          timeout: TIMEOUT_MS,
-          maxBuffer: MAX_BUFFER,
-        });
-        if (sresult.status !== 0 || sresult.error) {
-          if (options.failOnError || DEBUG_MODE) {
-            console.error(sresult.stdout, sresult.stderr);
-          }
-          options.failOnError && process.exit(1);
-        }
-        const sstdout = sresult.stdout;
-        if (sstdout) {
-          const cmdOutput = Buffer.from(sstdout).toString();
-          const parsedList = await parseGradleDep(
-            cmdOutput,
-            key,
-            gradleModules,
-            gradleRootPath,
+        const dlist = parsedList.pkgList;
+        if (parsedList.dependenciesList && parsedList.dependenciesList) {
+          dependencies = mergeDependencies(
+            dependencies,
+            parsedList.dependenciesList,
+            parentComponent,
           );
-          const dlist = parsedList.pkgList;
-          if (parsedList.dependenciesList && parsedList.dependenciesList) {
-            dependencies = mergeDependencies(
-              dependencies,
-              parsedList.dependenciesList,
-              parentComponent,
+        }
+        if (dlist?.length) {
+          if (DEBUG_MODE) {
+            console.log(
+              "Found",
+              dlist.length,
+              "packages in gradle project",
+              key,
             );
           }
-          if (dlist?.length) {
-            if (DEBUG_MODE) {
-              console.log(
-                "Found",
-                dlist.length,
-                "packages in gradle project",
-                key,
-              );
-            }
-            pkgList = pkgList.concat(dlist);
-          }
+          pkgList = pkgList.concat(dlist);
         }
-      } // for
+      }
     }
     if (pkgList.length) {
       if (parentComponent.components?.length) {

diff --git a/lib/helpers/utils.js b/lib/helpers/utils.js
@@ -2530,47 +2530,34 @@ export async function parseGradleDep(
     let scope = undefined;
     let profileName = undefined;
     if (retMap?.projects) {
-      if (process.env.GRADLE_MULTI_THREADED) {
-        const modulesToScan = retMap.projects.filter(
-          (module) => !gradleModules.has(module.replace(/^:/, "")),
+      const modulesToScan = retMap.projects.filter(
+        (module) => !gradleModules.has(module.replace(/^:/, "")),
+      );
+      if (modulesToScan.length > 0) {
+        const parallelPropTaskOut = executeParallelGradleProperties(
+          gradleRootPath,
+          modulesToScan,
+        );
+        const splitPropTaskOut = splitOutputByGradleProjects(
+          parallelPropTaskOut,
+          ["properties"],
         );
-        if (modulesToScan.length > 0) {
-          const parallelPropTaskOut = executeParallelGradleProperties(
-            gradleRootPath,
-            modulesToScan,
-          );
-          const splitPropTaskOut = splitOutputByGradleProjects(
-            parallelPropTaskOut,
-            ["properties"],
-          );
 
-          for (const [key, propTaskOut] of splitPropTaskOut.entries()) {
-            const propMap = parseGradleProperties(propTaskOut);
-            const rootSubProject = propMap.rootProject;
-            if (rootSubProject) {
-              const rootSubProjectObj = await buildObjectForGradleModule(
-                rootSubProject.replace(/^:/, ""),
-                propMap.metadata,
-              );
-              gradleModules.set(key.replace(/^:/, ""), rootSubProjectObj);
-            }
-          }
-        }
-      }
-      const subDependsOn = [];
-      for (const sd of retMap.projects) {
-        const moduleName = sd.replace(":", "");
-        if (!gradleModules.has(moduleName)) {
-          const propMap = executeGradleProperties(gradleRootPath, sd);
+        for (const [key, propTaskOut] of splitPropTaskOut.entries()) {
+          const propMap = parseGradleProperties(propTaskOut);
           const rootSubProject = propMap.rootProject;
           if (rootSubProject) {
             const rootSubProjectObj = await buildObjectForGradleModule(
               rootSubProject.replace(/^:/, ""),
               propMap.metadata,
             );
-            gradleModules.set(moduleName, rootSubProjectObj);
+            gradleModules.set(key.replace(/^:/, ""), rootSubProjectObj);
           }
         }
+      }
+      const subDependsOn = [];
+      for (const sd of retMap.projects) {
+        const moduleName = sd.replace(":", "");
         if (gradleModules.has(moduleName)) {
           subDependsOn.push(gradleModules.get(moduleName)["bom-ref"]);
         }
@@ -2942,7 +2929,7 @@ export function executeParallelGradleProperties(dir, allProjectsStr) {
     cwd: dir,
     encoding: "utf-8",
     shell: isWin,
-    maxBuffer: 1024 * 1024 * 25,
+    maxBuffer: MAX_BUFFER,
   });
   if (result.status !== 0 || result.error) {
     if (result.stderr) {