CM-44581 - Add support for running Gradle restore command on all subprojects (#278)

Author: morsa4406

cycode/cli/commands/scan/scan_command.py

@@ -13,6 +13,7 @@
 from cycode.cli.consts import (
     ISSUE_DETECTED_STATUS_CODE,
     NO_ISSUES_STATUS_CODE,
+    SCA_GRADLE_ALL_SUB_PROJECTS_FLAG,
     SCA_SKIP_RESTORE_DEPENDENCIES_FLAG,
 )
 from cycode.cli.models import Severity
@@ -110,6 +111,15 @@
     type=bool,
     required=False,
 )
+@click.option(
+    f'--{SCA_GRADLE_ALL_SUB_PROJECTS_FLAG}',
+    is_flag=True,
+    default=False,
+    help='When specified, Cycode will run gradle restore command for all sub projects. '
+    'Should run from root project directory ONLY!',
+    type=bool,
+    required=False,
+)
 @click.pass_context
 def scan_command(
     context: click.Context,
@@ -124,6 +134,7 @@ def scan_command(
     report: bool,
     no_restore: bool,
     sync: bool,
+    gradle_all_sub_projects: bool,
 ) -> int:
     """Scans for Secrets, IaC, SCA or SAST violations."""
     add_breadcrumb('scan')
@@ -145,6 +156,7 @@ def scan_command(
     context.obj['monitor'] = monitor
     context.obj['report'] = report
     context.obj[SCA_SKIP_RESTORE_DEPENDENCIES_FLAG] = no_restore
+    context.obj[SCA_GRADLE_ALL_SUB_PROJECTS_FLAG] = gradle_all_sub_projects
 
     _sca_scan_to_context(context, sca_scan)
 

cycode/cli/consts.py

@@ -224,3 +224,5 @@
 SCA_SHORTCUT_DEPENDENCY_PATHS = 2
 
 SCA_SKIP_RESTORE_DEPENDENCIES_FLAG = 'no-restore'
+
+SCA_GRADLE_ALL_SUB_PROJECTS_FLAG = 'gradle-all-sub-projects'

cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py

@@ -1,28 +1,70 @@
 import os
-from typing import List
+import re
+from typing import List, Optional, Set
 
 import click
 
+from cycode.cli.consts import SCA_GRADLE_ALL_SUB_PROJECTS_FLAG
 from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies
 from cycode.cli.models import Document
+from cycode.cli.utils.path_utils import get_path_from_context
+from cycode.cli.utils.shell_executor import shell
 
 BUILD_GRADLE_FILE_NAME = 'build.gradle'
 BUILD_GRADLE_KTS_FILE_NAME = 'build.gradle.kts'
 BUILD_GRADLE_DEP_TREE_FILE_NAME = 'gradle-dependencies-generated.txt'
+BUILD_GRADLE_ALL_PROJECTS_TIMEOUT = 180
+BUILD_GRADLE_ALL_PROJECTS_COMMAND = ['gradle', 'projects']
+ALL_PROJECTS_REGEX = r"[+-]{3} Project '(.*?)'"
 
 
 class RestoreGradleDependencies(BaseRestoreDependencies):
-    def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None:
+    def __init__(
+        self, context: click.Context, is_git_diff: bool, command_timeout: int, projects: Optional[Set[str]] = None
+    ) -> None:
         super().__init__(context, is_git_diff, command_timeout, create_output_file_manually=True)
+        if projects is None:
+            projects = set()
+        self.projects = self.get_all_projects() if self.is_gradle_sub_projects() else projects
+
+    def is_gradle_sub_projects(self) -> bool:
+        return self.context.obj.get(SCA_GRADLE_ALL_SUB_PROJECTS_FLAG)
 
     def is_project(self, document: Document) -> bool:
         return document.path.endswith(BUILD_GRADLE_FILE_NAME) or document.path.endswith(BUILD_GRADLE_KTS_FILE_NAME)
 
     def get_commands(self, manifest_file_path: str) -> List[List[str]]:
-        return [['gradle', 'dependencies', '-b', manifest_file_path, '-q', '--console', 'plain']]
+        return (
+            self.get_commands_for_sub_projects(manifest_file_path)
+            if self.is_gradle_sub_projects()
+            else [['gradle', 'dependencies', '-b', manifest_file_path, '-q', '--console', 'plain']]
+        )
 
     def get_lock_file_name(self) -> str:
         return BUILD_GRADLE_DEP_TREE_FILE_NAME
 
     def verify_restore_file_already_exist(self, restore_file_path: str) -> bool:
         return os.path.isfile(restore_file_path)
+
+    def get_working_directory(self, document: Document) -> Optional[str]:
+        return get_path_from_context(self.context) if self.is_gradle_sub_projects() else None
+
+    def get_all_projects(self) -> Set[str]:
+        projects_output = shell(
+            command=BUILD_GRADLE_ALL_PROJECTS_COMMAND,
+            timeout=BUILD_GRADLE_ALL_PROJECTS_TIMEOUT,
+            working_directory=get_path_from_context(self.context),
+        )
+
+        projects = re.findall(ALL_PROJECTS_REGEX, projects_output)
+
+        return set(projects)
+
+    def get_commands_for_sub_projects(self, manifest_file_path: str) -> List[List[str]]:
+        project_name = os.path.basename(os.path.dirname(manifest_file_path))
+        project_name = f':{project_name}'
+        return (
+            [['gradle', f'{project_name}:dependencies', '-q', '--console', 'plain']]
+            if project_name in self.projects
+            else []
+        )

cycode/cli/utils/scan_batch.py

@@ -50,7 +50,8 @@ def run_parallel_batched_scan(
     progress_bar: 'BaseProgressBar',
 ) -> Tuple[Dict[str, 'CliError'], List['LocalScanResult']]:
     max_size = consts.SCAN_BATCH_MAX_SIZE_IN_BYTES.get(scan_type, consts.DEFAULT_SCAN_BATCH_MAX_SIZE_IN_BYTES)
-    batches = split_documents_into_batches(documents, max_size)
+
+    batches = [documents] if scan_type == consts.SCA_SCAN_TYPE else split_documents_into_batches(documents, max_size)
 
     progress_bar.set_section_length(ScanProgressBarSection.SCAN, len(batches))  # * 3
     # TODO(MarshalX): we should multiply the count of batches in SCAN section because each batch has 3 steps: