feat: add network prefix and filter options (#406)

* feat: add network prefix and filter options

* fix: update network tests

* fix: add tests

* fix: pull all files for file fixes

* fix: added description to network-filter

Author: thomasrockhu-codecov

codecov_cli/commands/upload.py

@@ -164,6 +164,14 @@ def _turn_env_vars_into_dict(ctx, params, value):
         default="coverage",
         type=click.Choice(["coverage", "test_results"]),
     ),
+    click.option(
+        "--network-filter",
+        help="Specify a filter on the files listed in the network section of the Codecov report. This will only add files whose path begin with the specified filter. Useful for upload-specific path fixing",
+    ),
+    click.option(
+        "--network-prefix",
+        help="Specify a prefix on files listed in the network section of the Codecov report. Useful to help resolve path fixing",
+    ),
 ]
 
 
@@ -181,29 +189,31 @@ def do_upload(
     ctx: click.Context,
     commit_sha: str,
     report_code: str,
+    branch: typing.Optional[str],
     build_code: typing.Optional[str],
     build_url: typing.Optional[str],
-    job_code: typing.Optional[str],
+    disable_file_fixes: bool,
+    disable_search: bool,
+    dry_run: bool,
     env_vars: typing.Dict[str, str],
+    fail_on_error: bool,
+    files_search_exclude_folders: typing.List[pathlib.Path],
+    files_search_explicitly_listed_files: typing.List[pathlib.Path],
+    files_search_root_folder: pathlib.Path,
     flags: typing.List[str],
+    git_service: typing.Optional[str],
+    handle_no_reports_found: bool,
+    job_code: typing.Optional[str],
     name: typing.Optional[str],
+    network_filter: typing.Optional[str],
+    network_prefix: typing.Optional[str],
     network_root_folder: pathlib.Path,
-    files_search_root_folder: pathlib.Path,
-    files_search_exclude_folders: typing.List[pathlib.Path],
-    files_search_explicitly_listed_files: typing.List[pathlib.Path],
-    disable_search: bool,
-    disable_file_fixes: bool,
-    token: typing.Optional[str],
     plugin_names: typing.List[str],
-    branch: typing.Optional[str],
-    slug: typing.Optional[str],
     pull_request_number: typing.Optional[str],
-    use_legacy_uploader: bool,
-    fail_on_error: bool,
-    dry_run: bool,
-    git_service: typing.Optional[str],
-    handle_no_reports_found: bool,
     report_type: str,
+    slug: typing.Optional[str],
+    token: typing.Optional[str],
+    use_legacy_uploader: bool,
 ):
     versioning_system = ctx.obj["versioning_system"]
     codecov_yaml = ctx.obj["codecov_yaml"] or {}
@@ -214,60 +224,64 @@ def do_upload(
         "Starting upload processing",
         extra=dict(
             extra_log_attributes=dict(
-                upload_file_type=report_type,
-                commit_sha=commit_sha,
-                report_code=report_code,
+                branch=branch,
                 build_code=build_code,
                 build_url=build_url,
-                job_code=job_code,
+                commit_sha=commit_sha,
+                disable_file_fixes=disable_file_fixes,
+                disable_search=disable_search,
+                enterprise_url=enterprise_url,
                 env_vars=env_vars,
+                files_search_exclude_folders=files_search_exclude_folders,
+                files_search_explicitly_listed_files=files_search_explicitly_listed_files,
+                files_search_root_folder=files_search_root_folder,
                 flags=flags,
+                git_service=git_service,
+                handle_no_reports_found=handle_no_reports_found,
+                job_code=job_code,
                 name=name,
+                network_filter=network_filter,
+                network_prefix=network_prefix,
                 network_root_folder=network_root_folder,
-                files_search_root_folder=files_search_root_folder,
-                files_search_exclude_folders=files_search_exclude_folders,
-                files_search_explicitly_listed_files=files_search_explicitly_listed_files,
                 plugin_names=plugin_names,
-                token=token,
-                branch=branch,
-                slug=slug,
                 pull_request_number=pull_request_number,
-                git_service=git_service,
-                enterprise_url=enterprise_url,
-                disable_search=disable_search,
-                disable_file_fixes=disable_file_fixes,
-                handle_no_reports_found=handle_no_reports_found,
+                report_code=report_code,
+                slug=slug,
+                token=token,
+                upload_file_type=report_type,
             )
         ),
     )
     do_upload_logic(
         cli_config,
         versioning_system,
         ci_adapter,
-        upload_file_type=report_type,
-        commit_sha=commit_sha,
-        report_code=report_code,
+        branch=branch,
         build_code=build_code,
         build_url=build_url,
-        job_code=job_code,
+        commit_sha=commit_sha,
+        disable_file_fixes=disable_file_fixes,
+        disable_search=disable_search,
+        dry_run=dry_run,
+        enterprise_url=enterprise_url,
         env_vars=env_vars,
+        fail_on_error=fail_on_error,
+        files_search_exclude_folders=list(files_search_exclude_folders),
+        files_search_explicitly_listed_files=list(files_search_explicitly_listed_files),
+        files_search_root_folder=files_search_root_folder,
         flags=flags,
+        git_service=git_service,
+        handle_no_reports_found=handle_no_reports_found,
+        job_code=job_code,
         name=name,
+        network_filter=network_filter,
+        network_prefix=network_prefix,
         network_root_folder=network_root_folder,
-        files_search_root_folder=files_search_root_folder,
-        files_search_exclude_folders=list(files_search_exclude_folders),
-        files_search_explicitly_listed_files=list(files_search_explicitly_listed_files),
         plugin_names=plugin_names,
-        token=token,
-        branch=branch,
-        slug=slug,
         pull_request_number=pull_request_number,
+        report_code=report_code,
+        slug=slug,
+        token=token,
+        upload_file_type=report_type,
         use_legacy_uploader=use_legacy_uploader,
-        fail_on_error=fail_on_error,
-        dry_run=dry_run,
-        git_service=git_service,
-        enterprise_url=enterprise_url,
-        disable_search=disable_search,
-        handle_no_reports_found=handle_no_reports_found,
-        disable_file_fixes=disable_file_fixes,
     )

codecov_cli/commands/upload_process.py

@@ -31,6 +31,8 @@ def upload_process(
     env_vars: typing.Dict[str, str],
     flags: typing.List[str],
     name: typing.Optional[str],
+    network_filter: typing.Optional[str],
+    network_prefix: typing.Optional[str],
     network_root_folder: pathlib.Path,
     files_search_root_folder: pathlib.Path,
     files_search_exclude_folders: typing.List[pathlib.Path],

codecov_cli/services/upload/__init__.py

@@ -25,32 +25,34 @@ def do_upload_logic(
     versioning_system: VersioningSystemInterface,
     ci_adapter: CIAdapterBase,
     *,
-    commit_sha: str,
-    report_code: str,
+    branch: typing.Optional[str],
     build_code: typing.Optional[str],
     build_url: typing.Optional[str],
-    job_code: typing.Optional[str],
+    commit_sha: str,
+    disable_file_fixes: bool = False,
+    disable_search: bool = False,
+    dry_run: bool = False,
+    enterprise_url: typing.Optional[str],
     env_vars: typing.Dict[str, str],
+    fail_on_error: bool = False,
+    files_search_exclude_folders: typing.List[Path],
+    files_search_explicitly_listed_files: typing.List[Path],
+    files_search_root_folder: Path,
     flags: typing.List[str],
+    git_service: typing.Optional[str],
+    handle_no_reports_found: bool = False,
+    job_code: typing.Optional[str],
     name: typing.Optional[str],
+    network_filter: typing.Optional[str],
+    network_prefix: typing.Optional[str],
     network_root_folder: Path,
-    files_search_root_folder: Path,
-    files_search_exclude_folders: typing.List[Path],
-    files_search_explicitly_listed_files: typing.List[Path],
     plugin_names: typing.List[str],
-    token: str,
-    branch: typing.Optional[str],
-    slug: typing.Optional[str],
     pull_request_number: typing.Optional[str],
+    report_code: str,
+    slug: typing.Optional[str],
+    token: str,
     upload_file_type: str = "coverage",
     use_legacy_uploader: bool = False,
-    fail_on_error: bool = False,
-    dry_run: bool = False,
-    git_service: typing.Optional[str],
-    enterprise_url: typing.Optional[str],
-    disable_search: bool = False,
-    handle_no_reports_found: bool = False,
-    disable_file_fixes: bool = False,
 ):
     if upload_file_type == "coverage":
         preparation_plugins = select_preparation_plugins(cli_config, plugin_names)
@@ -63,7 +65,12 @@ def do_upload_logic(
         disable_search,
         upload_file_type,
     )
-    network_finder = select_network_finder(versioning_system)
+    network_finder = select_network_finder(
+        versioning_system,
+        network_filter=network_filter,
+        network_prefix=network_prefix,
+        network_root_folder=network_root_folder,
+    )
     collector = UploadCollector(
         preparation_plugins, network_finder, file_selector, disable_file_fixes
     )

codecov_cli/services/upload/network_finder.py

@@ -5,17 +5,39 @@
 
 
 class NetworkFinder(object):
-    def __init__(self, versioning_system: VersioningSystemInterface):
+    def __init__(
+        self,
+        versioning_system: VersioningSystemInterface,
+        network_filter: typing.Optional[str],
+        network_prefix: typing.Optional[str],
+        network_root_folder: pathlib.Path,
+    ):
         self.versioning_system = versioning_system
+        self.network_filter = network_filter
+        self.network_prefix = network_prefix
+        self.network_root_folder = network_root_folder
 
-    def find_files(
-        self,
-        network_root: typing.Optional[pathlib.Path] = None,
-        network_filter=None,
-        network_adjuster=None,
-    ) -> typing.List[str]:
-        return self.versioning_system.list_relevant_files(network_root)
+    def find_files(self, ignore_filters=False) -> typing.List[str]:
+        files = self.versioning_system.list_relevant_files(self.network_root_folder)
+
+        if not ignore_filters:
+            if self.network_filter:
+                files = [file for file in files if file.startswith(self.network_filter)]
+            if self.network_prefix:
+                files = [self.network_prefix + file for file in files]
+
+        return files
 
 
-def select_network_finder(versioning_system: VersioningSystemInterface):
-    return NetworkFinder(versioning_system)
+def select_network_finder(
+    versioning_system: VersioningSystemInterface,
+    network_filter: typing.Optional[str],
+    network_prefix: typing.Optional[str],
+    network_root_folder: pathlib.Path,
+):
+    return NetworkFinder(
+        versioning_system,
+        network_filter,
+        network_prefix,
+        network_root_folder,
+    )

codecov_cli/services/upload/upload_collector.py

@@ -36,10 +36,10 @@ def __init__(
         self.file_finder = file_finder
         self.disable_file_fixes = disable_file_fixes
 
-    def _produce_file_fixes_for_network(
-        self, network: typing.List[str]
+    def _produce_file_fixes(
+        self, files: typing.List[str]
     ) -> typing.List[UploadCollectionResultFileFixer]:
-        if not network or self.disable_file_fixes:
+        if not files or self.disable_file_fixes:
             return []
         # patterns that we don't need to specify a reason for
         empty_line_regex = re.compile(r"^\s*$")
@@ -94,7 +94,7 @@ def _produce_file_fixes_for_network(
         }
 
         result = []
-        for filename in network:
+        for filename in files:
             for glob, fix_patterns in file_regex_patterns.items():
                 if fnmatch(filename, glob):
                     result.append(self._get_file_fixes(filename, fix_patterns))
@@ -150,9 +150,9 @@ def generate_upload_data(self, report_type="coverage") -> UploadCollectionResult
             prep.run_preparation(self)
         logger.debug("Collecting relevant files")
         network = self.network_finder.find_files()
-        files = self.file_finder.find_files()
-        logger.info(f"Found {len(files)} {report_type} files to upload")
-        if not files:
+        report_files = self.file_finder.find_files()
+        logger.info(f"Found {len(report_files)} {report_type} files to report")
+        if not report_files:
             if report_type == "test_results":
                 error_message = "No JUnit XML reports found. Please review our documentation (https://docs.codecov.com/docs/test-result-ingestion-beta) to generate and upload the file."
             else:
@@ -163,13 +163,13 @@ def generate_upload_data(self, report_type="coverage") -> UploadCollectionResult
                     fg="red",
                 )
             )
-        for file in files:
+        for file in report_files:
             logger.info(f"> {file}")
         return UploadCollectionResult(
             network=network,
-            files=files,
+            files=report_files,
             file_fixes=(
-                self._produce_file_fixes_for_network(network)
+                self._produce_file_fixes(self.network_finder.find_files(True))
                 if report_type == "coverage"
                 else []
             ),

tests/commands/test_invoke_upload_process.py

@@ -118,6 +118,14 @@ def test_upload_process_options(mocker):
             "                                  The type of the file to upload, coverage by",
             "                                  default. Possible values are: testing,",
             "                                  coverage.",
+            "  --network-filter TEXT           Specify a filter on the files listed in the",
+            "                                  network section of the Codecov report. This",
+            "                                  will only add files whose path begin with the",
+            "                                  specified filter. Useful for upload-specific",
+            "                                  path fixing",
+            "  --network-prefix TEXT           Specify a prefix on files listed in the",
+            "                                  network section of the Codecov report. Useful",
+            "                                  to help resolve path fixing",
             "  --parent-sha TEXT               SHA (with 40 chars) of what should be the",
             "                                  parent of this commit",
             "  -h, --help                      Show this message and exit.",

tests/helpers/test_network_finder.py

@@ -6,11 +6,46 @@
 
 
 def test_find_files(mocker, tmp_path):
+    filenames = ["a.txt", "b.txt"]
+    filtered_filenames = []
 
-    expected_filenames = ["a.txt", "b.txt"]
+    mocked_vs = MagicMock()
+    mocked_vs.list_relevant_files.return_value = filenames
+
+    assert NetworkFinder(versioning_system=mocked_vs, network_filter=None, network_prefix=None, network_root_folder=tmp_path).find_files() == filenames
+    assert NetworkFinder(versioning_system=mocked_vs, network_filter="hello", network_prefix="bello", network_root_folder=tmp_path).find_files(False) == filtered_filenames
+    assert NetworkFinder(versioning_system=mocked_vs, network_filter="hello", network_prefix="bello", network_root_folder=tmp_path).find_files(True) == filenames
+    mocked_vs.list_relevant_files.assert_called_with(tmp_path)
+
+def test_find_files_with_filter(mocker, tmp_path):
+    filenames = ["hello/a.txt", "hello/c.txt", "bello/b.txt"]
+    filtered_filenames = ["hello/a.txt", "hello/c.txt"]
+
+    mocked_vs = MagicMock()
+    mocked_vs.list_relevant_files.return_value = filenames
+
+    assert NetworkFinder(versioning_system=mocked_vs, network_filter="hello", network_prefix=None, network_root_folder=tmp_path).find_files() == filtered_filenames
+    assert NetworkFinder(versioning_system=mocked_vs, network_filter="hello", network_prefix="bello", network_root_folder=tmp_path).find_files(True) == filenames
+    mocked_vs.list_relevant_files.assert_called_with(tmp_path)
+
+def test_find_files_with_prefix(mocker, tmp_path):
+    filenames = ["hello/a.txt", "hello/c.txt", "bello/b.txt"]
+    filtered_filenames = ["hellohello/a.txt", "hellohello/c.txt", "hellobello/b.txt"]
+
+    mocked_vs = MagicMock()
+    mocked_vs.list_relevant_files.return_value = filenames
+
+    assert NetworkFinder(versioning_system=mocked_vs, network_filter=None, network_prefix="hello", network_root_folder=tmp_path).find_files() == filtered_filenames
+    assert NetworkFinder(versioning_system=mocked_vs, network_filter="hello", network_prefix="bello", network_root_folder=tmp_path).find_files(True) == filenames
+    mocked_vs.list_relevant_files.assert_called_with(tmp_path)
+
+def test_find_files_with_filter_and_prefix(mocker, tmp_path):
+    filenames = ["hello/a.txt", "hello/c.txt", "bello/b.txt"]
+    filtered_filenames = ["bellohello/a.txt", "bellohello/c.txt"]
 
     mocked_vs = MagicMock()
-    mocked_vs.list_relevant_files.return_value = expected_filenames
+    mocked_vs.list_relevant_files.return_value = filenames
 
-    assert NetworkFinder(mocked_vs).find_files(tmp_path) == expected_filenames
+    assert NetworkFinder(versioning_system=mocked_vs, network_filter="hello", network_prefix="bello", network_root_folder=tmp_path).find_files() == filtered_filenames
+    assert NetworkFinder(versioning_system=mocked_vs, network_filter="hello", network_prefix="bello", network_root_folder=tmp_path).find_files(True) == filenames
     mocked_vs.list_relevant_files.assert_called_with(tmp_path)