diff --git a/AUTHORS.rst b/AUTHORS.rst index 9a8224dc7d..2e0e1715c4 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -87,6 +87,7 @@ The following organizations or individuals have contributed to ScanCode: - Shrijal Acharya @OctoPie23 - Shivam Chauhan @chashiv - Shivam Sandbhor @sbs2001 +- Sidheshwar Sarangal @SidheshwarSarangal - Steven Esser @majurg - Sushant Gupta @susg - Theodore Aptekarev @piiq diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cf4db41092..62b958a441 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,9 @@ Changelog Next release -------------- +- Support directory glob patterns in license rule ``referenced_filenames``. + https://github.com/aboutcode-org/scancode-toolkit/issues/4276 + - Fix the optional ``licenses`` extra dependency typo to install ``licensedcode-data``. https://github.com/aboutcode-org/scancode-toolkit/pull/5056 diff --git a/src/licensedcode/detection.py b/src/licensedcode/detection.py index 1def5f9e52..72eb468222 100644 --- a/src/licensedcode/detection.py +++ b/src/licensedcode/detection.py @@ -15,6 +15,7 @@ import uuid from enum import Enum from hashlib import sha1 +from fnmatch import fnmatchcase import attr from collections import defaultdict @@ -1990,6 +1991,73 @@ def find_referenced_resource(referenced_filename, resource, codebase, **kwargs): return resource +def find_referenced_resources( + referenced_filename, + resource, + codebase, + find_referenced_resource_func, +): + """ + Return a list of Resources matching ``referenced_filename`` for ``resource`` + in ``codebase``. ``referenced_filename`` can be an exact path or a glob + pattern. + + Exact paths use ``find_referenced_resource_func`` directly. Glob patterns + are matched one path segment at a time so that wildcards do not cross + directory boundaries. Each match is resolved again with + ``find_referenced_resource_func`` to retain its existing location rules. + """ + if not (referenced_filename and resource): + return [] + + referenced_filename = clean_path(referenced_filename) + if not any(character in referenced_filename for character in '*?['): + referenced_resource = find_referenced_resource_func( + referenced_filename=referenced_filename, + resource=resource, + codebase=codebase, + ) + return [referenced_resource] if referenced_resource else [] + + pattern_parts = referenced_filename.split('/') + referenced_resources = [] + seen_paths = set() + + for candidate in codebase.walk(skip_root=True): + candidate_path = as_posixpath(candidate.path) + candidate_parts = candidate_path.split('/') + if len(candidate_parts) < len(pattern_parts): + continue + + candidate_reference_parts = candidate_parts[-len(pattern_parts):] + if not all( + fnmatchcase(candidate_part, pattern_part) + for candidate_part, pattern_part in zip( + candidate_reference_parts, + pattern_parts, + ) + ): + continue + + candidate_reference = '/'.join(candidate_reference_parts) + referenced_resource = find_referenced_resource_func( + referenced_filename=candidate_reference, + resource=resource, + codebase=codebase, + ) + if not referenced_resource: + continue + + referenced_path = as_posixpath(referenced_resource.path) + if referenced_path != candidate_path or referenced_path in seen_paths: + continue + + seen_paths.add(referenced_path) + referenced_resources.append(referenced_resource) + + return referenced_resources + + def update_expressions_from_license_detections(resource, codebase): """ Set the `detected_license_expression` and `detected_license_expression_spdx` @@ -2046,14 +2114,23 @@ def update_detection_from_referenced_files( referenced_detections = [] referenced_resources = [] + seen_referenced_resource_paths = set() for referenced_filename in referenced_filenames: - referenced_resource = find_referenced_resource_func( + resolved_resources = find_referenced_resources( referenced_filename=referenced_filename, resource=resource, codebase=codebase, + find_referenced_resource_func=find_referenced_resource_func, ) - if referenced_resource and referenced_resource.license_detections: + for referenced_resource in resolved_resources: + if ( + referenced_resource.path in seen_referenced_resource_paths + or not referenced_resource.license_detections + ): + continue + + seen_referenced_resource_paths.add(referenced_resource.path) referenced_detections.extend( referenced_resource.license_detections ) diff --git a/tests/licensedcode/data/plugin_license/license_reference/scan/scan-ref-glob/license-notice.txt b/tests/licensedcode/data/plugin_license/license_reference/scan/scan-ref-glob/license-notice.txt new file mode 100644 index 0000000000..6a2cab4ab4 --- /dev/null +++ b/tests/licensedcode/data/plugin_license/license_reference/scan/scan-ref-glob/license-notice.txt @@ -0,0 +1 @@ +See the license files in licenses/*. diff --git a/tests/licensedcode/data/plugin_license/license_reference/scan/scan-ref-glob/licenses/COPYING b/tests/licensedcode/data/plugin_license/license_reference/scan/scan-ref-glob/licenses/COPYING new file mode 100644 index 0000000000..9620d5bcfc --- /dev/null +++ b/tests/licensedcode/data/plugin_license/license_reference/scan/scan-ref-glob/licenses/COPYING @@ -0,0 +1 @@ +Apache License, Version 2.0 diff --git a/tests/licensedcode/data/plugin_license/license_reference/scan/scan-ref-glob/licenses/LICENSE b/tests/licensedcode/data/plugin_license/license_reference/scan/scan-ref-glob/licenses/LICENSE new file mode 100644 index 0000000000..d1e1072ee5 --- /dev/null +++ b/tests/licensedcode/data/plugin_license/license_reference/scan/scan-ref-glob/licenses/LICENSE @@ -0,0 +1 @@ +MIT License diff --git a/tests/licensedcode/test_plugin_license_detection.py b/tests/licensedcode/test_plugin_license_detection.py index e18db47ccd..f1f5454ca9 100644 --- a/tests/licensedcode/test_plugin_license_detection.py +++ b/tests/licensedcode/test_plugin_license_detection.py @@ -11,6 +11,7 @@ from commoncode.testcase import FileDrivenTesting +from licensedcode.detection import find_referenced_resources from licensedcode.plugin_license import find_referenced_resource from scancode.cli_test_utils import check_json_scan from scancode.cli_test_utils import run_scan_click @@ -367,6 +368,28 @@ def test_find_referenced_resource_does_not_find_based_file_name_suffix(): assert result.path == 'scan-ref-dupe-name-suffix/LICENSE' +def test_find_referenced_resources_with_directory_glob(): + test_dir = test_env.get_test_loc('plugin_license/license_reference/scan/scan-ref-glob') + scan_loc = test_env.get_temp_file('json') + args = ['--license', '--json', scan_loc, test_dir] + run_scan_click(args) + + from commoncode.resource import VirtualCodebase + codebase = VirtualCodebase(scan_loc) + resource = codebase.get_resource(path='scan-ref-glob/license-notice.txt') + results = find_referenced_resources( + referenced_filename='licenses/*', + resource=resource, + codebase=codebase, + find_referenced_resource_func=find_referenced_resource, + ) + + assert [result.path for result in results] == [ + 'scan-ref-glob/licenses/COPYING', + 'scan-ref-glob/licenses/LICENSE', + ] + + def test_match_reference_license(): # Setup: Create a new scan to use for a virtual codebase test_dir = test_env.get_test_loc('plugin_license/license_reference/scan/scan-ref')