diff --git a/matchcode/api.py b/matchcode/api.py index 6d728536..578df8eb 100644 --- a/matchcode/api.py +++ b/matchcode/api.py @@ -265,11 +265,15 @@ def match(self, request): if not fingerprints: return Response() + ecosystems = request.query_params.getlist("ecosystems") + exclude_purls = request.query_params.getlist("exclude_purls") model_class = self.get_serializer().Meta.model results = [] unique_fingerprints = set(fingerprints) for fingerprint in unique_fingerprints: - matches = model_class.match(fingerprint) + matches = model_class.match( + fingerprint, ecosystems=ecosystems, exclude_purls=exclude_purls + ) for match in matches: _, bah128 = split_fingerprint(fingerprint) # Get fingerprint from the match @@ -357,6 +361,30 @@ class MatchingSerializer(ExcludeFromListViewMixin, serializers.ModelSerializer): discovered_dependencies_summary = serializers.SerializerMethodField() codebase_relations_summary = serializers.SerializerMethodField() + ecosystems = serializers.ChoiceField( + choices=( + ("", "---------"), + ("maven", "maven"), + ), + required=False, + allow_blank=True, + default="", + write_only=True, + help_text="Ecosystem to restrict the match index.", + ) + + exclude_purls = serializers.CharField( + required=False, + allow_blank=True, + default="", + write_only=True, + style={"base_template": "textarea.html"}, + help_text="Exclude PURLs (space or comma separated).", + ) + + ecosystems_filter = serializers.SerializerMethodField(read_only=True) + exclude_purls_filter = serializers.SerializerMethodField(read_only=True) + class Meta: model = Project fields = ( @@ -376,6 +404,10 @@ class Meta: "discovered_packages_summary", "discovered_dependencies_summary", "codebase_relations_summary", + "ecosystems", + "exclude_purls", + "ecosystems_filter", + "exclude_purls_filter", ) exclude_from_list_view = [ "resource_count", @@ -419,6 +451,12 @@ def get_codebase_relations_summary(self, project): queryset = project.codebaserelations.all() return count_group_by(queryset, "map_type") + def get_ecosystems_filter(self, project): + return (project.extra_data or {}).get("ecosystems", []) + + def get_exclude_purls_filter(self, project): + return (project.extra_data or {}).get("exclude_purls", []) + def validate_input_urls(self, value): """Add support for providing multiple URLs in a single string.""" return [url for entry in value for url in entry.split()] @@ -430,6 +468,18 @@ def create(self, validated_data, matching_pipeline_name="matching"): upload_file = validated_data.pop("upload_file", None) input_urls = validated_data.pop("input_urls", []) webhook_url = validated_data.pop("webhook_url", None) + ecosystems = validated_data.pop("ecosystems", "") + exclude_purls = validated_data.pop("exclude_purls", "") + + # Convert ecosystems to a list + if isinstance(ecosystems, str): + ecosystems = [ecosystems] if ecosystems else [] + + # Convert exclude_purls to a list; support spaces, commas, and newlines + if isinstance(exclude_purls, str): + exclude_purls = [ + purl.strip() for purl in exclude_purls.replace(",", " ").split() if purl.strip() + ] downloads, errors = fetch_urls(input_urls) if errors: @@ -437,6 +487,16 @@ def create(self, validated_data, matching_pipeline_name="matching"): project = super().create(validated_data) + project.extra_data = project.extra_data or {} + if ecosystems: + project.extra_data["ecosystems"] = ecosystems + + if exclude_purls: + project.extra_data["exclude_purls"] = exclude_purls + + if ecosystems or exclude_purls: + project.save() + if upload_file: project.add_uploads([upload_file]) @@ -580,7 +640,7 @@ class MatchingViewSet( """ Take a ScanCode.io JSON of a codebase `upload_file` or from a list of `input_urls` and run the ``matching`` pipeline - (https://github.com/aboutcode-org/purldb/blob/main/matchcode_pipeline/pipelines/matching.py) + (https://github.com/aboutcode-org/purldb/blob/main/matchcode/pipelines/matching.py) on it. The ``matching`` pipeline matches directory and resources of the codebase in diff --git a/matchcode/models.py b/matchcode/models.py index bda967cb..521647cf 100644 --- a/matchcode/models.py +++ b/matchcode/models.py @@ -33,6 +33,8 @@ from packagedb.models import Package from packagedb.models import Resource +from matchcode.utils import build_purl_filter + TRACE = False if TRACE: @@ -94,7 +96,7 @@ def index(cls, sha1, package): logger.error(msg) @classmethod - def match(cls, sha1): + def match(cls, sha1, ecosystems=None, exclude_purls=None): """Return a list of matched Packages that contains a file with a SHA1 value of `sha1`""" if TRACE: logger_debug(cls.__name__, "match:", "sha1:", sha1) @@ -104,6 +106,10 @@ def match(cls, sha1): sha1_in_bin = hexstring_to_binarray(sha1) matches = cls.objects.filter(sha1=sha1_in_bin) + if ecosystems: + matches = matches.filter(package__type__in=ecosystems) + if exclude_purls: + matches = matches.exclude(build_purl_filter(exclude_purls, relation_prefix="package__")) if TRACE: for match in matches: package = match.package @@ -226,7 +232,9 @@ def index(cls, fingerprint, resource_path, package): logger.error(msg) @classmethod - def match(cls, fingerprint, resource=None, exact_match=False): + def match( + cls, fingerprint, resource=None, exact_match=False, ecosystems=None, exclude_purls=None + ): """Return a list of matched Packages""" if TRACE: logger_debug( @@ -253,6 +261,12 @@ def match(cls, fingerprint, resource=None, exact_match=False): chunk3=chunk3, chunk4=chunk4, ) + if ecosystems: + matches = matches.filter(package__type__in=ecosystems) + if exclude_purls: + matches = matches.exclude( + build_purl_filter(exclude_purls, relation_prefix="package__") + ) return matches # Step 1: find fingerprints with matching chunks @@ -264,6 +278,11 @@ def match(cls, fingerprint, resource=None, exact_match=False): | models.Q(indexed_elements_count__range=frange, chunk4=chunk4) ) + if ecosystems: + matches = matches.filter(package__type__in=ecosystems) + if exclude_purls: + matches = matches.exclude(build_purl_filter(exclude_purls, relation_prefix="package__")) + if TRACE: for match in matches: dct = model_to_dict(match) @@ -478,7 +497,7 @@ def index(cls, fingerprint, position, resource, package): logger.error(msg) @classmethod - def match(cls, fingerprints): + def match(cls, fingerprints, ecosystems=None, exclude_purls=None): """ Return a list of PackageSnippetMatch for matched Package. """ @@ -499,6 +518,13 @@ def match(cls, fingerprints): # Step 0: get all fingerprint records that match with the input matched_fps = cls.objects.filter(fingerprint__in=only_fings) + if ecosystems: + matched_fps = matched_fps.filter(package__type__in=ecosystems) + if exclude_purls: + matched_fps = matched_fps.exclude( + build_purl_filter(exclude_purls, relation_prefix="package__") + ) + # Step 1: count Packages whose fingerprints appear # Step 1.1: get Packages that show up in the query packages = set(f.package for f in matched_fps.iterator()) @@ -518,7 +544,7 @@ def match(cls, fingerprints): return matches @classmethod - def match_resources(cls, fingerprints, top=None, **kwargs): + def match_resources(cls, fingerprints, top=None, ecosystems=None, exclude_purls=None, **kwargs): """ Return a list of ResourceSnippetMatch for matched Resources. Only return the ``top`` matches, or all matches if ``top`` is zero. @@ -562,6 +588,13 @@ def match_resources(cls, fingerprints, top=None, **kwargs): # Step 0: get all fingerprint records that match with the input matched_fps = cls.objects.filter(fingerprint__in=only_fings) + if ecosystems: + matched_fps = matched_fps.filter(package__type__in=ecosystems) + if exclude_purls: + matched_fps = matched_fps.exclude( + build_purl_filter(exclude_purls, relation_prefix="package__") + ) + # Step 1: get Resources that show up in the query resources = set(f.resource for f in matched_fps.iterator()) diff --git a/matchcode/pipes/matching.py b/matchcode/pipes/matching.py index 25edfe32..de3ea30b 100644 --- a/matchcode/pipes/matching.py +++ b/matchcode/pipes/matching.py @@ -38,6 +38,23 @@ from packagedb.models import Package from packagedb.models import Resource +from matchcode.utils import build_purl_filter + + +def get_filtering_kwargs(project): + """ + Extract ecosystems and exclude_purls from the project's extra_data. + """ + kwargs = {} + if project and project.extra_data: + ecosystems = project.extra_data.get("ecosystems") + if ecosystems: + kwargs["ecosystems"] = ecosystems + exclude_purls = project.extra_data.get("exclude_purls") + if exclude_purls: + kwargs["exclude_purls"] = exclude_purls + return kwargs + def get_project_resources_qs(project, resources): """ @@ -112,7 +129,15 @@ def match_purldb_package(project, resources_by_sha1, enhance_package_data=True, """ match_count = 0 sha1_list = list(resources_by_sha1.keys()) - results = Package.objects.filter(sha1__in=sha1_list).order_by() + results = Package.objects.filter(sha1__in=sha1_list) + + filters = get_filtering_kwargs(project) + if "ecosystems" in filters: + results = results.filter(type__in=filters["ecosystems"]) + if "exclude_purls" in filters: + results = results.exclude(build_purl_filter(filters["exclude_purls"])) + + results = results.order_by() # Process matched Package data for package in results: package_data = package.to_dict() @@ -145,11 +170,19 @@ def match_purldb_resource(project, resources_by_sha1, package_data_by_purldb_url match_count = 0 sha1_list = list(resources_by_sha1.keys()) results = ( - Resource.objects.filter(sha1__in=sha1_list) - .select_related("package") - .only("package__uuid") - .order_by() + Resource.objects.filter(sha1__in=sha1_list).select_related("package").only("package__uuid") ) + + filters = get_filtering_kwargs(project) + if "ecosystems" in filters: + results = results.filter(package__type__in=filters["ecosystems"]) + if "exclude_purls" in filters: + results = results.exclude( + build_purl_filter(filters["exclude_purls"], relation_prefix="package__") + ) + + results = results.order_by() + # Process match results for resource in results: # Get package data @@ -171,7 +204,13 @@ def match_purldb_resource(project, resources_by_sha1, package_data_by_purldb_url def match_purldb_resource_approximately(project, resource): """Match by approximation a single resource in the PurlDB.""" fingerprint = resource.extra_data.get("halo1", "") - results = ApproximateResourceContentIndex.match(fingerprint=fingerprint, resource=resource) + filters = get_filtering_kwargs(project) + results = ApproximateResourceContentIndex.match( + fingerprint=fingerprint, + resource=resource, + ecosystems=filters.get("ecosystems"), + exclude_purls=filters.get("exclude_purls"), + ) for result in results: package_data = result.package.to_dict() return create_package_from_purldb_data( @@ -185,9 +224,12 @@ def match_purldb_resource_approximately(project, resource): def match_purldb_resource_snippets(project, resource): """Match by approximation a single resource in the PurlDB.""" fingerprints = resource.extra_data.get("snippets", "") + filters = get_filtering_kwargs(project) results = SnippetIndex.match_resources( fingerprints=fingerprints, resource=resource, + ecosystems=filters.get("ecosystems"), + exclude_purls=filters.get("exclude_purls"), ) if results: matched_snippets = [] @@ -207,9 +249,12 @@ def match_purldb_resource_snippets(project, resource): def match_purldb_resource_stemmed_snippets(project, resource): """Match by approximation a single resource in the PurlDB.""" fingerprints = resource.extra_data.get("snippets", "") + filters = get_filtering_kwargs(project) results = StemmedSnippetIndex.match_resources( fingerprints=fingerprints, resource=resource, + ecosystems=filters.get("ecosystems"), + exclude_purls=filters.get("exclude_purls"), ) if results: matched_stemmed_snippets = [] @@ -229,8 +274,13 @@ def match_purldb_resource_stemmed_snippets(project, resource): def match_purldb_directory(project, resource, exact_match=False): """Match a single directory resource in the PurlDB.""" fingerprint = resource.extra_data.get("directory_content", "") + filters = get_filtering_kwargs(project) results = ApproximateDirectoryContentIndex.match( - fingerprint=fingerprint, resource=resource, exact_match=exact_match + fingerprint=fingerprint, + resource=resource, + exact_match=exact_match, + ecosystems=filters.get("ecosystems"), + exclude_purls=filters.get("exclude_purls"), ) for result in results: package_data = result.package.to_dict() diff --git a/matchcode/tests/test_api.py b/matchcode/tests/test_api.py index dacbf2b6..988bf574 100644 --- a/matchcode/tests/test_api.py +++ b/matchcode/tests/test_api.py @@ -188,6 +188,26 @@ def test_matchcode_pipeline_api_run_detail(self): self.assertIsNone(response.data["execution_time"]) self.assertEqual(Run.Status.NOT_STARTED, response.data["status"]) + @mock.patch("scanpipe.models.Run.execute_task_async") + def test_matching_pipeline_api_matching_create_with_filters(self, mock_execute_pipeline_task): + data = { + "ecosystems": "maven", + "exclude_purls": "pkg:maven/commons-io/commons-io@2.11.0, pkg:maven/other/other@1.0", + } + + response = self.csrf_client.post(self.matching_list_url, data) + self.assertEqual(status.HTTP_201_CREATED, response.status_code) + + project = Project.objects.get(uuid=response.data["uuid"]) + self.assertEqual(["maven"], project.extra_data["ecosystems"]) + self.assertEqual( + [ + "pkg:maven/commons-io/commons-io@2.11.0", + "pkg:maven/other/other@1.0", + ], + project.extra_data["exclude_purls"], + ) + class D2DPipelineAPITest(TransactionTestCase): data_location = Path(__file__).parent / "data" diff --git a/matchcode/tests/test_models.py b/matchcode/tests/test_models.py index 6292c3df..b5cc3769 100644 --- a/matchcode/tests/test_models.py +++ b/matchcode/tests/test_models.py @@ -126,6 +126,32 @@ def test_ExactPackageArchiveIndex_single_sha1_single_match(self): expected = [self.test_package1_metadata] self.assertEqual(expected, result) + def test_ExactPackageArchiveIndex_match_with_filters(self): + sha1 = self.test_package1.sha1 + + npm_package, _ = Package.objects.get_or_create( + filename="npm-package.tgz", + sha1=sha1, + type="npm", + name="npm-package", + version="1.0.0", + download_url="https://npm.example.com/npm-package.tgz", + ) + ExactPackageArchiveIndex.index(sha1, npm_package) + + # ecosystem filter – only maven packages should remain + results = ExactPackageArchiveIndex.match(sha1, ecosystems=["maven"]) + packages = [r.package for r in results] + self.assertIn(self.test_package1, packages) + self.assertNotIn(npm_package, packages) + + # exclude_purls filter – exclude self.test_package1 + exclude_purls = [self.test_package1.purl] + results = ExactPackageArchiveIndex.match(sha1, exclude_purls=exclude_purls) + packages = [r.package for r in results] + self.assertNotIn(self.test_package1, packages) + self.assertIn(npm_package, packages) + class ExactFileIndexModelTestCase(BaseModelTest): def test_ExactFileIndex_index(self): @@ -307,6 +333,36 @@ def test_ApproximateDirectoryContentIndex_match_subdir(self): ) self.check_codebase(codebase, expected, regen=FIXTURES_REGEN) + def test_ApproximateDirectoryContentIndex_match_with_filters(self): + index = ApproximateDirectoryContentIndex.objects.filter(package=self.test_package1).first() + fingerprint = index.fingerprint() + + # ecosystem filter – no maven packages indexed, so result should be empty + results = ApproximateDirectoryContentIndex.match( + fingerprint=fingerprint, + exact_match=True, + ecosystems=["maven"], + ) + self.assertEqual(results.count(), 0) + + # ecosystem filter – npm should return matches and all should be npm + results = ApproximateDirectoryContentIndex.match( + fingerprint=fingerprint, + exact_match=True, + ecosystems=["npm"], + ) + self.assertTrue(all(match.package.type == "npm" for match in results)) + + # exclude_purls – exclude test_package1 (the package we took the fingerprint from) + exclude_purls = [self.test_package1.purl] + results = ApproximateDirectoryContentIndex.match( + fingerprint=fingerprint, + exact_match=True, + exclude_purls=exclude_purls, + ) + packages = [match.package for match in results] + self.assertNotIn(self.test_package1, packages) + class ApproximateResourceMatchingIndexModelTestCase(MatchcodeTestCase): BASE_DIR = os.path.join(os.path.dirname(__file__), "testfiles") @@ -690,3 +746,41 @@ def test_SnippetIndex_match_resources_match_to_resource_with_less_duplicates(sel expected_match_detections = [Span(0, 153), Span(167, 398)] assert match.match_detections == expected_match_detections assert match.similarity == 0.9206349206349206 + + def test_SnippetIndex_match_with_filters(self): + mixed_fingerprints = self.test_resource1_snippets + self.test_resource3_snippets + + # ecosystem filter – npm only + results = SnippetIndex.match(fingerprints=mixed_fingerprints, ecosystems=["npm"]) + packages = {r.package for r in results} + self.assertEqual(packages, {self.test_package1}) + + # ecosystem filter – github only + results = SnippetIndex.match(fingerprints=mixed_fingerprints, ecosystems=["github"]) + packages = {r.package for r in results} + self.assertEqual(packages, {self.test_package2}) + + # exclude_purls – exclude test_package1 (npm) + exclude_purls = [self.test_package1.purl] + results = SnippetIndex.match(fingerprints=mixed_fingerprints, exclude_purls=exclude_purls) + packages = {r.package for r in results} + self.assertNotIn(self.test_package1, packages) + self.assertIn(self.test_package2, packages) + + def test_SnippetIndex_match_resources_with_filters(self): + test_file_loc = self.get_test_loc("match/approximate-file-matching/index-modified.js") + fingerprints = get_file_fingerprint_hashes(test_file_loc) + snippets = fingerprints["snippets"] + + # ecosystem filter – npm only + matches = SnippetIndex.match_resources(fingerprints=snippets, ecosystems=["npm"]) + self.assertTrue(all(m.package.type == "npm" for m in matches)) + + # ecosystem filter – github only + matches = SnippetIndex.match_resources(fingerprints=snippets, ecosystems=["github"]) + self.assertTrue(all(m.package.type == "github" for m in matches)) + + # exclude_purls – exclude test_package1 (npm) + exclude_purls = [self.test_package1.purl] + matches = SnippetIndex.match_resources(fingerprints=snippets, exclude_purls=exclude_purls) + self.assertFalse(any(m.package == self.test_package1 for m in matches)) diff --git a/matchcode/tests/test_utils.py b/matchcode/tests/test_utils.py new file mode 100644 index 00000000..89eba0d4 --- /dev/null +++ b/matchcode/tests/test_utils.py @@ -0,0 +1,86 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# purldb is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/purldb for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from django.db.models import Q + +from matchcode.utils import MatchcodeTestCase +from matchcode.utils import build_purl_filter + + +class BuildPurlFilterTestCase(MatchcodeTestCase): + def test_build_purl_filter_single_purl_no_prefix(self): + purls = ["pkg:maven/commons-io/commons-io@2.11.0"] + result = build_purl_filter(purls) + + expected = Q( + type="maven", + namespace="commons-io", + name="commons-io", + version="2.11.0", + qualifiers="", + subpath="", + ) + self.assertEqual(expected, result) + + def test_build_purl_filter_single_purl_with_prefix(self): + purls = ["pkg:maven/commons-io/commons-io@2.11.0"] + result = build_purl_filter(purls, relation_prefix="package__") + + expected = Q( + package__type="maven", + package__namespace="commons-io", + package__name="commons-io", + package__version="2.11.0", + package__qualifiers="", + package__subpath="", + ) + self.assertEqual(expected, result) + + def test_build_purl_filter_multiple_purls(self): + purls = [ + "pkg:maven/commons-io/commons-io@2.11.0", + "pkg:npm/lodash@4.17.21", + ] + result = build_purl_filter(purls) + + expected = Q( + type="maven", + namespace="commons-io", + name="commons-io", + version="2.11.0", + qualifiers="", + subpath="", + ) | Q( + type="npm", + namespace="", + name="lodash", + version="4.17.21", + qualifiers="", + subpath="", + ) + self.assertEqual(expected, result) + self.assertEqual(Q.OR, result.connector) + + def test_build_purl_filter_with_qualifiers(self): + purls = ["pkg:maven/commons-io/commons-io@2.11.0?classifier=sources"] + result = build_purl_filter(purls) + + expected = Q( + type="maven", + namespace="commons-io", + name="commons-io", + version="2.11.0", + qualifiers="classifier=sources", + subpath="", + ) + self.assertEqual(expected, result) + + def test_build_purl_filter_empty_list_returns_empty_q(self): + result = build_purl_filter([]) + self.assertEqual(Q(), result) diff --git a/matchcode/utils.py b/matchcode/utils.py index a59cb9b6..0a830cfd 100644 --- a/matchcode/utils.py +++ b/matchcode/utils.py @@ -13,6 +13,7 @@ import posixpath from unittest import TestCase +from django.db.models import Q from django.test import TestCase as DjangoTestCase from commoncode.resource import VirtualCodebase @@ -23,6 +24,8 @@ from matchcode.tests import FIXTURES_REGEN from minecode.utils_test import JsonBasedTestingMixin +from packageurl import PackageURL + ############## TEST UTILITIES ############## """ The conventions used for the tests are: @@ -263,3 +266,32 @@ def index_package_directories(package): vc = compute_codebase_directory_fingerprints(vc) return index_resource_fingerprints(vc, package) + + +def build_purl_filter(exclude_purls, relation_prefix=""): + """ + Return a Q object for filtering packages by their component fields, + given a list of purl strings. + + `relation_prefix` is prepended to each field name to allow filtering + through a foreign key relation (e.g., "package__"). + """ + q = Q() + for purl_str in exclude_purls: + purl = PackageURL.from_string(purl_str) + + qualifiers = purl.qualifiers or {} + if isinstance(qualifiers, dict): + qualifiers = "&".join(f"{key}={value}" for key, value in qualifiers.items()) + + q |= Q( + **{ + f"{relation_prefix}type": purl.type, + f"{relation_prefix}namespace": purl.namespace or "", + f"{relation_prefix}name": purl.name, + f"{relation_prefix}version": purl.version or "", + f"{relation_prefix}qualifiers": qualifiers, + f"{relation_prefix}subpath": purl.subpath or "", + } + ) + return q diff --git a/purldb/settings.py b/purldb/settings.py index badfa0f1..4968d1a7 100644 --- a/purldb/settings.py +++ b/purldb/settings.py @@ -105,6 +105,11 @@ # API DATA_UPLOAD_MAX_NUMBER_FIELDS = env.int("DATA_UPLOAD_MAX_NUMBER_FIELDS", default=2048) +# Allow large webhook payloads from scancode.io +DATA_UPLOAD_MAX_MEMORY_SIZE = env.int( + "DATA_UPLOAD_MAX_MEMORY_SIZE", + default=200 * 1024 * 1024, # 200 MB +) # Database DATABASES = {