Matching for a specific ecosystem and without matching itself

matchcode/api.py

@@ -265,11 +265,15 @@ def match(self, request):
         if not fingerprints:
             return Response()
 
+        ecosystems = request.query_params.getlist("ecosystems")
+        exclude_purls = request.query_params.getlist("exclude_purls")
         model_class = self.get_serializer().Meta.model
         results = []
         unique_fingerprints = set(fingerprints)
         for fingerprint in unique_fingerprints:
-            matches = model_class.match(fingerprint)
+            matches = model_class.match(
+                fingerprint, ecosystems=ecosystems, exclude_purls=exclude_purls
+            )
             for match in matches:
                 _, bah128 = split_fingerprint(fingerprint)
                 # Get fingerprint from the match
@@ -357,6 +361,30 @@ class MatchingSerializer(ExcludeFromListViewMixin, serializers.ModelSerializer):
     discovered_dependencies_summary = serializers.SerializerMethodField()
     codebase_relations_summary = serializers.SerializerMethodField()
 
+    ecosystems = serializers.ChoiceField(
+        choices=(
+            ("", "---------"),
+            ("maven", "maven"),
+        ),
+        required=False,
+        allow_blank=True,
+        default="",
+        write_only=True,
+        help_text="Ecosystem to restrict the match index.",
+    )
+
+    exclude_purls = serializers.CharField(
+        required=False,
+        allow_blank=True,
+        default="",
+        write_only=True,
+        style={"base_template": "textarea.html"},
+        help_text="Exclude PURLs (space or comma separated).",
+    )
+
+    ecosystems_filter = serializers.SerializerMethodField(read_only=True)
+    exclude_purls_filter = serializers.SerializerMethodField(read_only=True)
+
     class Meta:
         model = Project
         fields = (
@@ -376,6 +404,10 @@ class Meta:
             "discovered_packages_summary",
             "discovered_dependencies_summary",
             "codebase_relations_summary",
+            "ecosystems",
+            "exclude_purls",
+            "ecosystems_filter",
+            "exclude_purls_filter",
         )
         exclude_from_list_view = [
             "resource_count",
@@ -419,6 +451,12 @@ def get_codebase_relations_summary(self, project):
         queryset = project.codebaserelations.all()
         return count_group_by(queryset, "map_type")
 
+    def get_ecosystems_filter(self, project):
+        return (project.extra_data or {}).get("ecosystems", [])
+
+    def get_exclude_purls_filter(self, project):
+        return (project.extra_data or {}).get("exclude_purls", [])
+
     def validate_input_urls(self, value):
         """Add support for providing multiple URLs in a single string."""
         return [url for entry in value for url in entry.split()]
@@ -430,13 +468,35 @@ def create(self, validated_data, matching_pipeline_name="matching"):
         upload_file = validated_data.pop("upload_file", None)
         input_urls = validated_data.pop("input_urls", [])
         webhook_url = validated_data.pop("webhook_url", None)
+        ecosystems = validated_data.pop("ecosystems", "")
+        exclude_purls = validated_data.pop("exclude_purls", "")
+
+        # Convert ecosystems to a list
+        if isinstance(ecosystems, str):
+            ecosystems = [ecosystems] if ecosystems else []
+
+        # Convert exclude_purls to a list; support spaces, commas, and newlines
+        if isinstance(exclude_purls, str):
+            exclude_purls = [
+                purl.strip() for purl in exclude_purls.replace(",", " ").split() if purl.strip()
+            ]
 
         downloads, errors = fetch_urls(input_urls)
         if errors:
             raise serializers.ValidationError("Could not fetch: " + "\n".join(errors))
 
         project = super().create(validated_data)
 
+        project.extra_data = project.extra_data or {}
+        if ecosystems:
+            project.extra_data["ecosystems"] = ecosystems
+
+        if exclude_purls:
+            project.extra_data["exclude_purls"] = exclude_purls
+
+        if ecosystems or exclude_purls:
+            project.save()
+
         if upload_file:
             project.add_uploads([upload_file])
 
@@ -580,7 +640,7 @@ class MatchingViewSet(
     """
     Take a ScanCode.io JSON of a codebase `upload_file` or from a list of
     `input_urls` and run the ``matching`` pipeline
-    (https://github.com/aboutcode-org/purldb/blob/main/matchcode_pipeline/pipelines/matching.py)
+    (https://github.com/aboutcode-org/purldb/blob/main/matchcode/pipelines/matching.py)
     on it.
 
     The ``matching`` pipeline matches directory and resources of the codebase in

matchcode/models.py

@@ -33,6 +33,8 @@
 from packagedb.models import Package
 from packagedb.models import Resource
 
+from matchcode.utils import build_purl_filter
+
 TRACE = False
 
 if TRACE:
@@ -94,7 +96,7 @@ def index(cls, sha1, package):
             logger.error(msg)
 
     @classmethod
-    def match(cls, sha1):
+    def match(cls, sha1, ecosystems=None, exclude_purls=None):
         """Return a list of matched Packages that contains a file with a SHA1 value of `sha1`"""
         if TRACE:
             logger_debug(cls.__name__, "match:", "sha1:", sha1)
@@ -104,6 +106,10 @@ def match(cls, sha1):
 
         sha1_in_bin = hexstring_to_binarray(sha1)
         matches = cls.objects.filter(sha1=sha1_in_bin)
+        if ecosystems:
+            matches = matches.filter(package__type__in=ecosystems)
+        if exclude_purls:
+            matches = matches.exclude(build_purl_filter(exclude_purls, relation_prefix="package__"))
         if TRACE:
             for match in matches:
                 package = match.package
@@ -226,7 +232,9 @@ def index(cls, fingerprint, resource_path, package):
             logger.error(msg)
 
     @classmethod
-    def match(cls, fingerprint, resource=None, exact_match=False):
+    def match(
+        cls, fingerprint, resource=None, exact_match=False, ecosystems=None, exclude_purls=None
+    ):
         """Return a list of matched Packages"""
         if TRACE:
             logger_debug(
@@ -253,6 +261,12 @@ def match(cls, fingerprint, resource=None, exact_match=False):
                 chunk3=chunk3,
                 chunk4=chunk4,
             )
+            if ecosystems:
+                matches = matches.filter(package__type__in=ecosystems)
+            if exclude_purls:
+                matches = matches.exclude(
+                    build_purl_filter(exclude_purls, relation_prefix="package__")
+                )
             return matches
 
         # Step 1: find fingerprints with matching chunks
@@ -264,6 +278,11 @@ def match(cls, fingerprint, resource=None, exact_match=False):
             | models.Q(indexed_elements_count__range=frange, chunk4=chunk4)
         )
 
+        if ecosystems:
+            matches = matches.filter(package__type__in=ecosystems)
+        if exclude_purls:
+            matches = matches.exclude(build_purl_filter(exclude_purls, relation_prefix="package__"))
+
         if TRACE:
             for match in matches:
                 dct = model_to_dict(match)
@@ -478,7 +497,7 @@ def index(cls, fingerprint, position, resource, package):
             logger.error(msg)
 
     @classmethod
-    def match(cls, fingerprints):
+    def match(cls, fingerprints, ecosystems=None, exclude_purls=None):
         """
         Return a list of PackageSnippetMatch for matched Package.
         """
@@ -499,6 +518,13 @@ def match(cls, fingerprints):
         # Step 0: get all fingerprint records that match with the input
         matched_fps = cls.objects.filter(fingerprint__in=only_fings)
 
+        if ecosystems:
+            matched_fps = matched_fps.filter(package__type__in=ecosystems)
+        if exclude_purls:
+            matched_fps = matched_fps.exclude(
+                build_purl_filter(exclude_purls, relation_prefix="package__")
+            )
+
         # Step 1: count Packages whose fingerprints appear
         # Step 1.1: get Packages that show up in the query
         packages = set(f.package for f in matched_fps.iterator())
@@ -518,7 +544,7 @@ def match(cls, fingerprints):
         return matches
 
     @classmethod
-    def match_resources(cls, fingerprints, top=None, **kwargs):
+    def match_resources(cls, fingerprints, top=None, ecosystems=None, exclude_purls=None, **kwargs):
         """
         Return a list of ResourceSnippetMatch for matched Resources.
         Only return the ``top`` matches, or all matches if ``top`` is zero.
@@ -562,6 +588,13 @@ def match_resources(cls, fingerprints, top=None, **kwargs):
         # Step 0: get all fingerprint records that match with the input
         matched_fps = cls.objects.filter(fingerprint__in=only_fings)
 
+        if ecosystems:
+            matched_fps = matched_fps.filter(package__type__in=ecosystems)
+        if exclude_purls:
+            matched_fps = matched_fps.exclude(
+                build_purl_filter(exclude_purls, relation_prefix="package__")
+            )
+
         # Step 1: get Resources that show up in the query
         resources = set(f.resource for f in matched_fps.iterator())
 

matchcode/pipes/matching.py

@@ -38,6 +38,23 @@
 from packagedb.models import Package
 from packagedb.models import Resource
 
+from matchcode.utils import build_purl_filter
+
+
+def get_filtering_kwargs(project):
+    """
+    Extract ecosystems and exclude_purls from the project's extra_data.
+    """
+    kwargs = {}
+    if project and project.extra_data:
+        ecosystems = project.extra_data.get("ecosystems")
+        if ecosystems:
+            kwargs["ecosystems"] = ecosystems
+        exclude_purls = project.extra_data.get("exclude_purls")
+        if exclude_purls:
+            kwargs["exclude_purls"] = exclude_purls
+    return kwargs
+
 
 def get_project_resources_qs(project, resources):
     """
@@ -112,7 +129,15 @@ def match_purldb_package(project, resources_by_sha1, enhance_package_data=True,
     """
     match_count = 0
     sha1_list = list(resources_by_sha1.keys())
-    results = Package.objects.filter(sha1__in=sha1_list).order_by()
+    results = Package.objects.filter(sha1__in=sha1_list)
+
+    filters = get_filtering_kwargs(project)
+    if "ecosystems" in filters:
+        results = results.filter(type__in=filters["ecosystems"])
+    if "exclude_purls" in filters:
+        results = results.exclude(build_purl_filter(filters["exclude_purls"]))
+
+    results = results.order_by()
     # Process matched Package data
     for package in results:
         package_data = package.to_dict()
@@ -145,11 +170,19 @@ def match_purldb_resource(project, resources_by_sha1, package_data_by_purldb_url
     match_count = 0
     sha1_list = list(resources_by_sha1.keys())
     results = (
-        Resource.objects.filter(sha1__in=sha1_list)
-        .select_related("package")
-        .only("package__uuid")
-        .order_by()
+        Resource.objects.filter(sha1__in=sha1_list).select_related("package").only("package__uuid")
     )
+
+    filters = get_filtering_kwargs(project)
+    if "ecosystems" in filters:
+        results = results.filter(package__type__in=filters["ecosystems"])
+    if "exclude_purls" in filters:
+        results = results.exclude(
+            build_purl_filter(filters["exclude_purls"], relation_prefix="package__")
+        )
+
+    results = results.order_by()
+
     # Process match results
     for resource in results:
         # Get package data
@@ -171,7 +204,13 @@ def match_purldb_resource(project, resources_by_sha1, package_data_by_purldb_url
 def match_purldb_resource_approximately(project, resource):
     """Match by approximation a single resource in the PurlDB."""
     fingerprint = resource.extra_data.get("halo1", "")
-    results = ApproximateResourceContentIndex.match(fingerprint=fingerprint, resource=resource)
+    filters = get_filtering_kwargs(project)
+    results = ApproximateResourceContentIndex.match(
+        fingerprint=fingerprint,
+        resource=resource,
+        ecosystems=filters.get("ecosystems"),
+        exclude_purls=filters.get("exclude_purls"),
+    )
     for result in results:
         package_data = result.package.to_dict()
         return create_package_from_purldb_data(
@@ -185,9 +224,12 @@ def match_purldb_resource_approximately(project, resource):
 def match_purldb_resource_snippets(project, resource):
     """Match by approximation a single resource in the PurlDB."""
     fingerprints = resource.extra_data.get("snippets", "")
+    filters = get_filtering_kwargs(project)
     results = SnippetIndex.match_resources(
         fingerprints=fingerprints,
         resource=resource,
+        ecosystems=filters.get("ecosystems"),
+        exclude_purls=filters.get("exclude_purls"),
     )
     if results:
         matched_snippets = []
@@ -207,9 +249,12 @@ def match_purldb_resource_snippets(project, resource):
 def match_purldb_resource_stemmed_snippets(project, resource):
     """Match by approximation a single resource in the PurlDB."""
     fingerprints = resource.extra_data.get("snippets", "")
+    filters = get_filtering_kwargs(project)
     results = StemmedSnippetIndex.match_resources(
         fingerprints=fingerprints,
         resource=resource,
+        ecosystems=filters.get("ecosystems"),
+        exclude_purls=filters.get("exclude_purls"),
     )
     if results:
         matched_stemmed_snippets = []
@@ -229,8 +274,13 @@ def match_purldb_resource_stemmed_snippets(project, resource):
 def match_purldb_directory(project, resource, exact_match=False):
     """Match a single directory resource in the PurlDB."""
     fingerprint = resource.extra_data.get("directory_content", "")
+    filters = get_filtering_kwargs(project)
     results = ApproximateDirectoryContentIndex.match(
-        fingerprint=fingerprint, resource=resource, exact_match=exact_match
+        fingerprint=fingerprint,
+        resource=resource,
+        exact_match=exact_match,
+        ecosystems=filters.get("ecosystems"),
+        exclude_purls=filters.get("exclude_purls"),
     )
     for result in results:
         package_data = result.package.to_dict()

matchcode/tests/test_api.py

@@ -188,6 +188,26 @@ def test_matchcode_pipeline_api_run_detail(self):
         self.assertIsNone(response.data["execution_time"])
         self.assertEqual(Run.Status.NOT_STARTED, response.data["status"])
 
+    @mock.patch("scanpipe.models.Run.execute_task_async")
+    def test_matching_pipeline_api_matching_create_with_filters(self, mock_execute_pipeline_task):
+        data = {
+            "ecosystems": "maven",
+            "exclude_purls": "pkg:maven/commons-io/commons-io@2.11.0, pkg:maven/other/other@1.0",
+        }
+
+        response = self.csrf_client.post(self.matching_list_url, data)
+        self.assertEqual(status.HTTP_201_CREATED, response.status_code)
+
+        project = Project.objects.get(uuid=response.data["uuid"])
+        self.assertEqual(["maven"], project.extra_data["ecosystems"])
+        self.assertEqual(
+            [
+                "pkg:maven/commons-io/commons-io@2.11.0",
+                "pkg:maven/other/other@1.0",
+            ],
+            project.extra_data["exclude_purls"],
+        )
+
 
 class D2DPipelineAPITest(TransactionTestCase):
     data_location = Path(__file__).parent / "data"