send errors to stderr

Signed-off-by: tdruez <tdruez@aboutcode.org>

Author: tdruez

dejacode_toolkit/vulnerablecode.py

@@ -60,6 +60,7 @@ def get_vulnerable_purls(self, packages, details=False, timeout=10):
         """
         Return a list of PURLs for which at least one `affected_by_vulnerabilities`
         was found in the VulnerableCodeDB for the given list of `packages`.
+        Returns None when the API call fails (e.g. timeout or network error).
         """
         plain_purls = get_plain_purls(packages)
 
@@ -71,7 +72,9 @@ def get_vulnerable_purls(self, packages, details=False, timeout=10):
             details=details,
             timeout=timeout,
         )
-        return (vulnerable_purls or {}).get("results") or []
+        if vulnerable_purls is None:
+            return None
+        return vulnerable_purls.get("results") or []
 
     def get_package_url_available_types(self):
         """Return the list of supported package types from the VulnerableCode API."""

vulnerabilities/fetch.py

@@ -6,6 +6,7 @@
 # See https://aboutcode.org for more information about AboutCode FOSS projects.
 #
 
+import logging
 from timeit import default_timer as timer
 
 from django.contrib.contenttypes.models import ContentType
@@ -19,14 +20,19 @@
 from component_catalog.models import Package
 from component_catalog.models import PackageAffectedByVulnerability
 from dejacode_toolkit.vulnerablecode import VulnerableCode
+from dejacode_toolkit.vulnerablecode import get_plain_purls
 from dje.models import DejacodeUser
 from dje.utils import chunked_queryset
 from dje.utils import humanize_time
 from notification.models import find_and_fire_hook
 from vulnerabilities.models import Vulnerability
 
+logger = logging.getLogger("dje")
 
-def fetch_from_vulnerablecode(dataspace, batch_size, update, timeout, log_func=None, verbosity=1):
+
+def fetch_from_vulnerablecode(
+    dataspace, batch_size, update, timeout, log_func=None, err_func=None, verbosity=1
+):
     """Fetch vulnerability data from VulnerableCode for all eligible packages in ``dataspace``."""
     start_time = timer()
     vulnerablecode = VulnerableCode(dataspace)
@@ -57,6 +63,7 @@ def fetch_from_vulnerablecode(dataspace, batch_size, update, timeout, log_func=N
         update=update,
         timeout=timeout,
         log_func=log_func,
+        err_func=err_func,
         verbosity=verbosity,
     )
 
@@ -73,7 +80,14 @@ def fetch_from_vulnerablecode(dataspace, batch_size, update, timeout, log_func=N
 
 
 def fetch_for_packages(
-    queryset, dataspace, batch_size=50, update=True, timeout=None, log_func=None, verbosity=1
+    queryset,
+    dataspace,
+    batch_size=50,
+    update=True,
+    timeout=None,
+    log_func=None,
+    err_func=None,
+    verbosity=1,
 ):
     from product_portfolio.models import ProductPackage
 
@@ -100,6 +114,21 @@ def fetch_for_packages(
         vc_entries = vulnerablecode.get_vulnerable_purls(batch, details=True, timeout=timeout)
         api_elapsed = timer() - api_start
 
+        if vc_entries is None:
+            failed_purls = get_plain_purls(batch)
+            error_msg = (
+                f"VulnerableCode API call failed for batch {index} "
+                f"({len(failed_purls)} purls, "
+                f"progress: {intcomma(progress_count)}/{intcomma(object_count)}). "
+                f"Purls: {' '.join(failed_purls)}"
+            )
+            logger.error(error_msg)
+            if err_func:
+                err_func(error_msg)
+            elif log_func:
+                log_func(f"  API call failed for batch {index} - skipping.")
+            continue
+
         if log_func and verbosity >= 2:
             log_func(
                 f"  API call: {humanize_time(api_elapsed)} ({len(vc_entries)} vulnerable purls)"

vulnerabilities/management/commands/fetchvulnerabilities.py

@@ -14,7 +14,11 @@
 
 
 class Command(DataspacedCommand):
-    help = "Fetch vulnerabilities for the provided Dataspace"
+    help = (
+        "Fetch vulnerabilities for the provided Dataspace. "
+        "Progress is written to stdout; API errors (timeouts, network failures) go to stderr. "
+        "To capture errors separately: ./manage.py fetchvulnerabilities 2>errors.log"
+    )
 
     def add_arguments(self, parser):
         super().add_arguments(parser)
@@ -55,6 +59,7 @@ def handle(self, *args, **options):
                 update=True,
                 timeout=timeout,
                 log_func=self.stdout.write,
+                err_func=self.stderr.write,
                 verbosity=options["verbosity"],
             )
         except ValueError as error:

vulnerabilities/models.py

@@ -77,12 +77,11 @@ def added_or_updated_today(self):
         return self.filter(last_modified_date__gte=today)
 
 
-# AdvisoryV2
 class Vulnerability(HistoryDateFieldsMixin, DataspacedModel):
     """
     A software vulnerability with a unique identifier and alternate aliases.
 
-    Adapted from the VulnerableCode models at
+    Adapted from the VulnerableCode AdvisoryV2 model at
     https://github.com/nexB/vulnerablecode/blob/main/vulnerabilities/models.py
 
     Note that this model implements the HistoryDateFieldsMixin but not the