Fix pypi purl publishing issues

Signed-off-by: Ayan Sinha Mahapatra <asmahapatra@aboutcode.org>

Author: AyanSinhaMahapatra

minecode_pipelines/pipelines/__init__.py

@@ -193,8 +193,12 @@ def _mine_and_publish_packageurls(
         )
         checkout["file_to_commit"].add(purl_file)
         checkout["file_processed_count"] += 1
+        if logger:
+            logger(f"{checkout['repo'].working_dir}: {checkout['file_processed_count']} / {batch_size}")
 
         if len(checkout["file_to_commit"]) > batch_size:
+            if logger:
+                logger(f"Trying to commit PackageURLs.")
             pipes.commit_and_push_checkout(
                 local_checkout=checkout,
                 commit_message=commit_msg_func(checkout["commit_count"] + 1),

minecode_pipelines/pipelines/mine_pypi.py

@@ -30,7 +30,7 @@ class MinePypi(MineCodeBasePipeline):
     a FederatedCode repo.
     """
 
-    package_batch_size = 1000
+    package_batch_size = 100
 
     @classmethod
     def steps(cls):
@@ -70,7 +70,7 @@ def mine_packageurls(self):
         )
 
     def save_check_point(self):
-        pypi.update_mined_packages_in_checkpoint(
+        pypi.save_mined_packages_in_checkpoint(
             packages_mined=self.packages_mined,
             config_repo=self.config_repo,
             logger=self.log,

minecode_pipelines/pipes/pypi.py

@@ -246,7 +246,7 @@ def mine_and_publish_pypi_packageurls(packages_to_sync, packages_mined, logger=N
 
         # fetch packageURLs for package
         name = package.get("name")
-        if logger:
+        if logger and LOG_PACKAGEURL_DETAILS:
             logger(f"getting packageURLs for package: {name}")
 
         # get repo and path for package
@@ -268,7 +268,7 @@ def mine_and_publish_pypi_packageurls(packages_to_sync, packages_mined, logger=N
         yield base_purl, packageurls
 
 
-def update_mined_packages_in_checkpoint(packages_mined, config_repo, logger=None):
+def save_mined_packages_in_checkpoint(packages_mined, config_repo, logger=None):
     """Update mined packages checkpoint after processing a batch of packages."""
     if logger:
         logger(f"Checkpointing processed packages to: {PYPI_PACKAGES_CHECKPOINT_PATH}")