fix

Author: Adityakk9031

minecode_pipelines/pipelines/mine_meson.py

@@ -20,8 +20,7 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
-import json
-from pathlib import Path
+import requests
 
 from minecode_pipelines.pipes import meson
 from minecode_pipelines.pipelines import MineCodeBasePipeline
@@ -31,38 +30,34 @@
 class MineMeson(MineCodeBasePipeline):
     """Pipeline to mine Meson WrapDB packages and publish them to FederatedCode repo."""
 
-    MESON_WRAPDB_REPO = "https://github.com/mesonbuild/wrapdb"
+    MESON_WRAPDB_RELEASES_URL = (
+        "https://raw.githubusercontent.com/mesonbuild/wrapdb/master/releases.json"
+    )
 
     @classmethod
     def steps(cls):
         return (
             cls.check_federatedcode_eligibility,
             cls.create_federatedcode_working_dir,
-            cls.clone_wrapdb_index,
+            cls.fetch_wrapdb_releases,
             cls.fetch_federation_config,
             cls.mine_and_publish_packageurls,
             cls.delete_working_dir,
         )
 
-    def clone_wrapdb_index(self):
-        """Clone the Meson WrapDB repository."""
-        self.wrapdb_repo = federatedcode.clone_repository(
-            repo_url=self.MESON_WRAPDB_REPO,
-            clone_path=self.working_path / "wrapdb",
-            logger=self.log,
-        )
+    def fetch_wrapdb_releases(self):
+        """Fetch the Meson WrapDB releases.json index."""
+        try:
+            response = requests.get(self.MESON_WRAPDB_RELEASES_URL, timeout=30)
+            response.raise_for_status()
+            self.releases = response.json()
+        except Exception as e:
+            self.log(f"Failed to fetch releases.json: {e}")
+            self.releases = {}
 
     def packages_count(self):
-        releases_path = Path(self.wrapdb_repo.working_dir) / "releases.json"
-        if not releases_path.exists():
-            return 0
-        with open(releases_path, encoding="utf-8") as f:
-            releases = json.load(f)
-        return len(releases)
+        return len(self.releases) if hasattr(self, "releases") and self.releases else 0
 
     def mine_packageurls(self):
         """Yield PackageURLs from Meson WrapDB releases.json."""
-        return meson.mine_meson_packageurls(
-            wrapdb_repo=self.wrapdb_repo,
-            logger=self.log,
-        )
+        return meson.mine_meson_packageurls(releases=self.releases, logger=self.log)

minecode_pipelines/pipes/meson.py

@@ -20,9 +20,6 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
-import json
-from pathlib import Path
-
 from packageurl import PackageURL
 
 
@@ -59,19 +56,11 @@ def get_meson_packages(package_name, package_data):
     return base_purl, versioned_purls
 
 
-def mine_meson_packageurls(wrapdb_repo, logger):
+def mine_meson_packageurls(releases, logger):
     """
-    Yield ``(base_purl, [versioned_purl_strings])`` tuples from a cloned
-    Meson WrapDB repository by parsing its ``releases.json``.
+    Yield ``(base_purl, [versioned_purl_strings])`` tuples from a
+    Meson WrapDB ``releases.json`` mapping.
     """
-    releases_path = Path(wrapdb_repo.working_dir) / "releases.json"
-    if not releases_path.exists():
-        logger(f"releases.json not found at {releases_path}")
-        return
-
-    with open(releases_path, encoding="utf-8") as f:
-        releases = json.load(f)
-
     for package_name, package_data in releases.items():
         if not package_data:
             continue

minecode_pipelines/tests/pipes/test_meson.py

@@ -15,6 +15,7 @@
 from minecode_pipelines.pipes.meson import get_meson_packages
 
 DATA_DIR = Path(__file__).parent.parent / "test_data" / "meson"
+EXPECTED_PATH = DATA_DIR / "expected_purls.json"
 
 
 class MesonPipeTests(TestCase):
@@ -53,36 +54,22 @@ def test_get_meson_packages_no_versions_key(self):
         self.assertEqual(versioned_purls, [])
 
     def test_get_meson_packages_from_releases_json(self):
-        """Test parsing packages from the test releases.json fixture."""
+        """Test parsing packages from the test releases.json fixture with data-driven expected output."""
         releases_path = DATA_DIR / "releases.json"
         with open(releases_path, encoding="utf-8") as f:
             releases = json.load(f)
 
-        all_results = []
+        actual = {}
         for package_name, package_data in releases.items():
             if not package_data:
                 continue
-            all_results.append(
-                get_meson_packages(
-                    package_name=package_name,
-                    package_data=package_data,
-                )
+            base_purl, versioned_purls = get_meson_packages(
+                package_name=package_name,
+                package_data=package_data,
             )
+            actual[str(base_purl)] = sorted(versioned_purls)
 
-        self.assertEqual(len(all_results), 3)  # ogg, zlib, catch2
-
-        # Check ogg
-        ogg_base, ogg_purls = all_results[0]
-        self.assertEqual(str(ogg_base), "pkg:meson/ogg")
-        self.assertEqual(len(ogg_purls), 4)
-
-        # Check zlib
-        zlib_base, zlib_purls = all_results[1]
-        self.assertEqual(str(zlib_base), "pkg:meson/zlib")
-        self.assertEqual(len(zlib_purls), 3)
-
-        # Check catch2
-        catch2_base, catch2_purls = all_results[2]
-        self.assertEqual(str(catch2_base), "pkg:meson/catch2")
-        self.assertEqual(len(catch2_purls), 2)
-        self.assertIn("pkg:meson/catch2@3.5.2-1", catch2_purls)
+        with open(EXPECTED_PATH, encoding="utf-8") as ef:
+            expected = json.load(ef)
+        filtered_actual = {k: actual[k] for k in expected.keys()}
+        self.assertEqual(filtered_actual, expected)

minecode_pipelines/tests/test_data/meson/expected_purls.json

@@ -0,0 +1,17 @@
+{
+  "pkg:meson/ogg": [
+    "pkg:meson/ogg@1.3.5-1",
+    "pkg:meson/ogg@1.3.5-2",
+    "pkg:meson/ogg@1.3.5-3",
+    "pkg:meson/ogg@1.3.6-1"
+  ],
+  "pkg:meson/zlib": [
+    "pkg:meson/zlib@1.2.13-1",
+    "pkg:meson/zlib@1.2.13-2",
+    "pkg:meson/zlib@1.3.1-1"
+  ],
+  "pkg:meson/catch2": [
+    "pkg:meson/catch2@3.4.0-1",
+    "pkg:meson/catch2@3.5.2-1"
+  ]
+}