Support indexing of directory listed and GNU packages

minecode/tests/test_generic.py

@@ -7,17 +7,15 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-import os
-
 from django.test import TestCase as DjangoTestCase
+from packagedcode.maven import _parse
+from packageurl import PackageURL
 
+from minecode.route import NoRouteAvailable
 from minecode.utils_test import JsonBasedTesting
 from minecode.visitors import generic
 from packagedb.models import Package
 
-from packagedcode.maven import _parse
-from packageurl import PackageURL
-
 
 class GenericPriorityQueueTests(JsonBasedTesting, DjangoTestCase):
     def test_process_request(self):
@@ -40,15 +38,10 @@ def test_process_request_no_download_url(self):
         package_count = Package.objects.all().count()
         self.assertEqual(0, package_count)
 
-        purl = 'pkg:generic/test@1.0.0'
-        error_msg = generic.process_request(purl)
+        purl = "pkg:generic/test@1.0.0"
 
-        self.assertEqual(
-            f'package_url {purl} does not contain a download_url qualifier',
-            error_msg
-        )
-        package_count = Package.objects.all().count()
-        self.assertEqual(0, package_count)
+        with self.assertRaises(NoRouteAvailable):
+            generic.process_request(purl)
 
     def test_map_generic_package(self):
         package_count = Package.objects.all().count()
@@ -66,3 +59,22 @@ def test_map_generic_package(self):
         self.assertEqual('test', package.name)
         self.assertEqual('1.0.0', package.version)
         self.assertEqual('http://example.com/test.tar.gz', package.download_url)
+
+    def test_process_request_dir_listed(self):
+        package_count = Package.objects.all().count()
+        self.assertEqual(0, package_count)
+
+        purl = "pkg:generic/ipkg@0.99.33"
+        error_msg = generic.process_request_dir_listed(purl)
+
+        self.assertEqual(None, error_msg)
+        package_count = Package.objects.all().count()
+        self.assertEqual(1, package_count)
+
+        package = Package.objects.first()
+        self.assertEqual("ipkg", package.name)
+        self.assertEqual("0.99.33", package.version)
+        self.assertEqual(
+            "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg_0.99.33_arm.ipk",
+            package.download_url,
+        )

minecode/tests/test_gnu.py

@@ -0,0 +1,61 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+
+import os
+
+from django.test import TestCase
+from mock import patch
+
+from minecode.utils_test import JsonBasedTesting
+from minecode.visitors import gnu
+from packagedb.models import Package
+
+
+class GnuPriorityQueueTests(JsonBasedTesting, TestCase):
+    test_data_dir = os.path.join(os.path.dirname(__file__), "testfiles")
+
+    def setUp(self):
+        super(GnuPriorityQueueTests, self).setUp()
+        glibc_data_loc = self.get_test_loc("gnu/glibc/index.html")
+
+        with open(glibc_data_loc) as f:
+            self.glibc_data_content = f.read()
+
+    @patch("requests.get")
+    def test_process_request(self, mock_get):
+        mock_get.side_effect = [
+            type(
+                "Response",
+                (),
+                {
+                    "content": self.glibc_data_content.encode(),
+                    "raise_for_status": lambda: None,
+                },
+            )
+        ]
+
+        package_count = Package.objects.all().count()
+        self.assertEqual(0, package_count)
+
+        purl = "pkg:gnu/glibc@2.15"
+        error_msg = gnu.process_request(purl)
+
+        self.assertEqual(None, error_msg)
+        package_count = Package.objects.all().count()
+        self.assertEqual(1, package_count)
+
+        package = Package.objects.first()
+        self.assertEqual("glibc", package.name)
+        self.assertEqual("2.15", package.version)
+        print(package.download_url)
+        self.assertEqual(
+            "https://ftp.gnu.org/pub/gnu/glibc/glibc-2.15.tar.gz",
+            package.download_url,
+        )