More WIP with some PyPI tests passing

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

src/packagedcode/__init__.py

@@ -179,7 +179,6 @@
     pypi.PythonSdistPkgInfoFile,
     pypi.PythonSetupPyHandler,
     pypi.SetupCfgHandler,
-    pypi.ToxIniHandler,
 
     readme.ReadmeHandler,
 

src/packagedcode/models.py

@@ -376,7 +376,6 @@ class DependentPackage(ModelMixin):
          )
 
 
-
 @attr.attributes(slots=True)
 class Dependency(DependentPackage):
     """
@@ -664,7 +663,8 @@ class PackageData(IdentifiablePackageData):
 
     datasource_id = String(
         label='datasource id',
-        help='Datasource identifier for the source of these package data.'
+        help='Datasource identifier for the source of these package data.',
+        repr=True,
     )
 
     def to_dict(self, with_details=True, **kwargs):
@@ -850,11 +850,12 @@ def is_datafile(cls, location, filetypes=tuple()):
             loc = as_posixpath(location)
             if any(fnmatch(loc, pat) for pat in cls.path_patterns):
                 filetypes = filetypes or cls.filetypes
-                if filetypes:
+                if not filetypes:
+                    return True
+                else:
                     T = contenttype.get_type(location)
                     actual_type = T.filetype_file.lower()
                     return any(ft in actual_type for ft in filetypes)
-                return True
 
     @classmethod
     def parse(cls, location):
@@ -1033,12 +1034,9 @@ def assemble_from_many(cls, pkgdata_resources, codebase,):
                 )
 
             package_uid = package.package_uid
-
             resource.for_packages.append(package_uid)
             resource.save(codebase)
 
-            yield package
-
             # in all cases yield possible dependencies
             dependent_packages = package_data.dependencies
             if dependent_packages:
@@ -1057,6 +1055,8 @@ def assemble_from_many(cls, pkgdata_resources, codebase,):
             res.for_packages.append(package_uid)
             res.save(codebase)
 
+        yield package
+
     @classmethod
     def assemble_from_many_datafiles(cls, datafile_name_patterns, directory, codebase):
         """

src/packagedcode/plugin_package.py

@@ -156,17 +156,22 @@ def create_package_and_deps(codebase, **kwargs):
         for package_data in resource.package_data:
             package_data = PackageData.from_dict(package_data)
 
+            if TRACE:
+                logger_debug('  create_package_and_deps: package_data:', package_data)
+
             # Find a handler for this package datasource to assemble collect
             # packages and deps
 
             handler = get_package_handler(package_data)
-            for item in handler.assemble(
+            items = handler.assemble(
                 package_data=package_data,
                 resource=resource,
-                codebase=codebase
-            ):
+                codebase=codebase,
+            )
+
+            for item in items:
                 if TRACE:
-                    logger_debug('create_package_and_deps: item:', item,)
+                    logger_debug('    create_package_and_deps: item:', item,)
 
                 if isinstance(item, Package):
                     packages_top_level.append(item)

src/packagedcode/pypi.py

@@ -135,9 +135,10 @@ def assemble(cls, package_data, resource, codebase):
             'Pipfile',
         ) + PipRequirementsFileHandler.path_patterns
 
+        parent = resource.parent(codebase)
         yield from cls.assemble_from_many_datafiles(
             datafile_name_patterns=datafile_name_patterns,
-            directory=resource.parent(codebase),
+            directory=parent,
             codebase=codebase,
         )
 
@@ -357,7 +358,7 @@ def get_file_references(dist):
     if not dist:
         return
 
-    for filepath in dist.files():
+    for filepath in dist.files or []:
         # FIXME: the path is relative to the "site-packages" directory or the
         # root of a wheel but this should be a scan path
         ref = models.FileReference(
@@ -542,19 +543,6 @@ class SetupCfgHandler(BaseExtractedPythonLayout):
     documentation_url = 'https://peps.python.org/pep-0390/'
 
 
-class ToxIniHandler(BaseDependencyFileHandler):
-    datasource_id = 'pypi_tox_ini'
-    path_patterns = ('*tox.ini',)
-    default_package_type = 'pypi'
-    default_primary_language = 'Python'
-    description = 'Tox tox.ini'
-    documentation_url = 'https://tox.wiki'
-
-    @classmethod
-    def assemble(cls, package_data, resource, codebase):
-        pass
-
-
 class PipfileHandler(BaseDependencyFileHandler):
     datasource_id = 'pipfile'
     path_patterns = ('*Pipfile',)
@@ -635,7 +623,7 @@ class PipRequirementsFileHandler(BaseDependencyFileHandler):
         '*requirements/*.txt',
         '*requirements/*.pip',
         '*requirements/*.in',
-        '*reqs.txt'
+        '*reqs.txt',
     )
 
     default_package_type = 'pypi'
@@ -1047,7 +1035,6 @@ def get_dparse2_supported_file_name(file_name):
         'Pipfile',
         'conda.yml',
         'setup.cfg',
-        'tox.ini',
     )
 
     for dfile_name in dfile_names: