Merge latest

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

docs/source/contribute/contrib_dev.rst

@@ -109,7 +109,20 @@ These are enabled by adding a ``--test-suite`` option to the pytest command.
   extensive data-driven and data validations (for package, copyright and license
   detection)
 
+In some cases we need to regenerate test data when expected behavious/result data
+structures change, and we have an environement variable to regenerate test data.
+`SCANCODE_REGEN_TEST_FIXTURES` is present in `scancode_config` and this can be
+set to regenerate test data for specific tests like this:
 
+``SCANCODE_REGEN_TEST_FIXTURES=yes pytest -vvs tests/packagedcode/test_package_models.py``
+
+This command will only regenerate test data for only the tests in `test_package_models.py`,
+and we can further specify the tests to regen by using more pytest options like `--lf` and
+`-k test_instances`. 
+
+If test data is regenerated, it is important to review the diff for test files and
+carefully go through all of it to make sure there are no unintended changes there,
+and then commit all the regenerated test data.
 
 .. _scancode_toolkit_development_thirdparty_libraries:
 

src/packagedcode/__init__.py

@@ -29,7 +29,6 @@
 from packagedcode import jar_manifest
 from packagedcode import maven
 from packagedcode import misc
-from packagedcode import models
 from packagedcode import msi
 from packagedcode import npm
 from packagedcode import nuget

src/packagedcode/bower.py

@@ -108,7 +108,9 @@ def parse(cls, location):
                 )
             )
 
-        yield cls(
+        yield models.PackageData(
+            datasource_id=cls.datasource_id,
+            type=cls.default_package_type,
             name=name,
             description=description,
             version=version,

src/packagedcode/cocoapods.py

@@ -187,7 +187,7 @@ def parse(cls, location):
             if isinstance(pod, dict):
                 for main_pod, _dep_pods in pod.items():
 
-                    purl, xreq = parse_dep_requirements(main_pod).to_string()
+                    purl, xreq = parse_dep_requirements(main_pod)
 
                     dependencies.append(
                         models.DependentPackage(
@@ -203,7 +203,7 @@ def parse(cls, location):
 
             elif isinstance(pod, str):
 
-                purl, xreq = parse_dep_requirements(pod).to_string()
+                purl, xreq = parse_dep_requirements(pod)
 
                 dependencies.append(
                     models.DependentPackage(
@@ -218,7 +218,7 @@ def parse(cls, location):
                 )
 
         yield models.PackageData(
-            datasouource_id=cls.datasource_id,
+            datasource_id=cls.datasource_id,
             type=cls.default_package_type,
             primary_language=cls.default_primary_language,
             dependencies=dependencies,

src/packagedcode/freebsd.py

@@ -51,6 +51,8 @@ def parse(cls, location):
             freebsd_manifest = saneyaml.load(loc)
 
         package_data = models.PackageData(
+            datasource_id=cls.datasource_id,
+            type=cls.default_package_type,
             qualifiers=dict(
                 arch=freebsd_manifest.get('arch'),
                 origin=freebsd_manifest.get('origin'),

src/packagedcode/haxe.py

@@ -63,39 +63,40 @@ def parse(cls, location):
         }
         """
         with io.open(location, encoding='utf-8') as loc:
-            package_data = json.load(loc)
+            json_data = json.load(loc)
 
-        name = package_data.get('name')
-        version = package_data.get('version')
+        name = json_data.get('name')
+        version = json_data.get('version')
 
         package_data = models.PackageData(
             datasource_id=cls.datasource_id,
             type=cls.default_package_type,
             name=name,
             version=version,
-            homepage_url=package_data.get('url'),
-            declared_license=package_data.get('license'),
-            keywords=package_data.get('tags'),
-            description=package_data.get('description'),
+            homepage_url=json_data.get('url'),
+            declared_license=json_data.get('license'),
+            keywords=json_data.get('tags'),
+            description=json_data.get('description'),
+            primary_language=cls.default_primary_language,
         )
 
         if name and version:
-            download_url = f'https://lib.haxe.org/p//{name}/{version}/download/'
+            download_url = f'https://lib.haxe.org/p/{name}/{version}/download/'
             package_data.repository_download_url = download_url
             package_data.download_url = download_url
 
         if name:
             package_data.repository_homepage_url = f'https://lib.haxe.org/p/{name}'
 
-        for contrib in package_data.get('contributors', []):
+        for contrib in json_data.get('contributors', []):
             party = models.Party(
                 type=models.party_person,
                 name=contrib,
                 role='contributor',
                 url='https://lib.haxe.org/u/{}'.format(contrib))
             package_data.parties.append(party)
 
-        for dep_name, dep_version in package_data.get('dependencies', {}).items():
+        for dep_name, dep_version in json_data.get('dependencies', {}).items():
             dep_version = dep_version and dep_version.strip()
             is_resolved = bool(dep_version)
             dep_purl = PackageURL(

src/packagedcode/maven.py

@@ -1153,7 +1153,8 @@ def parse(
 
     scm = pom.scm or {}
     urls = build_vcs_and_code_view_urls(scm)
-    urls .update(get_urls(
+    print(urls)
+    urls.update(get_urls(
         namespace=group_id,
         name=artifact_id, version=version,
         qualifiers=qualifiers,

src/packagedcode/opam.py

@@ -22,7 +22,7 @@ class OpamFileHandler(models.DatafileHandler):
     datasource_id = 'opam_file'
     path_patterns = ('*opam',)
     default_package_type = 'opam'
-    default_primary_language = 'OCaml'
+    default_primary_language = 'Ocaml'
     description = 'Ocaml Opam file'
     documentation_url = 'https://opam.ocaml.org/doc/Manual.html#Common-file-format'
 
@@ -39,8 +39,8 @@ def parse(cls, location):
         for dep in deps:
             package_dependencies.append(
                 models.DependentPackage(
-                    purl=dep.to_string(),
-                    extracted_requirement=dep.version,
+                    purl=dep["purl"],
+                    extracted_requirement=dep["version"],
                     scope='dependency',
                     is_runtime=True,
                     is_optional=False,
@@ -91,6 +91,8 @@ def parse(cls, location):
             )
 
         yield models.PackageData(
+            datasource_id=cls.datasource_id,
+            type=cls.default_package_type,
             name=name,
             version=version,
             vcs_url=vcs_url,
@@ -106,7 +108,8 @@ def parse(cls, location):
             parties=parties,
             dependencies=package_dependencies,
             api_data_url=api_data_url,
-            repository_homepage_url=repository_homepage_url
+            repository_homepage_url=repository_homepage_url,
+            primary_language=cls.default_primary_language
         )
 
     @classmethod
@@ -262,7 +265,10 @@ def parse_opam_from_text(text):
                 if parsed_dep:
                     version = parsed_dep.group('version').strip('{ } ').replace('"', '')
                     name = parsed_dep.group('name').strip()
-                    value.append(PackageURL(type='opam', name=name, version=version))
+                    value.append(dict(
+                        purl=PackageURL(type='opam', name=name).to_string(),
+                        version=version,
+                    ))
             opam_data[key] = value
 
         elif key == 'src':  # Get multiline src

src/packagedcode/phpcomposer.py

@@ -70,7 +70,7 @@ def parse(cls, location):
         with io.open(location, encoding='utf-8') as loc:
             package_json = json.load(loc)
 
-        yield build_package_data(cls, package_json)
+        yield build_package_data(package_json)
 
 
 def get_repository_homepage_url(namespace, name):
@@ -174,11 +174,11 @@ def parse(cls, location):
             package_data = json.load(loc)
 
         packages = [
-            build_package_data(cls, p)
+            build_package_data(p)
             for p in package_data.get('packages', [])
         ]
         packages_dev = [
-            build_package_data(cls, p)
+            build_package_data(p)
             for p in package_data.get('packages-dev', [])
         ]
 

src/packagedcode/pypi.py

@@ -355,7 +355,7 @@ def get_file_references(dist):
     """
     Yield FileReference found in a ``dist`` importlib_metadata.Distribution.
     """
-    if not dist:
+    if not dist.files:
         return
 
     for filepath in dist.files or []: