Virtual Codebase Integration

Signed-off-by: Pratik Dey <pratikrocks.dey11@gmail.com>

Author: pratik0316

setup.cfg

@@ -31,7 +31,7 @@ include_package_data = true
 zip_safe = false
 install_requires =
     bitarray==1.1.0
-    commoncode>=21.5.12
+    commoncode>=21.6.11
     click
     simplejson
     unicodecsv

src/deltacode/__init__.py

@@ -28,12 +28,12 @@
 import os
 from collections import OrderedDict
 
-from deltacode.models import File
-from deltacode.models import Scan
 from deltacode import utils
+from deltacode.exceptions import FileError as FileError
 from commoncode import paths
 from commoncode.resource import VirtualCodebase
 
+
 from pkg_resources import get_distribution, DistributionNotFound
 
 try:
@@ -55,38 +55,35 @@ class DeltaCode(object):
     def __init__(self, new_path, old_path, options):
         self.codebase1 = None
         self.codebase2 = None
-        self.new_files_fingerprint = (
-            dict()
-        )  # map of { {new_file1:fingerprint},{new_file2:fingerprint},...} it will be needed when we need the fingerprints
-        self.old_files_fingerprint = (
-            dict()
-        )  # map of { {old_file1:fingerprint},{old_file2:fingerprint},...}
         self.options = options
         self.deltas = []
         self.errors = []
 
-        try:
+        if os.path.isfile(new_path) and os.path.isfile(old_path):
             self.codebase1 = VirtualCodebase(new_path)
             self.codebase2 = VirtualCodebase(old_path)
 
-        except Exception as exception:
-            self.errors.append(str(exception))
-
-        if self.codebase1 is not None or self.codebase2 is not None:
-            self.stats = Stat(
-                self.codebase1.compute_counts(), self.codebase2.compute_counts()
+        else:
+            error_message = (
+                "{} is expected to be a file".format(new_path)
+                if not os.path.isfile(new_path)
+                else "{} is expected to be a file".format(old_path)
             )
-            self.new_files_errors = []
-            self.old_files_errors = []
-            self.determine_delta()
-            self.license_diff()
-            self.copyright_diff()
-            self.stats.calculate_stats()
-            self.similarity()
-            # Sort deltas by score, descending, i.e., high > low, and then by
-            # factors, alphabetically.  Run the least significant sort first.
-            self.deltas.sort(key=lambda Delta: Delta.factors, reverse=False)
-            self.deltas.sort(key=lambda Delta: Delta.score, reverse=True)
+            raise FileError(error_message)
+        self.stats = Stat(
+            self.codebase1.compute_counts(), self.codebase2.compute_counts()
+        )
+        self.new_files_errors = []
+        self.old_files_errors = []
+        self.determine_delta()
+        self.license_diff()
+        self.copyright_diff()
+        self.stats.calculate_stats()
+        self.similarity()
+        # Sort deltas by score, descending, i.e., high > low, and then by
+        # factors, alphabetically.  Run the least significant sort first.
+        self.deltas.sort(key=lambda Delta: Delta.factors, reverse=False)
+        self.deltas.sort(key=lambda Delta: Delta.score, reverse=True)
 
     def similarity(self):
         """
@@ -99,17 +96,21 @@ def similarity(self):
         for delta in self.deltas:
             if delta.new_file == None or delta.old_file == None:
                 continue
-            new_fingerprint = self.new_files_fingerprint.get(delta.new_file.path, None)
-            old_fingerprint = self.old_files_fingerprint.get(delta.old_file.path, None)
+            new_fingerprint = (
+                delta.new_file.fingerprint
+                if hasattr(delta.new_file, "fingerprint")
+                else None
+            )
+            old_fingerprint = (
+                delta.old_file.fingerprint
+                if hasattr(delta.old_file, "fingerprint")
+                else None
+            )
 
             if new_fingerprint == None or old_fingerprint == None:
                 continue
-            new_fingerprint = utils.bitarray_from_hex(
-                self.new_files_fingerprint[delta.new_file.path]
-            )
-            old_fingerprint = utils.bitarray_from_hex(
-                self.old_files_fingerprint[delta.old_file.path]
-            )
+            new_fingerprint = utils.bitarray_from_hex(delta.new_file.fingerprint)
+            old_fingerprint = utils.bitarray_from_hex(delta.old_file.fingerprint)
 
             hamming_distance = utils.hamming_distance(new_fingerprint, old_fingerprint)
             if hamming_distance > 0 and hamming_distance <= SIMILARITY_LIMIT:
@@ -119,13 +120,8 @@ def similarity(self):
                 )
 
     def create_deltas(
-        self, new_resource, old_resource, new_path, old_path, score, count, status
+        self, new_resource, old_resource, new_path, old_path, score, status
     ):
-        count = count + 1
-        if new_resource:
-            new_resource.path = new_path
-        if old_resource:
-            old_resource.path = old_path
         delta = Delta(score, new_resource, old_resource)
         delta.status = status
         self.deltas.append(delta)
@@ -137,105 +133,107 @@ def determine_delta(self):
         from either scan.
         """
 
-        old_files_sha1_considered_in_deltas = dict()
-
-        Delta.NEW_CODEBASE_OFFSET, Delta.OLD_CODEBASE_OFFSET = utils.align_trees(
-            self.codebase1, self.codebase2
-        )
+        old_resource_considered = dict()
+        try:
+            Delta.NEW_CODEBASE_OFFSET, Delta.OLD_CODEBASE_OFFSET = utils.align_trees(
+                self.codebase1, self.codebase2
+            )
+        except utils.AlignmentException:
+            Delta.NEW_CODEBASE_OFFSET, Delta.OLD_CODEBASE_OFFSET = 0, 0
 
         for new_resource in self.codebase1.walk():
             if new_resource.is_file:
                 path_new = "/".join(
                     paths.split(new_resource.path)[Delta.NEW_CODEBASE_OFFSET :]
                 )
+
+                old_resource = self.codebase2.get_resource_from_path(path_new)
+
+                if old_resource and old_resource.sha1 == new_resource.sha1:
+                    old_resource_considered[old_resource.path] = 1
+                    path_old = "/".join(
+                        paths.split(old_resource.path)[Delta.OLD_CODEBASE_OFFSET :]
+                    )
+                    self.create_deltas(
+                        new_resource, old_resource, path_new, path_old, 0, "unmodified",
+                    )
+                    self.stats.num_unmodified += 1
+
+                    continue
+
                 ADDED = True
                 for old_resource in self.codebase2.walk():
+                    if old_resource.path in old_resource_considered.keys():
+                        continue
+                    path_old = "/".join(
+                        paths.split(old_resource.path)[Delta.OLD_CODEBASE_OFFSET :]
+                    )
                     if (
                         old_resource.is_file
-                        and not old_resource.sha1
-                        in old_files_sha1_considered_in_deltas.keys()
+                        and not old_resource.path in old_resource_considered.keys()
                     ):
-                        path_old = "/".join(
-                            paths.split(old_resource.path)[Delta.OLD_CODEBASE_OFFSET :]
-                        )
+
                         if path_new == path_old:
                             ADDED = False
                             if new_resource.sha1 == old_resource.sha1:
-                                old_files_sha1_considered_in_deltas[
-                                    old_resource.sha1
-                                ] = 1
+                                old_resource_considered[old_resource.path] = 1
                                 self.create_deltas(
                                     new_resource,
                                     old_resource,
                                     path_new,
                                     path_old,
                                     0,
-                                    self.stats.num_unmodified,
                                     "unmodified",
                                 )
+                                self.stats.num_unmodified += 1
                                 break
                             else:
-                                old_files_sha1_considered_in_deltas[
-                                    old_resource.sha1
-                                ] = 1
+                                old_resource_considered[old_resource.path] = 1
                                 self.create_deltas(
                                     new_resource,
                                     old_resource,
                                     path_new,
                                     path_old,
                                     20,
-                                    self.stats.num_modified,
                                     "modified",
                                 )
+                                self.stats.num_modified += 1
                                 break
                         else:
                             if new_resource.sha1 == old_resource.sha1:
-                                old_files_sha1_considered_in_deltas[
-                                    old_resource.sha1
-                                ] = 1
+                                old_resource_considered[old_resource.path] = 1
                                 ADDED = False
                                 self.create_deltas(
                                     new_resource,
                                     old_resource,
                                     path_new,
                                     path_old,
                                     0,
-                                    self.stats.num_moved,
                                     "moved",
                                 )
+                                self.stats.num_moved += 1
                                 break
 
                 if ADDED:
                     self.create_deltas(
-                        new_resource,
-                        None,
-                        path_new,
-                        None,
-                        100,
-                        self.stats.num_added,
-                        "added",
+                        new_resource, None, path_new, None, 100, "added",
                     )
+                    self.stats.num_added += 1
 
         for old_resource_remaining in self.codebase2.walk():
             if (
                 old_resource_remaining.is_file
-                and old_resource_remaining.sha1
-                not in old_files_sha1_considered_in_deltas.keys()
+                and old_resource_remaining.path not in old_resource_considered.keys()
             ):
                 path_old = "/".join(
                     paths.split(old_resource_remaining.path)[
                         Delta.OLD_CODEBASE_OFFSET :
                     ]
                 )
                 self.create_deltas(
-                    None,
-                    old_resource_remaining,
-                    None,
-                    path_old,
-                    0,
-                    self.stats.num_removed,
-                    "removed",
+                    None, old_resource_remaining, None, path_old, 0, "removed",
                 )
+                self.stats.num_removed += 1
 
     def license_diff(self):
         """
@@ -376,19 +374,23 @@ def licenses_to_dict(self, file):
             return []
 
     def file_to_dict(self, deltacode, file, new_file=True):
+
         path_offset = (
             Delta.NEW_CODEBASE_OFFSET if new_file else Delta.OLD_CODEBASE_OFFSET
         )
         if file:
             return OrderedDict(
                 [
-                    ("path", file.path),
+                    ("path", "/".join(paths.split(file.path)[path_offset:])),
                     ("type", file.type),
                     ("name", file.name),
                     ("size", file.size),
                     ("sha1", file.sha1),
-                    ("fingerprint", deltacode.old_files_fingerprint.get(file.path, "")),
-                    ("original_path", "/".join(paths.split(file.path)[path_offset:])),
+                    (
+                        "fingerprint",
+                        file.fingerprint if hasattr(file, "fingerprint") else "",
+                    ),
+                    ("original_path", file.path),
                     ("licenses", self.licenses_to_dict(file)),
                     ("copyrights", self.copyrights_to_dict(file)),
                 ]
@@ -399,6 +401,11 @@ def to_dict(self, deltacode):
         Return an OrderedDict comprising the 'factors', 'score' and new and old
         'path' attributes of the object.
         """
+        if (
+            not deltacode.options.get("--all-delta-types", "") == True
+            and self.status == "unmodified"
+        ):
+            return
         if self.new_file:
             new_file = self.new_file.to_dict()
         else:

src/deltacode/exceptions.py

@@ -0,0 +1,35 @@
+#
+# Copyright (c) 2017-2018 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/deltacode/
+# The DeltaCode software is licensed under the Apache License version 2.0.
+# Data generated with DeltaCode require an acknowledgment.
+# DeltaCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with DeltaCode or any DeltaCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with DeltaCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  DeltaCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  DeltaCode is a free and open source software analysis tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/deltacode/ for support and download.
+#
+
+
+class FileError(Exception):
+    def __init__(self, *args):
+        if args:
+            self.message = args[0]
+        else:
+            self.message = None
+
+    def __str__(self):
+        return self.message

src/deltacode/test_utils.py

@@ -34,6 +34,7 @@
 import json
 
 from commoncode.system import on_windows
+from commoncode import paths
 from commoncode.resource import VirtualCodebase
 
 def run_scan_click(options, monkeypatch=None, test_mode=True, expected_rc=0, env=None):
@@ -149,6 +150,10 @@ def streamline_errors(errors):
         errors[i] = cleaned_error
 
 
+def get_aligned_path(delta, path, new_file):
+    OFFSET = delta.NEW_CODEBASE_OFFSET if new_file else delta.OLD_CODEBASE_OFFSET
+    return "/".join(paths.split(path)[OFFSET:])
+
 def streamline_headers(headers):
     """
     Modify the `headers` list of mappings in place to make it easier to test.

src/deltacode/utils.py

@@ -55,10 +55,12 @@ def update_added_from_license_info(delta, unique_categories):
     one or more categories to its 'factors' attribute if there has
     been a license change.
     """
-    new_licenses = delta.new_file.licenses or []
+    new_licenses = (
+        delta.new_file.licenses if hasattr(delta.new_file, "licenses") else []
+    )
 
     new_categories = set(license["category"] for license in new_licenses)
-    if delta.new_file.licenses:
+    if hasattr(delta.new_file, "licenses"):
         delta.update(20, "license info added")
         for category in new_categories:
             # no license ==> 'Copyleft Limited'or higher
@@ -200,7 +202,7 @@ def deltas(deltacode, all_delta_types=False):
     for delta in deltacode.deltas:
         if all_delta_types is True:
             yield delta.to_dict(deltacode)
-        elif not delta.is_unmodified():
+        elif not delta.status == "unmodified":
             yield delta.to_dict(deltacode)