Merge pull request #42 from nexB/4-handle-moved-files

4 handle moved files

Author: steven-esser

src/deltacode/__init__.py

@@ -46,12 +46,14 @@ def __init__(self, new_path, old_path):
         self.deltas = OrderedDict([
             ('added', []),
             ('removed', []),
+            ('moved', []),
             ('modified', []),
             ('unmodified', [])
         ])
 
         if self.new.path != '' and self.old.path != '':
             self.determine_delta()
+            self.determine_moved()
 
     def align_scan(self):
         """
@@ -128,30 +130,86 @@ def determine_delta(self):
         assert new_files_visited == self.new.files_count, "Number of visited files({})) does not match total_files({}) in the new scan".format(new_files_visited, self.new.files_count)
         assert old_files_visited == self.old.files_count, "Number of visited files({})) does not match total_files({}) in the old scan".format(old_files_visited, self.old.files_count)
 
+    def determine_moved(self):
+        """
+        Modify the OrderedDict of Delta objects by creating an index of
+        'removed' Delta objects and an index of 'added' Delta objects indexed
+        by their 'sha1' attribute, identifying any unique pairs of Deltas in
+        both indices with the same 'sha1' and File 'name' attributes, and
+        converting each such pair of 'added' and 'removed' Delta objects to a
+        'moved' Delta object.
+        """
+        added = self.index_deltas('sha1', [i for i in self.deltas['added']])
+        removed = self.index_deltas('sha1', [i for i in self.deltas['removed']])
+
+        # TODO: should it be iteritems() or items()
+        for added_sha1, added_deltas in added.iteritems():
+            for removed_sha1, removed_deltas in removed.iteritems():
+
+                # check for matching sha1s on both sides
+                if utils.check_moved(added_sha1, added_deltas, removed_sha1, removed_deltas):
+                    self.update_deltas(added_deltas.pop(), removed_deltas.pop())
+
+    def update_deltas(self, added, removed):
+        """
+        Convert the matched 'added' and 'removed' Delta objects to a combined
+        'moved' Delta object and delete the 'added' and 'removed' objects.
+        """
+        self.deltas.get('moved').append(Delta(added.new_file, removed.old_file, 'moved'))
+        self.deltas.get('added').remove(added)
+        self.deltas.get('removed').remove(removed)
+
+    def index_deltas(self, index_key='path', delta_list=[]):
+        """
+        Return a dictionary of a list of Delta objects indexed by the key
+        passed via the 'index_key' variable.  If no 'index_key' variable is
+        passed, the dict is keyed by the Delta object's 'path' variable.  For a
+        'removed' Delta object, use the variable from the 'old_file'; for all
+        other Delta objects (e.g., 'added'), use the 'new_file'.  This function
+        does not currently catch the AttributeError exception.
+        """
+        index = {}
+
+        for delta in delta_list:
+            if delta.category == 'removed':
+                key = getattr(delta.old_file, index_key)
+            else:
+                key = getattr(delta.new_file, index_key)
+
+            if index.get(key) is None:
+                index[key] = []
+                index[key].append(delta)
+            else:
+                index[key].append(delta)
+
+        return index
+
     def get_stats(self):
         """
         Given a list of Delta objects, return a 'counts' dictionary keyed by
         category -- i.e., the keys of the determine_delta() OrderedDict of
         Delta objects -- that contains the count as a value for each category.
         """
-        added, modified, removed, unmodified = 0, 0, 0, 0
+        added, modified, moved, removed, unmodified = 0, 0, 0, 0, 0
 
         added = len(self.deltas['added'])
         modified = len(self.deltas['modified'])
+        moved = len(self.deltas['moved'])
         removed = len(self.deltas['removed'])
         unmodified = len(self.deltas['unmodified'])
 
-        return OrderedDict([('added', added), ('modified', modified), ('removed', removed), ('unmodified', unmodified)])
+        return OrderedDict([('added', added), ('modified', modified), ('moved', moved), ('removed', removed), ('unmodified', unmodified)])
 
     def to_dict(self):
         """
         Given an OrderedDict of Delta objects, return an OrderedDict of Delta
-        objects grouping the objects under the keys 'added', 'removed',
+        objects grouping the objects under the keys 'added', 'removed', 'moved',
         'modified' or 'unmodified'.
         """
         return OrderedDict([
             ('added', [d.to_dict() for d in self.deltas.get('added')]),
             ('removed', [d.to_dict() for d in self.deltas.get('removed')]),
+            ('moved', [d.to_dict() for d in self.deltas.get('moved')]),
             ('modified', [d.to_dict() for d in self.deltas.get('modified')]),
             ('unmodified', [d.to_dict() for d in self.deltas.get('unmodified')]),
         ])
@@ -161,7 +219,7 @@ class Delta(object):
     """
     A tuple reflecting a comparison of two files -- each of which is a File
     object -- and the category that characterizes the comparison:
-    'added', 'modified', 'removed' or 'unmodified'.
+    'added', 'modified', 'moved', 'removed' or 'unmodified'.
     """
     def __init__(self, new_file=None, old_file=None, delta_type=None):
         self.new_file = new_file if new_file else File()
@@ -219,6 +277,15 @@ def to_dict(self):
                 ('type', self.old_file.type),
                 ('size', self.old_file.size)
             ])
+        elif self.category == 'moved':
+            return OrderedDict([
+                ('category', 'moved'),
+                ('path', self.new_file.path),
+                ('old_path', self.old_file.path),
+                ('name', self.new_file.name),
+                ('type', self.new_file.type),
+                ('size', self.new_file.size)
+            ])
         elif self.category == 'modified':
             return OrderedDict([
                 ('category', 'modified'),

src/deltacode/cli.py

@@ -46,13 +46,14 @@ def write_csv(delta, result_file):
     """
     with open(result_file, 'wb') as out:
         csv_out = csv.writer(out)
-        csv_out.writerow(['Type of delta', 'Path', 'Name', 'Type', 'Size'])
+        csv_out.writerow(['Type of delta', 'Path', 'Name', 'Type', 'Size', 'Old Path'])
         for row in [(
             f.category,
             f.old_file.path if f.category == 'removed' else f.new_file.path,
             f.old_file.name if f.category == 'removed' else f.new_file.name,
             f.old_file.type if f.category == 'removed' else f.new_file.type,
-            f.old_file.size if f.category == 'removed' else f.new_file.size)
+            f.old_file.size if f.category == 'removed' else f.new_file.size,
+            f.old_file.path if f.category == 'moved' else '')
                 for d in delta.deltas for f in delta.deltas.get(d)]:
                     csv_out.writerow(row)
 

src/deltacode/utils.py

@@ -101,3 +101,16 @@ def fix_trees(a_files, b_files):
     for b_file in b_files:
         b_file.original_path = b_file.path
         b_file.path = '/'.join(paths.split(b_file.path)[b_offset:])
+
+
+def check_moved(added_sha1, added_deltas, removed_sha1, removed_deltas):
+    """
+    Return True if there is only one pair of matching 'added' and 'removed'
+    Delta objects and their respective File objects have the same 'name' attribute.
+    """
+    if added_sha1 != removed_sha1:
+        return False
+    if len(added_deltas) != 1 or len(removed_deltas) != 1:
+        return False
+    if added_deltas[0].new_file.name == removed_deltas[0].old_file.name:
+        return True

tests/data/cli/1_file_moved.csv

@@ -0,0 +1,9 @@
+Type of delta,Path,Name,Type,Size,Old Path
+moved,b/a4.py,a4.py,file,200,a/a4.py
+unmodified,a/a3.py,a3.py,file,200,
+unmodified,b/b4.py,b4.py,file,200,
+unmodified,a/a2.py,a2.py,file,200,
+unmodified,b/b2.py,b2.py,file,200,
+unmodified,b/b1.py,b1.py,file,200,
+unmodified,b/b3.py,b3.py,file,200,
+unmodified,a/a1.py,a1.py,file,200,

tests/data/cli/1_file_moved_and_1_copy.csv

@@ -0,0 +1,11 @@
+Type of delta,Path,Name,Type,Size,Old Path
+added,b/a4.py,a4.py,file,200,
+added,b/a4_copy.py,a4_copy.py,file,200,
+removed,a/a4.py,a4.py,file,200,
+unmodified,a/a3.py,a3.py,file,200,
+unmodified,b/b4.py,b4.py,file,200,
+unmodified,a/a2.py,a2.py,file,200,
+unmodified,b/b2.py,b2.py,file,200,
+unmodified,b/b1.py,b1.py,file,200,
+unmodified,b/b3.py,b3.py,file,200,
+unmodified,a/a1.py,a1.py,file,200,

tests/data/cli/1_file_moved_and_added.csv

@@ -0,0 +1,11 @@
+Type of delta,Path,Name,Type,Size,Old Path
+added,b/a4.py,a4.py,file,200,
+added,c/a4.py,a4.py,file,200,
+removed,a/a4.py,a4.py,file,200,
+unmodified,a/a3.py,a3.py,file,200,
+unmodified,b/b4.py,b4.py,file,200,
+unmodified,a/a2.py,a2.py,file,200,
+unmodified,b/b2.py,b2.py,file,200,
+unmodified,b/b1.py,b1.py,file,200,
+unmodified,b/b3.py,b3.py,file,200,
+unmodified,a/a1.py,a1.py,file,200,

tests/data/cli/added1.csv

@@ -1,10 +1,10 @@
-Type of delta,Path,Name,Type,Size
-added,a/a5.py,a5.py,file,200
-unmodified,a/a3.py,a3.py,file,200
-unmodified,b/b4.py,b4.py,file,200
-unmodified,a/a2.py,a2.py,file,200
-unmodified,b/b2.py,b2.py,file,200
-unmodified,b/b1.py,b1.py,file,200
-unmodified,b/b3.py,b3.py,file,200
-unmodified,a/a4.py,a4.py,file,200
-unmodified,a/a1.py,a1.py,file,200
+Type of delta,Path,Name,Type,Size,Old Path
+added,a/a5.py,a5.py,file,200,
+unmodified,a/a3.py,a3.py,file,200,
+unmodified,b/b4.py,b4.py,file,200,
+unmodified,a/a2.py,a2.py,file,200,
+unmodified,b/b2.py,b2.py,file,200,
+unmodified,b/b1.py,b1.py,file,200,
+unmodified,b/b3.py,b3.py,file,200,
+unmodified,a/a4.py,a4.py,file,200,
+unmodified,a/a1.py,a1.py,file,200,

tests/data/cli/license_info_added.csv

@@ -1,3 +1,3 @@
-Type of delta,Path,Name,Type,Size
-license info added,some/path/a/a1.py,a1.py,file,350
-unmodified,some/path/b/b1.py,b1.py,file,290
+Type of delta,Path,Name,Type,Size,Old Path
+license info added,some/path/a/a1.py,a1.py,file,350,
+unmodified,some/path/b/b1.py,b1.py,file,290,

tests/data/cli/license_info_added_below_cutoff_score.csv

@@ -1,3 +1,3 @@
-Type of delta,Path,Name,Type,Size
-license info added,some/path/a/a1.py,a1.py,file,350
-unmodified,some/path/b/b1.py,b1.py,file,290
+Type of delta,Path,Name,Type,Size,Old Path
+license info added,some/path/a/a1.py,a1.py,file,350,
+unmodified,some/path/b/b1.py,b1.py,file,290,

tests/data/cli/license_info_removed.csv

@@ -1,3 +1,3 @@
-Type of delta,Path,Name,Type,Size
-license info removed,some/path/a/a1.py,a1.py,file,350
-unmodified,some/path/b/b1.py,b1.py,file,290
+Type of delta,Path,Name,Type,Size,Old Path
+license info removed,some/path/a/a1.py,a1.py,file,350,
+unmodified,some/path/b/b1.py,b1.py,file,290,