Merge pull request #40 from nexB/fix-json-output-memory-usage

Fix json output memory usage

Author: steven-esser

src/deltacode/cli.py

@@ -31,12 +31,15 @@
 import json
 
 import click
+import simplejson
 
 from deltacode import DeltaCode
 from deltacode import __version__
+from deltacode.utils import deltas
 
 
-def generate_csv(delta, result_file):
+# FIXME: update the function argument delta to deltacode
+def write_csv(delta, result_file):
     """
     Using the DeltaCode object, create a .csv file
     containing the primary information from the Delta objects.
@@ -54,27 +57,28 @@ def generate_csv(delta, result_file):
                     csv_out.writerow(row)
 
 
-def generate_json(delta, result_file):
+def write_json(deltacode, outfile):
     """
     Using the DeltaCode object, create a .json file
     containing the primary information from the Delta objects.
     """
-    output = OrderedDict([
+    results = OrderedDict([
         ('deltacode_version', __version__),
-        ('deltacode_stats', delta.get_stats()),
-        ('deltas', delta.to_dict())
+        ('deltacode_stats', deltacode.get_stats()),
+        ('deltas', deltas(deltacode)),
     ])
 
-    with open(result_file, 'w') as outfile:
-        json.dump(output, outfile, indent=4)
+    # TODO: add toggle for pretty printing
+    simplejson.dump(results, outfile, iterable_as_array=True, indent=2)
+    outfile.write('\n')
 
 
 @click.command()
 @click.help_option('-h', '--help')
 @click.option('-n', '--new', required=True, prompt=False, type=click.Path(exists=True, readable=True), help='Identify the path to the "new" scan file')
 @click.option('-o', '--old', required=True, prompt=False, type=click.Path(exists=True, readable=True), help='Identify the path to the "old" scan file')
 @click.option('-c', '--csv-file', prompt=False, type=click.Path(exists=False), help='Identify the path to the .csv output file')
-@click.option('-j', '--json-file', prompt=False, type=click.Path(exists=False), help='Identify the path to the .json output file')
+@click.option('-j', '--json-file', prompt=False, default='-', type=click.File(mode='wb', lazy=False), help='Identify the path to the .json output file')
 def cli(new, old, csv_file, json_file):
     """
     Identify the changes that need to be made to the 'old'
@@ -84,14 +88,11 @@ def cli(new, old, csv_file, json_file):
     option is selected, print the JSON results to the console.
     """
     # do the delta
-    delta = DeltaCode(new, old)
+    deltacode = DeltaCode(new, old)
 
     # output to csv
     if csv_file:
-        generate_csv(delta, csv_file)
+        write_csv(deltacode, csv_file)
     # generate JSON output
-    elif json_file:
-        generate_json(delta, json_file)
-    # print to stdout
     else:
-        print(json.dumps(delta.to_dict()))
+        write_json(deltacode, json_file)

src/deltacode/utils.py

@@ -31,6 +31,15 @@
 from commoncode import paths
 
 
+def deltas(deltacode):
+    """
+    Return a generator of Delta dictionaries for JSON serialized ouput.
+    """
+    for category, deltas in deltacode.deltas.iteritems():
+        for delta in deltas:
+            yield delta.to_dict()
+
+
 class AlignmentException(Exception):
     """
     Named exception for alignment errors.

tests/test_cli.py

@@ -75,102 +75,102 @@ class TestCLI(FileBasedTesting):
 
     test_data_dir = os.path.join(os.path.dirname(__file__), 'data')
 
-    def test_generate_csv_added(self):
+    def test_write_csv_added(self):
         new_scan = self.get_test_loc('cli/new_added1.json')
         old_scan = self.get_test_loc('cli/old_added1.json')
 
         delta = DeltaCode(new_scan, old_scan)
         result_file = self.get_temp_file('.csv')
-        cli.generate_csv(delta, result_file)
+        cli.write_csv(delta, result_file)
         expected_file = self.get_test_loc('cli/added1.csv')
         check_csvs(result_file, expected_file)
 
-    def test_generate_csv_modified(self):
+    def test_write_csv_modified(self):
         new_scan = self.get_test_loc('cli/new_modified1.json')
         old_scan = self.get_test_loc('cli/old_modified1.json')
 
         delta = DeltaCode(new_scan, old_scan)
         result_file = self.get_temp_file('.csv')
-        cli.generate_csv(delta, result_file)
+        cli.write_csv(delta, result_file)
         expected_file = self.get_test_loc('cli/modified1.csv')
         check_csvs(result_file, expected_file)
 
-    def test_generate_csv_removed(self):
+    def test_write_csv_removed(self):
         new_scan = self.get_test_loc('cli/new_removed1.json')
         old_scan = self.get_test_loc('cli/old_removed1.json')
 
         delta = DeltaCode(new_scan, old_scan)
         result_file = self.get_temp_file('.csv')
-        cli.generate_csv(delta, result_file)
+        cli.write_csv(delta, result_file)
         expected_file = self.get_test_loc('cli/removed1.csv')
         check_csvs(result_file, expected_file)
 
-    def test_generate_csv_renamed(self):
+    def test_write_csv_renamed(self):
         new_scan = self.get_test_loc('cli/new_renamed1.json')
         old_scan = self.get_test_loc('cli/old_renamed1.json')
 
         delta = DeltaCode(new_scan, old_scan)
         result_file = self.get_temp_file('.csv')
-        cli.generate_csv(delta, result_file)
+        cli.write_csv(delta, result_file)
         expected_file = self.get_test_loc('cli/renamed1.csv')
         check_csvs(result_file, expected_file)
 
-    def test_generate_csv_modified_new_license_added(self):
+    def test_write_csv_modified_new_license_added(self):
         new_scan = self.get_test_loc('cli/scan_modified_new_license_added.json')
         old_scan = self.get_test_loc('cli/scan_modified_old_license_added.json')
 
         delta = DeltaCode(new_scan, old_scan)
         result_file = self.get_temp_file('.csv')
-        cli.generate_csv(delta, result_file)
+        cli.write_csv(delta, result_file)
         expected_file = self.get_test_loc('cli/modified_new_license_added.csv')
         check_csvs(result_file, expected_file)
 
-    def test_generate_csv_modified_new_license_added_low_score(self):
+    def test_write_csv_modified_new_license_added_low_score(self):
         new_scan = self.get_test_loc('cli/scan_modified_new_license_added_low_score.json')
         old_scan = self.get_test_loc('cli/scan_modified_old_license_added_low_score.json')
 
         delta = DeltaCode(new_scan, old_scan)
         result_file = self.get_temp_file('.csv')
-        cli.generate_csv(delta, result_file)
+        cli.write_csv(delta, result_file)
         expected_file = self.get_test_loc('cli/modified_new_license_added_low_score.csv')
         check_csvs(result_file, expected_file)
 
-    def test_generate_csv_license_info_removed(self):
+    def test_write_csv_license_info_removed(self):
         new_scan = self.get_test_loc('cli/scan_new_license_info_removed.json')
         old_scan = self.get_test_loc('cli/scan_old_license_info_removed.json')
 
         delta = DeltaCode(new_scan, old_scan)
         result_file = self.get_temp_file('.csv')
-        cli.generate_csv(delta, result_file)
+        cli.write_csv(delta, result_file)
         expected_file = self.get_test_loc('cli/license_info_removed.csv')
         check_csvs(result_file, expected_file)
 
-    def test_generate_csv_license_info_added(self):
+    def test_write_csv_license_info_added(self):
         new_scan = self.get_test_loc('cli/scan_new_license_info_added.json')
         old_scan = self.get_test_loc('cli/scan_old_license_info_added.json')
 
         delta = DeltaCode(new_scan, old_scan)
         result_file = self.get_temp_file('.csv')
-        cli.generate_csv(delta, result_file)
+        cli.write_csv(delta, result_file)
         expected_file = self.get_test_loc('cli/license_info_added.csv')
         check_csvs(result_file, expected_file)
 
-    def test_generate_csv_license_info_removed_below_cutoff_score(self):
+    def test_write_csv_license_info_removed_below_cutoff_score(self):
         new_scan = self.get_test_loc('cli/scan_new_license_info_removed_below_cutoff_score.json')
         old_scan = self.get_test_loc('cli/scan_old_license_info_removed_below_cutoff_score.json')
 
         delta = DeltaCode(new_scan, old_scan)
         result_file = self.get_temp_file('.csv')
-        cli.generate_csv(delta, result_file)
+        cli.write_csv(delta, result_file)
         expected_file = self.get_test_loc('cli/license_info_removed_below_cutoff_score.csv')
         check_csvs(result_file, expected_file)
 
-    def test_generate_csv_license_info_added_below_cutoff_score(self):
+    def test_write_csv_license_info_added_below_cutoff_score(self):
         new_scan = self.get_test_loc('cli/scan_new_license_info_added_below_cutoff_score.json')
         old_scan = self.get_test_loc('cli/scan_old_license_info_added_below_cutoff_score.json')
 
         delta = DeltaCode(new_scan, old_scan)
         result_file = self.get_temp_file('.csv')
-        cli.generate_csv(delta, result_file)
+        cli.write_csv(delta, result_file)
         expected_file = self.get_test_loc('cli/license_info_added_below_cutoff_score.csv')
         check_csvs(result_file, expected_file)