Add support for downloading http and ftp urls

README.rst

@@ -0,0 +1,33 @@
+*****
+Fetchcode
+*****
+It is a library to reliably fetch code via HTTP, FTP and version control systems. 
+
+Installation
+############
+Clone the repo using 
+`git clone https://github.com/nexB/fetchcode`
+
+Then install all the requirements using 
+`pip3 install -r requirements.txt`
+
+Running test suite
+#################
+
+To run test suite
+`python3 -m pytest`
+
+Usage of API to fetch HTTP/S and FTP URLs
+#########################################
+```
+from fetchcode import fetch
+url = 'A Http or FTP URL'
+location = 'Location of file'
+# This returns a response object which has attributes
+# 'content_type' content type of the file
+# 'location' the absolute location of the files that was fetched
+# 'scheme' scheme of the URL
+# 'size' size of the retrieved content in bytes
+# 'url' fetched URL
+resp = fetch(url = url)
+```

fetchcode/__init__.py

@@ -0,0 +1,107 @@
+# fetchcode is a free software tool from nexB Inc. and others.
+# Visit https://github.com/nexB/fetchcode for support and download.
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# http://nexb.com and http://aboutcode.org
+#
+# This software is licensed under the Apache License version 2.0.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at:
+# http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+
+from ftplib import FTP
+from mimetypes import MimeTypes
+import os
+import tempfile
+from urllib.parse import urlparse
+
+import requests
+
+
+class Response:
+    def __init__(self, location, content_type, size, url):
+        """
+        Represent the response from fetching a URL with:
+        - `location`: the absolute location of the files that was fetched
+        - `content_type`: content type of the file
+        - `size`: size of the retrieved content in bytes
+        - `url`: fetched URL
+        """
+        self.url = url
+        self.size = size
+        self.content_type = content_type
+        self.location = location
+
+
+def fetch_http(url, location):
+    """
+    Return a `Response` object built from fetching the content at a HTTP/HTTPS based `url` URL string
+    saving the content in a file at `location`   
+    """
+    r = requests.get(url)
+    with open(location, 'wb') as f:
+        f.write(r.content)
+
+    content_type = r.headers.get('content-type')
+    size = r.headers.get('content-length')
+    size = int(size) if size else None
+
+    resp = Response(location=location, content_type=content_type, size=size, url=url)
+
+    return resp
+
+
+def fetch_ftp(url, location):
+    """
+    Return a `Response` object built from fetching the content at a FTP based `url` URL string
+    saving the content in a file at `location`  
+    """
+    url_parts = urlparse(url)
+
+    netloc = url_parts.netloc
+    path = url_parts.path
+    dir, file = os.path.split(path)
+
+    ftp = FTP(netloc)
+    ftp.login()
+
+    size = ftp.size(path)
+    mime = MimeTypes()
+    mime_type = mime.guess_type(file)
+    if mime_type:
+        content_type = mime_type[0]
+    else:
+        content_type = None
+
+    ftp.cwd(dir)
+    file = 'RETR {}'.format(file)
+    with open(location, 'wb') as f:
+        ftp.retrbinary(file, f.write)
+    ftp.close()
+
+    resp = Response(location=location, content_type=content_type, size=size, url=url)
+    return resp
+
+
+def fetch(url):
+    """
+    Return a `Response` object built from fetching the content at the `url` URL string and store content at a temporary file.
+    """
+
+    temp = tempfile.NamedTemporaryFile(delete=False)
+    location = temp.name
+
+    url_parts = urlparse(url)
+    scheme = url_parts.scheme
+
+    fetchers = {'ftp': fetch_ftp, 'http': fetch_http, 'https': fetch_http}
+
+    if scheme in fetchers:
+        return fetchers.get(scheme)(url, location)
+
+    raise Exception('Not a supported/known scheme.')

tests/test_fetch.py

@@ -0,0 +1,65 @@
+# fetchcode is a free software tool from nexB Inc. and others.
+# Visit https://github.com/nexB/fetchcode for support and download.
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# http://nexb.com and http://aboutcode.org
+#
+# This software is licensed under the Apache License version 2.0.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at:
+# http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+
+from unittest import mock
+
+import pytest
+
+from fetchcode import fetch
+
+
+@mock.patch('fetchcode.requests.get')
+def test_fetch_http_with_tempfile(mock_get):
+    mock_get.return_value.headers = {
+        'content-type': 'image/png',
+        'content-length': '1000999',
+    }
+
+    with mock.patch('fetchcode.open', mock.mock_open()) as mocked_file:
+        url = 'https://raw.githubusercontent.com/TG1999/converge/master/assets/Group%2022.png'
+        response = fetch(url=url)
+        assert response is not None
+        assert 1000999 == response.size
+        assert url == response.url
+        assert 'image/png' == response.content_type
+
+
+@mock.patch('fetchcode.FTP')
+def test_fetch_with_wrong_url(mock_get):
+    with pytest.raises(Exception) as e_info:
+        url = 'ftp://speedtest/1KB.zip'
+        response = fetch(url=url)
+        assert 'Not a valid URL' == e_info
+
+
+@mock.patch('fetchcode.FTP', autospec=True)
+def test_fetch_ftp_with_tempfile(mock_ftp_constructor):
+    mock_ftp = mock_ftp_constructor.return_value
+    mock_ftp_constructor.return_value.size.return_value = 1024
+    with mock.patch('fetchcode.open', mock.mock_open()) as mocked_file:
+        response = fetch('ftp://speedtest.tele2.net/1KB.zip')
+        assert 1024 == response.size
+        mock_ftp_constructor.assert_called_with('speedtest.tele2.net')
+        assert mock_ftp.login.called == True
+        mock_ftp.cwd.assert_called_with('/')
+        assert mock_ftp.retrbinary.called
+
+
+def test_fetch_with_scheme_not_present():
+    with pytest.raises(Exception) as e_info:
+        url = 'abc://speedtest/1KB.zip'
+        response = fetch(url=url)
+        assert 'Not a supported/known scheme.' == e_info