diff --git a/README.md b/README.md deleted file mode 100644 index 089e8088..00000000 --- a/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# fetchcode -A library to reliably fetch code via HTTP, FTP and version control systems. - -# Installation -Clone the repo using -`git clone https://github.com/nexB/fetchcode` - -Then install all the requirements using -`pip3 install -r requirements.txt` - -# Running Test Suite - -To run test suite: -`python3 -m pytest` - diff --git a/README.rst b/README.rst new file mode 100644 index 00000000..3e9565bd --- /dev/null +++ b/README.rst @@ -0,0 +1,33 @@ +***** +Fetchcode +***** +It is a library to reliably fetch code via HTTP, FTP and version control systems. + +Installation +############ +Clone the repo using +`git clone https://github.com/nexB/fetchcode` + +Then install all the requirements using +`pip3 install -r requirements.txt` + +Running test suite +################# + +To run test suite +`python3 -m pytest` + +Usage of API to fetch HTTP/S and FTP URLs +######################################### +``` +from fetchcode import fetch +url = 'A Http or FTP URL' +location = 'Location of file' +# This returns a response object which has attributes +# 'content_type' content type of the file +# 'location' the absolute location of the files that was fetched +# 'scheme' scheme of the URL +# 'size' size of the retrieved content in bytes +# 'url' fetched URL +resp = fetch(url = url) +``` diff --git a/fetchcode/__init__.py b/fetchcode/__init__.py new file mode 100644 index 00000000..c573991f --- /dev/null +++ b/fetchcode/__init__.py @@ -0,0 +1,107 @@ +# fetchcode is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/fetchcode for support and download. +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and http://aboutcode.org +# +# This software is licensed under the Apache License version 2.0. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: +# http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +from ftplib import FTP +from mimetypes import MimeTypes +import os +import tempfile +from urllib.parse import urlparse + +import requests + + +class Response: + def __init__(self, location, content_type, size, url): + """ + Represent the response from fetching a URL with: + - `location`: the absolute location of the files that was fetched + - `content_type`: content type of the file + - `size`: size of the retrieved content in bytes + - `url`: fetched URL + """ + self.url = url + self.size = size + self.content_type = content_type + self.location = location + + +def fetch_http(url, location): + """ + Return a `Response` object built from fetching the content at a HTTP/HTTPS based `url` URL string + saving the content in a file at `location` + """ + r = requests.get(url) + with open(location, 'wb') as f: + f.write(r.content) + + content_type = r.headers.get('content-type') + size = r.headers.get('content-length') + size = int(size) if size else None + + resp = Response(location=location, content_type=content_type, size=size, url=url) + + return resp + + +def fetch_ftp(url, location): + """ + Return a `Response` object built from fetching the content at a FTP based `url` URL string + saving the content in a file at `location` + """ + url_parts = urlparse(url) + + netloc = url_parts.netloc + path = url_parts.path + dir, file = os.path.split(path) + + ftp = FTP(netloc) + ftp.login() + + size = ftp.size(path) + mime = MimeTypes() + mime_type = mime.guess_type(file) + if mime_type: + content_type = mime_type[0] + else: + content_type = None + + ftp.cwd(dir) + file = 'RETR {}'.format(file) + with open(location, 'wb') as f: + ftp.retrbinary(file, f.write) + ftp.close() + + resp = Response(location=location, content_type=content_type, size=size, url=url) + return resp + + +def fetch(url): + """ + Return a `Response` object built from fetching the content at the `url` URL string and store content at a temporary file. + """ + + temp = tempfile.NamedTemporaryFile(delete=False) + location = temp.name + + url_parts = urlparse(url) + scheme = url_parts.scheme + + fetchers = {'ftp': fetch_ftp, 'http': fetch_http, 'https': fetch_http} + + if scheme in fetchers: + return fetchers.get(scheme)(url, location) + + raise Exception('Not a supported/known scheme.') diff --git a/fetchcode/api.py b/fetchcode/api.py deleted file mode 100644 index dae3d827..00000000 --- a/fetchcode/api.py +++ /dev/null @@ -1,54 +0,0 @@ -# fetchcode is a free software tool from nexB Inc. and others. -# Visit https://github.com/nexB/fetchcode for support and download. -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# http://nexb.com and http://aboutcode.org -# -# This software is licensed under the Apache License version 2.0. -# -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: -# http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. - -import tempfile - -import requests - - -class Response: - """ - Represent the response from fetching a URL with: -- `location`: the absolute location of the files that was fetched -- `content_type`: content type of the file -- `size`: size of the retrieved content in bytes -- `url`: fetched URL - """ - def __init__(self, location, content_type, size, url): - self.location = location - self.content_type = content_type - self.size = size - self.url = url - - -def fetch(url): - """ - Return a `Response` object built from fetching the content at the `url` URL string. - """ - r = requests.get(url) - - temp = tempfile.NamedTemporaryFile(delete=False) - filename = temp.name - - with open(filename,'wb') as f: - f.write(r.content) - - resp = Response(location=filename, - content_type=r.headers['content-type'], - size=int(r.headers['content-length']), - url=url) - - return resp diff --git a/tests/test_api.py b/tests/test_api.py deleted file mode 100644 index eb9f75ca..00000000 --- a/tests/test_api.py +++ /dev/null @@ -1,32 +0,0 @@ -# fetchcode is a free software tool from nexB Inc. and others. -# Visit https://github.com/nexB/fetchcode for support and download. -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# http://nexb.com and http://aboutcode.org -# -# This software is licensed under the Apache License version 2.0. -# -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: -# http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. - -import os -from unittest import mock - -from fetchcode import api - -@mock.patch('fetchcode.api.requests.get') -def test_fetch(mock_get): - - with mock.patch('fetchcode.api.open', mock.mock_open()) as mocked_file: - url = 'https://raw.githubusercontent.com/lk-geimfari/unittest/master/logo.png' - response = api.fetch(url=url) - assert response is not None - assert hasattr(response,'size') - assert hasattr(response,'location') - assert hasattr(response,'url') - assert hasattr(response,'content_type') diff --git a/tests/test_fetch.py b/tests/test_fetch.py new file mode 100644 index 00000000..09b0bf4a --- /dev/null +++ b/tests/test_fetch.py @@ -0,0 +1,65 @@ +# fetchcode is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/fetchcode for support and download. +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and http://aboutcode.org +# +# This software is licensed under the Apache License version 2.0. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: +# http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +from unittest import mock + +import pytest + +from fetchcode import fetch + + +@mock.patch('fetchcode.requests.get') +def test_fetch_http_with_tempfile(mock_get): + mock_get.return_value.headers = { + 'content-type': 'image/png', + 'content-length': '1000999', + } + + with mock.patch('fetchcode.open', mock.mock_open()) as mocked_file: + url = 'https://raw.githubusercontent.com/TG1999/converge/master/assets/Group%2022.png' + response = fetch(url=url) + assert response is not None + assert 1000999 == response.size + assert url == response.url + assert 'image/png' == response.content_type + + +@mock.patch('fetchcode.FTP') +def test_fetch_with_wrong_url(mock_get): + with pytest.raises(Exception) as e_info: + url = 'ftp://speedtest/1KB.zip' + response = fetch(url=url) + assert 'Not a valid URL' == e_info + + +@mock.patch('fetchcode.FTP', autospec=True) +def test_fetch_ftp_with_tempfile(mock_ftp_constructor): + mock_ftp = mock_ftp_constructor.return_value + mock_ftp_constructor.return_value.size.return_value = 1024 + with mock.patch('fetchcode.open', mock.mock_open()) as mocked_file: + response = fetch('ftp://speedtest.tele2.net/1KB.zip') + assert 1024 == response.size + mock_ftp_constructor.assert_called_with('speedtest.tele2.net') + assert mock_ftp.login.called == True + mock_ftp.cwd.assert_called_with('/') + assert mock_ftp.retrbinary.called + + +def test_fetch_with_scheme_not_present(): + with pytest.raises(Exception) as e_info: + url = 'abc://speedtest/1KB.zip' + response = fetch(url=url) + assert 'Not a supported/known scheme.' == e_info