Add skeleton files to repo

.gitattributes

@@ -0,0 +1,3 @@
+# Ignore all Git auto CR/LF line endings conversions
+* -text
+pyproject.toml export-subst

.gitignore

@@ -112,6 +112,7 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+tmp
 
 # Spyder project settings
 .spyderproject

.readthedocs.yml

@@ -0,0 +1,18 @@
+# .readthedocs.yml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Where the Sphinx conf.py file is located
+sphinx:
+   configuration: docs/source/conf.py
+
+# Setting the python version and doc build requirements
+python:
+  install:
+    - method: pip
+      path: .
+      extra_requirements:
+        - docs

AUTHORS.rst

@@ -5,4 +5,4 @@ The following organizations or individuals have contributed to this repo:
 - Philippe Ombredanne @ pombredanne
 - Steven Esser @ majurg
 - Tushar Goel @ TG1999
- 
+

CHANGELOG.rst

@@ -2,4 +2,4 @@ Release notes
 -------------
 ### Version 0.0.1 2020-10-29
 
-Initial release
+Initial release

MANIFEST.in

@@ -0,0 +1,15 @@
+graft src
+
+include *.LICENSE
+include NOTICE
+include *.ABOUT
+include *.toml
+include *.yml
+include *.rst
+include setup.*
+include configure*
+include requirements*
+include .git*
+
+global-exclude *.py[co] __pycache__ *.*~
+

NOTICE

@@ -0,0 +1,19 @@
+#
+# Copyright (c) nexB Inc. and others.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Visit https://aboutcode.org and https://github.com/nexB/ for support and download.
+# ScanCode is a trademark of nexB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

README.rst

@@ -1,15 +1,15 @@
 *********
 Fetchcode
 *********
-It is a library to reliably fetch code via HTTP, FTP and version control systems. 
+It is a library to reliably fetch code via HTTP, FTP and version control systems.
 
 Installation
 ############
-Clone the repo using 
+Clone the repo using
 
 :code:`git clone https://github.com/nexB/fetchcode`
 
-Then install all the requirements using 
+Then install all the requirements using
 
 :code:`pip3 install -r requirements.txt`
 
@@ -33,4 +33,4 @@ Usage of API to fetch HTTP/S and FTP URLs
     # 'scheme' scheme of the URL
     # 'size' size of the retrieved content in bytes
     # 'url' fetched URL
-    resp = fetch(url = url)
+    resp = fetch(url = url)

apache-2.0.LICENSE

@@ -1,4 +1,3 @@
-
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
@@ -199,4 +198,4 @@
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
-   limitations under the License.
+   limitations under the License.

azure-pipelines.yml

@@ -0,0 +1,64 @@
+
+################################################################################
+# We use Azure to run the full tests suites on multiple Python 3.x
+# on multiple Windows, macOS and Linux versions all on 64 bits
+# These jobs are using VMs with Azure-provided Python builds
+################################################################################
+
+jobs:
+
+    - template: etc/ci/azure-posix.yml
+      parameters:
+          job_name: ubuntu16_cpython
+          image_name: ubuntu-16.04
+          python_versions: ['3.6', '3.7', '3.8', '3.9']
+          test_suites:
+              all: tmp/bin/pytest -vvs
+
+    - template: etc/ci/azure-posix.yml
+      parameters:
+          job_name: ubuntu18_cpython
+          image_name: ubuntu-18.04
+          python_versions: ['3.6', '3.7', '3.8', '3.9']
+          test_suites:
+              all: tmp/bin/pytest -n 2 -vvs
+
+    - template: etc/ci/azure-posix.yml
+      parameters:
+          job_name: ubuntu20_cpython
+          image_name: ubuntu-20.04
+          python_versions: ['3.6', '3.7', '3.8', '3.9']
+          test_suites:
+              all: tmp/bin/pytest -n 2 -vvs
+
+    - template: etc/ci/azure-posix.yml
+      parameters:
+          job_name: macos1014_cpython
+          image_name: macos-10.14
+          python_versions: ['3.6', '3.7', '3.8', '3.9']
+          test_suites:
+              all: tmp/bin/pytest -n 2 -vvs
+
+    - template: etc/ci/azure-posix.yml
+      parameters:
+          job_name: macos1015_cpython
+          image_name: macos-10.15
+          python_versions: ['3.6', '3.7', '3.8', '3.9']
+          test_suites:
+              all: tmp/bin/pytest -n 2 -vvs
+
+    - template: etc/ci/azure-win.yml
+      parameters:
+          job_name: win2016_cpython
+          image_name: vs2017-win2016
+          python_versions: ['3.6', '3.7', '3.8', '3.9']
+          test_suites:
+              all: tmp\Scripts\pytest -n 2 -vvs
+
+    - template: etc/ci/azure-win.yml
+      parameters:
+          job_name: win2019_cpython
+          image_name: windows-2019
+          python_versions: ['3.6', '3.7', '3.8', '3.9']
+          test_suites:
+              all: tmp\Scripts\pytest -n 2 -vvs

configure

@@ -1,44 +1,169 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved.
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/ for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
 #
 
 set -e
 #set -x
 
-# source this script for a basic setup and configuration for local development
+################################
+# A configuration script to set things up:
+# create a virtualenv and install or update thirdparty packages.
+# Source this script for initial configuration
+# Use configure --help for details
+#
+# This script will search for a virtualenv.pyz app in etc/thirdparty/virtualenv.pyz
+# Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default
+################################
+CLI_ARGS=$1
 
-CONFIGURE_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+################################
+# Defaults. Change these variables to customize this script
+################################
 
+# Requirement arguments passed to pip and used by default or with --dev.
+REQUIREMENTS="--editable ."
+DEV_REQUIREMENTS="--editable .[testing]"
 
-if [[ "$1" == "--clean" ]]; then
-    rm -rf "$CONFIGURE_ROOT_DIR/tmp"
-    exit
+# where we create a virtualenv
+VIRTUALENV_DIR=tmp
+
+# Cleanable files and directories with the --clean option
+CLEANABLE="
+    build
+	tmp"
+
+# extra  arguments passed to pip
+PIP_EXTRA_ARGS=" "
+
+# the URL to download virtualenv.pyz if needed
+VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz
+################################
+
+
+################################
+# Current directory where this script lives
+CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin
+
+
+################################
+# Set the quiet flag to empty if not defined
+if [[ "$CFG_QUIET" == "" ]]; then
+    CFG_QUIET=" "
 fi
 
 
-if [[ "$PYTHON_EXE" == "" ]]; then
-    PYTHON_EXE=python3
+################################
+# find a proper Python to run
+# Use environment variables or a file if available.
+# Otherwise the latest Python by default.
+if [[ "$PYTHON_EXECUTABLE" == "" ]]; then
+    # check for a file named PYTHON_EXECUTABLE
+    if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then
+        PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE")
+    else
+        PYTHON_EXECUTABLE=python3
+    fi
 fi
 
 
-function setup {
-    # create a virtualenv on Python
-    mkdir -p $CONFIGURE_ROOT_DIR/tmp
-    wget -O $CONFIGURE_ROOT_DIR/tmp/virtualenv.pyz https://bootstrap.pypa.io/virtualenv.pyz
-    $PYTHON_EXE $CONFIGURE_ROOT_DIR/tmp/virtualenv.pyz $CONFIGURE_ROOT_DIR/tmp
-    source $CONFIGURE_ROOT_DIR/tmp/bin/activate
-    $CONFIGURE_ROOT_DIR/tmp/bin/pip install --upgrade pip virtualenv setuptools wheel
+################################
+cli_help() {
+    echo An initial configuration script
+    echo "  usage: ./configure [options]"
+    echo
+    echo The default is to configure for regular use. Use --dev for development.
+    echo
+    echo The options are:
+    echo " --clean: clean built and installed files and exit."
+    echo " --dev:   configure the environment for development."
+    echo " --help:  display this help message and exit."
+    echo
+    echo By default, the python interpreter version found in the path is used.
+    echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to
+    echo configure another Python executable interpreter to use. If this is not
+    echo set, a file named PYTHON_EXECUTABLE containing a single line with the
+    echo path of the Python executable to use will be checked last.
+    set +e
+    exit
 }
 
 
-setup
+clean() {
+    # Remove cleanable file and directories and files from the root dir.
+    echo "* Cleaning ..."
+    for cln in $CLEANABLE;
+        do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}";
+    done
+    set +e
+    exit
+}
 
-$CONFIGURE_ROOT_DIR/tmp/bin/pip install -e .[testing]
-$CONFIGURE_ROOT_DIR/tmp/bin/pip install fetchcode
 
-if [ -f "$CONFIGURE_ROOT_DIR/tmp/bin/activate" ]; then
-    source "$CONFIGURE_ROOT_DIR/tmp/bin/activate"
-fi
+create_virtualenv() {
+    # create a virtualenv for Python
+    # Note: we do not use the bundled Python 3 "venv" because its behavior and
+    # presence is not consistent across Linux distro and sometimes pip is not
+    # included either by default. The virtualenv.pyz app cures all these issues.
+
+    VENV_DIR="$1"
+    if [ ! -f "$CFG_BIN_DIR/python" ]; then
+
+        mkdir -p "$CFG_ROOT_DIR/$VENV_DIR"
+
+        if [ -f "$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz" ]; then
+            VIRTUALENV_PYZ="$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz"
+        else
+            VIRTUALENV_PYZ="$CFG_ROOT_DIR/$VENV_DIR/virtualenv.pyz"
+            wget -O "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL"
+        fi
+
+        $PYTHON_EXECUTABLE "$VIRTUALENV_PYZ" \
+            --wheel embed --pip embed --setuptools embed \
+            --seeder pip \
+            --never-download \
+            --no-periodic-update \
+            --no-vcs-ignore \
+            $CFG_QUIET \
+            "$CFG_ROOT_DIR/$VENV_DIR"
+    fi
+}
+
+
+install_packages() {
+    # install requirements in virtualenv
+    # note: --no-build-isolation means that pip/wheel/setuptools will not
+    # be reinstalled a second time and reused from the virtualenv and this
+    # speeds up the installation.
+    # We always have the PEP517 build dependencies installed already.
+
+    "$CFG_BIN_DIR/pip" install \
+        --upgrade \
+        --no-build-isolation \
+        $CFG_QUIET \
+        $PIP_EXTRA_ARGS \
+        $1
+}
+
+
+################################
+# Main command line entry point
+CFG_DEV_MODE=0
+CFG_REQUIREMENTS=$REQUIREMENTS
+
+case "$CLI_ARGS" in
+    --help)    cli_help;;
+    --clean)   clean;;
+    --dev)     CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;;
+esac
+
+create_virtualenv "$VIRTUALENV_DIR"
+install_packages "$CFG_REQUIREMENTS"
+. "$CFG_BIN_DIR/activate"
 
-set +e
+set +e