From 1524f58c1429a55cadc4a1449a8eebbff2671264 Mon Sep 17 00:00:00 2001 From: Panu Matilainen Date: Fri, 21 Nov 2008 23:29:54 +0200 Subject: [PATCH 001/160] Add basic version comparison testing for install/upgrade --- tests/rpmvercmp.at | 96 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 tests/rpmvercmp.at diff --git a/tests/rpmvercmp.at b/tests/rpmvercmp.at new file mode 100644 index 00000000..08b8df8a --- /dev/null +++ b/tests/rpmvercmp.at @@ -0,0 +1,96 @@ +# rpmvercmp.at: rpm version comparison tests + +# ------------------------------ +# Test normal upgrade +AT_SETUP([rpm -U upgrade to newer]) +AT_CHECK([ +RPMDB_CLEAR +rm -rf "${TOPDIR}" + +for v in "1.0" "2.0"; do + run rpmbuild --quiet -bb \ + --define "ver $v" \ + ${RPMDATA}/SPECS/versiontest.spec +done + +runroot rpm -U "${TOPDIR}"/RPMS/noarch/versiontest-1.0-1.noarch.rpm +runroot rpm -U "${TOPDIR}"/RPMS/noarch/versiontest-2.0-1.noarch.rpm +], +[0], +[ignore], +[ignore]) +AT_CLEANUP + +# Test upgrading to older package (should fail) +AT_SETUP([rpm -U upgrade to older]) +AT_CHECK([ +RPMDB_CLEAR + +runroot rpm -U "${TOPDIR}"/RPMS/noarch/versiontest-2.0-1.noarch.rpm +runroot rpm -U "${TOPDIR}"/RPMS/noarch/versiontest-1.0-1.noarch.rpm +], +[2], +[ignore], +[ignore]) +AT_CLEANUP + +# Test downgrading to older package with --oldpackage +AT_SETUP([rpm -U --oldpackage downgrade]) +AT_CHECK([ +RPMDB_CLEAR + +runroot rpm -U "${TOPDIR}"/RPMS/noarch/versiontest-2.0-1.noarch.rpm +runroot rpm -U --oldpackage "${TOPDIR}"/RPMS/noarch/versiontest-1.0-1.noarch.rpm +], +[0], +[ignore], +[ignore]) +AT_CLEANUP + +# Test upgrade of different versions in same transaction +# XXX test that only 2.0-1 was installed +AT_SETUP([rpm -U two versions of same package]) +AT_CHECK([ +RPMDB_CLEAR + +runroot rpm -U \ + "${TOPDIR}"/RPMS/noarch/versiontest-1.0-1.noarch.rpm \ + "${TOPDIR}"/RPMS/noarch/versiontest-2.0-1.noarch.rpm +], +[0], +[ignore], +[ignore]) +AT_CLEANUP + +# Test install of two different versions in same transaction +# TODO: test that both got installed +AT_SETUP([rpm -i two versions of same package]) +AT_CHECK([ +RPMDB_CLEAR + +runroot rpm -i \ + "${TOPDIR}"/RPMS/noarch/versiontest-1.0-1.noarch.rpm \ + "${TOPDIR}"/RPMS/noarch/versiontest-2.0-1.noarch.rpm +], +[0], +[ignore], +[ignore]) +AT_CLEANUP + +# Test install of two different versions in same transaction +# TODO: test only one was installed +AT_SETUP([rpm -i identical versions of same package]) +AT_CHECK([ +RPMDB_CLEAR + +# TODO test just one was installed, this only emits warnings +runroot rpm -ivh \ + "${TOPDIR}"/RPMS/noarch/versiontest-1.0-1.noarch.rpm \ + "${TOPDIR}"/RPMS/noarch/versiontest-1.0-1.noarch.rpm +], +[0], +[ignore], +[ignore]) +AT_CLEANUP + +# TODO: the same with epoch vs no epoch From b19736bd3566c1ad889051bcce41a112883a3593 Mon Sep 17 00:00:00 2001 From: Panu Matilainen Date: Sat, 22 Nov 2008 13:26:43 +0200 Subject: [PATCH 002/160] Add banners for nicer test output --- tests/rpmvercmp.at | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/rpmvercmp.at b/tests/rpmvercmp.at index 08b8df8a..a125f3ac 100644 --- a/tests/rpmvercmp.at +++ b/tests/rpmvercmp.at @@ -1,5 +1,7 @@ # rpmvercmp.at: rpm version comparison tests +AT_BANNER([RPM version comparison]) + # ------------------------------ # Test normal upgrade AT_SETUP([rpm -U upgrade to newer]) From 82d1e73f65c334e7966388da066384767f1619e5 Mon Sep 17 00:00:00 2001 From: Panu Matilainen Date: Fri, 19 Mar 2010 10:49:17 +0200 Subject: [PATCH 003/160] Verify the various version comparison results in testsuite --- tests/rpmvercmp.at | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/tests/rpmvercmp.at b/tests/rpmvercmp.at index a125f3ac..4d1252dd 100644 --- a/tests/rpmvercmp.at +++ b/tests/rpmvercmp.at @@ -17,10 +17,12 @@ done runroot rpm -U "${TOPDIR}"/RPMS/noarch/versiontest-1.0-1.noarch.rpm runroot rpm -U "${TOPDIR}"/RPMS/noarch/versiontest-2.0-1.noarch.rpm +runroot rpm -q versiontest ], [0], -[ignore], -[ignore]) +[versiontest-2.0-1.noarch +], +[]) AT_CLEANUP # Test upgrading to older package (should fail) @@ -32,8 +34,9 @@ runroot rpm -U "${TOPDIR}"/RPMS/noarch/versiontest-2.0-1.noarch.rpm runroot rpm -U "${TOPDIR}"/RPMS/noarch/versiontest-1.0-1.noarch.rpm ], [2], -[ignore], -[ignore]) +[], +[ package versiontest-2.0-1.noarch (which is newer than versiontest-1.0-1.noarch) is already installed +]) AT_CLEANUP # Test downgrading to older package with --oldpackage @@ -43,14 +46,15 @@ RPMDB_CLEAR runroot rpm -U "${TOPDIR}"/RPMS/noarch/versiontest-2.0-1.noarch.rpm runroot rpm -U --oldpackage "${TOPDIR}"/RPMS/noarch/versiontest-1.0-1.noarch.rpm +runroot rpm -q versiontest ], [0], -[ignore], +[versiontest-1.0-1.noarch +], [ignore]) AT_CLEANUP # Test upgrade of different versions in same transaction -# XXX test that only 2.0-1 was installed AT_SETUP([rpm -U two versions of same package]) AT_CHECK([ RPMDB_CLEAR @@ -58,14 +62,15 @@ RPMDB_CLEAR runroot rpm -U \ "${TOPDIR}"/RPMS/noarch/versiontest-1.0-1.noarch.rpm \ "${TOPDIR}"/RPMS/noarch/versiontest-2.0-1.noarch.rpm +runroot rpm -q versiontest ], [0], -[ignore], -[ignore]) +[versiontest-2.0-1.noarch +], +[]) AT_CLEANUP # Test install of two different versions in same transaction -# TODO: test that both got installed AT_SETUP([rpm -i two versions of same package]) AT_CHECK([ RPMDB_CLEAR @@ -73,10 +78,13 @@ RPMDB_CLEAR runroot rpm -i \ "${TOPDIR}"/RPMS/noarch/versiontest-1.0-1.noarch.rpm \ "${TOPDIR}"/RPMS/noarch/versiontest-2.0-1.noarch.rpm +runroot rpm -q versiontest ], [0], -[ignore], -[ignore]) +[versiontest-2.0-1.noarch +versiontest-1.0-1.noarch +], +[]) AT_CLEANUP # Test install of two different versions in same transaction @@ -85,14 +93,15 @@ AT_SETUP([rpm -i identical versions of same package]) AT_CHECK([ RPMDB_CLEAR -# TODO test just one was installed, this only emits warnings -runroot rpm -ivh \ +runroot rpm -i \ "${TOPDIR}"/RPMS/noarch/versiontest-1.0-1.noarch.rpm \ "${TOPDIR}"/RPMS/noarch/versiontest-1.0-1.noarch.rpm +runroot rpm -q versiontest ], [0], -[ignore], -[ignore]) +[versiontest-1.0-1.noarch +], +[]) AT_CLEANUP # TODO: the same with epoch vs no epoch From 4750ca0db5934636bbcbc75fa237401a671c37e9 Mon Sep 17 00:00:00 2001 From: Panu Matilainen Date: Mon, 22 Nov 2010 10:36:54 +0200 Subject: [PATCH 004/160] Move the current "version compare" tests to rpm install tests - These are more about rpm install/upgrade behavior than pure version comparison, although obviously version comparison is involved --- tests/rpmvercmp.at | 103 --------------------------------------------- 1 file changed, 103 deletions(-) diff --git a/tests/rpmvercmp.at b/tests/rpmvercmp.at index 4d1252dd..ab1e462b 100644 --- a/tests/rpmvercmp.at +++ b/tests/rpmvercmp.at @@ -2,106 +2,3 @@ AT_BANNER([RPM version comparison]) -# ------------------------------ -# Test normal upgrade -AT_SETUP([rpm -U upgrade to newer]) -AT_CHECK([ -RPMDB_CLEAR -rm -rf "${TOPDIR}" - -for v in "1.0" "2.0"; do - run rpmbuild --quiet -bb \ - --define "ver $v" \ - ${RPMDATA}/SPECS/versiontest.spec -done - -runroot rpm -U "${TOPDIR}"/RPMS/noarch/versiontest-1.0-1.noarch.rpm -runroot rpm -U "${TOPDIR}"/RPMS/noarch/versiontest-2.0-1.noarch.rpm -runroot rpm -q versiontest -], -[0], -[versiontest-2.0-1.noarch -], -[]) -AT_CLEANUP - -# Test upgrading to older package (should fail) -AT_SETUP([rpm -U upgrade to older]) -AT_CHECK([ -RPMDB_CLEAR - -runroot rpm -U "${TOPDIR}"/RPMS/noarch/versiontest-2.0-1.noarch.rpm -runroot rpm -U "${TOPDIR}"/RPMS/noarch/versiontest-1.0-1.noarch.rpm -], -[2], -[], -[ package versiontest-2.0-1.noarch (which is newer than versiontest-1.0-1.noarch) is already installed -]) -AT_CLEANUP - -# Test downgrading to older package with --oldpackage -AT_SETUP([rpm -U --oldpackage downgrade]) -AT_CHECK([ -RPMDB_CLEAR - -runroot rpm -U "${TOPDIR}"/RPMS/noarch/versiontest-2.0-1.noarch.rpm -runroot rpm -U --oldpackage "${TOPDIR}"/RPMS/noarch/versiontest-1.0-1.noarch.rpm -runroot rpm -q versiontest -], -[0], -[versiontest-1.0-1.noarch -], -[ignore]) -AT_CLEANUP - -# Test upgrade of different versions in same transaction -AT_SETUP([rpm -U two versions of same package]) -AT_CHECK([ -RPMDB_CLEAR - -runroot rpm -U \ - "${TOPDIR}"/RPMS/noarch/versiontest-1.0-1.noarch.rpm \ - "${TOPDIR}"/RPMS/noarch/versiontest-2.0-1.noarch.rpm -runroot rpm -q versiontest -], -[0], -[versiontest-2.0-1.noarch -], -[]) -AT_CLEANUP - -# Test install of two different versions in same transaction -AT_SETUP([rpm -i two versions of same package]) -AT_CHECK([ -RPMDB_CLEAR - -runroot rpm -i \ - "${TOPDIR}"/RPMS/noarch/versiontest-1.0-1.noarch.rpm \ - "${TOPDIR}"/RPMS/noarch/versiontest-2.0-1.noarch.rpm -runroot rpm -q versiontest -], -[0], -[versiontest-2.0-1.noarch -versiontest-1.0-1.noarch -], -[]) -AT_CLEANUP - -# Test install of two different versions in same transaction -# TODO: test only one was installed -AT_SETUP([rpm -i identical versions of same package]) -AT_CHECK([ -RPMDB_CLEAR - -runroot rpm -i \ - "${TOPDIR}"/RPMS/noarch/versiontest-1.0-1.noarch.rpm \ - "${TOPDIR}"/RPMS/noarch/versiontest-1.0-1.noarch.rpm -runroot rpm -q versiontest -], -[0], -[versiontest-1.0-1.noarch -], -[]) -AT_CLEANUP - -# TODO: the same with epoch vs no epoch From 6296e37791dfa0d5a0bc295cef2df132efaee7ce Mon Sep 17 00:00:00 2001 From: Panu Matilainen Date: Mon, 22 Nov 2010 11:40:44 +0200 Subject: [PATCH 005/160] Add bunch of rpmvercmp() algorithm tests to the test-suite --- tests/rpmvercmp.at | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/rpmvercmp.at b/tests/rpmvercmp.at index ab1e462b..c705e451 100644 --- a/tests/rpmvercmp.at +++ b/tests/rpmvercmp.at @@ -1,4 +1,31 @@ # rpmvercmp.at: rpm version comparison tests +m4_define([RPMVERCMP],[ +AT_SETUP([rpmvercmp($1, $2) = $3]) +AT_KEYWORDS([vercmp]) +AT_CHECK([run rpm --eval '%{lua: print(rpm.vercmp("$1", "$2"))}'], [0], [$3 +], []) +AT_CLEANUP +]) + AT_BANNER([RPM version comparison]) +RPMVERCMP(1.0, 1.0, 0) +RPMVERCMP(1.0, 2.0, -1) +RPMVERCMP(2.0, 2.0.1, -1) +RPMVERCMP(2.0.1a, 2.0.1, 1) +RPMVERCMP(5.5p1, 5.5p2, -1) +RPMVERCMP(5.6p1, 5.5p2, 1) +RPMVERCMP(10a2, 10b2, -1) +RPMVERCMP(1.0a, 1.0aa, -1) +RPMVERCMP(6.0rc1, 6.0, 1) +RPMVERCMP(6.0.rc1, 6.0, 1) +RPMVERCMP(10.1.1.0001, 10.1.1.39, -1) +RPMVERCMP(5.0, 4.999.9, 1) +RPMVERCMP(2.030, 2.031, -1) +RPMVERCMP(20101122, 20101121, 1) +RPMVERCMP(2.0, 2_0, 0) + +dnl The results are highly dubious when non-ascii characters are involved, +dnl these are "obviously" different versions but rpm thinks they're equal +dnl RPMVERCMP(1.1.α, 1.1.β, 0) From d5e825169d7eb1c0026645fe0f6c576020aa640a Mon Sep 17 00:00:00 2001 From: Panu Matilainen Date: Fri, 1 Apr 2011 17:18:46 +0300 Subject: [PATCH 006/160] Switch the two remaining rpm runs under fakechroot - Everything in the test-suite except rpmbuild execution is now performed in the fakechroot environment --- tests/rpmvercmp.at | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/rpmvercmp.at b/tests/rpmvercmp.at index c705e451..bc32686f 100644 --- a/tests/rpmvercmp.at +++ b/tests/rpmvercmp.at @@ -3,7 +3,7 @@ m4_define([RPMVERCMP],[ AT_SETUP([rpmvercmp($1, $2) = $3]) AT_KEYWORDS([vercmp]) -AT_CHECK([run rpm --eval '%{lua: print(rpm.vercmp("$1", "$2"))}'], [0], [$3 +AT_CHECK([runroot rpm --eval '%{lua: print(rpm.vercmp("$1", "$2"))}'], [0], [$3 ], []) AT_CLEANUP ]) From 145c442cbdd39d53cb9323141902f9414aafcfea Mon Sep 17 00:00:00 2001 From: Panu Matilainen Date: Fri, 20 Apr 2012 13:29:21 +0300 Subject: [PATCH 007/160] Add further version comparison test-cases - Add pile of further tests for the more common constructs, collect various (old and new) corner cases from bugzilla. --- tests/rpmvercmp.at | 93 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 87 insertions(+), 6 deletions(-) diff --git a/tests/rpmvercmp.at b/tests/rpmvercmp.at index bc32686f..6e92fb79 100644 --- a/tests/rpmvercmp.at +++ b/tests/rpmvercmp.at @@ -12,20 +12,101 @@ AT_BANNER([RPM version comparison]) RPMVERCMP(1.0, 1.0, 0) RPMVERCMP(1.0, 2.0, -1) +RPMVERCMP(2.0, 1.0, 1) + +RPMVERCMP(2.0.1, 2.0.1, 0) RPMVERCMP(2.0, 2.0.1, -1) +RPMVERCMP(2.0.1, 2.0, 1) + +RPMVERCMP(2.0.1a, 2.0.1a, 0) RPMVERCMP(2.0.1a, 2.0.1, 1) +RPMVERCMP(2.0.1, 2.0.1a, -1) + +RPMVERCMP(5.5p1, 5.5p1, 0) RPMVERCMP(5.5p1, 5.5p2, -1) +RPMVERCMP(5.5p2, 5.5p1, 1) + +RPMVERCMP(5.5p10, 5.5p10, 0) +RPMVERCMP(5.5p1, 5.5p10, -1) +RPMVERCMP(5.5p10, 5.5p1, 1) + +RPMVERCMP(10xyz, 10.1xyz, -1) +RPMVERCMP(10.1xyz, 10xyz, 1) + +RPMVERCMP(xyz10, xyz10, 0) +RPMVERCMP(xyz10, xyz10.1, -1) +RPMVERCMP(xyz10.1, xyz10, 1) + +RPMVERCMP(xyz.4, xyz.4, 0) +RPMVERCMP(xyz.4, 8, -1) +RPMVERCMP(8, xyz.4, 1) +RPMVERCMP(xyz.4, 2, -1) +RPMVERCMP(2, xyz.4, 1) + +RPMVERCMP(5.5p2, 5.6p1, -1) RPMVERCMP(5.6p1, 5.5p2, 1) + +RPMVERCMP(5.6p1, 6.5p1, -1) +RPMVERCMP(6.5p1, 5.6p1, 1) + +RPMVERCMP(6.0.rc1, 6.0, 1) +RPMVERCMP(6.0, 6.0.rc1, -1) + +RPMVERCMP(10b2, 10a1, 1) RPMVERCMP(10a2, 10b2, -1) + +RPMVERCMP(1.0aa, 1.0aa, 0) RPMVERCMP(1.0a, 1.0aa, -1) -RPMVERCMP(6.0rc1, 6.0, 1) -RPMVERCMP(6.0.rc1, 6.0, 1) -RPMVERCMP(10.1.1.0001, 10.1.1.39, -1) +RPMVERCMP(1.0aa, 1.0a, 1) + +RPMVERCMP(10.0001, 10.0001, 0) +RPMVERCMP(10.0001, 10.1, 0) +RPMVERCMP(10.1, 10.0001, 0) +RPMVERCMP(10.0001, 10.0039, -1) +RPMVERCMP(10.0039, 10.0001, 1) + +RPMVERCMP(4.999.9, 5.0, -1) RPMVERCMP(5.0, 4.999.9, 1) -RPMVERCMP(2.030, 2.031, -1) + +RPMVERCMP(20101121, 20101121, 0) +RPMVERCMP(20101121, 20101122, -1) RPMVERCMP(20101122, 20101121, 1) + +RPMVERCMP(2_0, 2_0, 0) RPMVERCMP(2.0, 2_0, 0) +RPMVERCMP(2_0, 2.0, 0) + +dnl RhBug:178798 case +RPMVERCMP(a, a, 0) +RPMVERCMP(a+, a+, 0) +RPMVERCMP(a+, a_, 0) +RPMVERCMP(a_, a+, 0) +RPMVERCMP(+a, +a, 0) +RPMVERCMP(+a, _a, 0) +RPMVERCMP(_a, +a, 0) +RPMVERCMP(+_, +_, 0) +RPMVERCMP(_+, +_, 0) +RPMVERCMP(_+, _+, 0) +RPMVERCMP(+, _, 0) +RPMVERCMP(_, +, 0) + +dnl These are included here to document current, arguably buggy behaviors +dnl for reference purposes and for easy checking against unintended +dnl behavior changes. +dnl +dnl AT_BANNER([RPM version comparison oddities]) +dnl RhBug:811992 case +dnl RPMVERCMP(1b.fc17, 1b.fc17, 0) +dnl RPMVERCMP(1b.fc17, 1.fc17, -1) +dnl RPMVERCMP(1.fc17, 1b.fc17, 1) +dnl RPMVERCMP(1g.fc17, 1g.fc17, 0) +dnl RPMVERCMP(1g.fc17, 1.fc17, 1) +dnl RPMVERCMP(1.fc17, 1g.fc17, -1) -dnl The results are highly dubious when non-ascii characters are involved, -dnl these are "obviously" different versions but rpm thinks they're equal +dnl Non-ascii characters are considered equal so these are all the same, eh... +dnl RPMVERCMP(1.1.α, 1.1.α, 0) dnl RPMVERCMP(1.1.α, 1.1.β, 0) +dnl RPMVERCMP(1.1.β, 1.1.α, 0) +dnl RPMVERCMP(1.1.αα, 1.1.α, 0) +dnl RPMVERCMP(1.1.α, 1.1.ββ, 0) +dnl RPMVERCMP(1.1.ββ, 1.1.αα, 0) From da565a87de475284099345bdb94a225730019b03 Mon Sep 17 00:00:00 2001 From: Panu Matilainen Date: Mon, 23 Apr 2012 11:16:04 +0300 Subject: [PATCH 008/160] Add a few basic test-cases for tilde in version comparison --- tests/rpmvercmp.at | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/rpmvercmp.at b/tests/rpmvercmp.at index 6e92fb79..2a25bdd9 100644 --- a/tests/rpmvercmp.at +++ b/tests/rpmvercmp.at @@ -90,6 +90,16 @@ RPMVERCMP(_+, _+, 0) RPMVERCMP(+, _, 0) RPMVERCMP(_, +, 0) +dnl Basic testcases for tilde sorting +RPMVERCMP(1.0~rc1, 1.0~rc1, 0) +RPMVERCMP(1.0~rc1, 1.0, -1) +RPMVERCMP(1.0, 1.0~rc1, 1) +RPMVERCMP(1.0~rc1, 1.0~rc2, -1) +RPMVERCMP(1.0~rc2, 1.0~rc1, 1) +RPMVERCMP(1.0~rc1~git123, 1.0~rc1~git123, 0) +RPMVERCMP(1.0~rc1~git123, 1.0~rc1, -1) +RPMVERCMP(1.0~rc1, 1.0~rc1~git123, 1) + dnl These are included here to document current, arguably buggy behaviors dnl for reference purposes and for easy checking against unintended dnl behavior changes. From 74822a3ff8ebe886b89769c097a3670a3fed2e6f Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Sat, 23 Sep 2017 04:32:23 -0400 Subject: [PATCH 009/160] move pkgcore module to src directory --- src/pkgcore/ebuild/cpv.py | 401 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 401 insertions(+) create mode 100644 src/pkgcore/ebuild/cpv.py diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py new file mode 100644 index 00000000..a8bb899c --- /dev/null +++ b/src/pkgcore/ebuild/cpv.py @@ -0,0 +1,401 @@ +# Copyright: 2005-2011 Brian Harring +# Copyright: 2005 Jason Stubbs +# License: GPL2/BSD + + +"""gentoo ebuild specific base package class""" + +__all__ = ("CPV", "versioned_CPV", "unversioned_CPV") + +from itertools import izip + +from snakeoil.compatibility import cmp +from snakeoil.demandload import demandload, demand_compile_regexp +from snakeoil.klass import inject_richcmp_methods_from_cmp + +from pkgcore.ebuild.errors import InvalidCPV +from pkgcore.package import base + +# do this to break the cycle. +demandload("pkgcore.ebuild:atom") + +demand_compile_regexp( + 'suffix_regexp', '^(alpha|beta|rc|pre|p)(\\d*)$') + +suffix_value = {"pre": -2, "p": 1, "alpha": -4, "beta": -3, "rc": -1} + +# while the package section looks fugly, there is a reason for it- +# to prevent version chunks from showing up in the package + +demand_compile_regexp( + 'isvalid_version_re', + r"^(?:\d+)(?:\.\d+)*[a-zA-Z]?(?:_(p(?:re)?|beta|alpha|rc)\d*)*$") + +demand_compile_regexp( + 'isvalid_cat_re', r"^(?:[a-zA-Z0-9][-a-zA-Z0-9+._]*(?:/(?!$))?)+$") + +# empty string is fine, means a -- was encounter. +demand_compile_regexp( + '_pkg_re', r"^[a-zA-Z0-9+_]+$") + + +def isvalid_pkg_name(chunks): + if not chunks[0] or chunks[0][0] == '+': + # this means a leading -; additionally, '+asdf' is disallowed + return False + mf = _pkg_re.match + if not all(mf(s) for s in chunks[:-1]): + return False + if chunks[-1]: + return mf(chunks[-1]) and not isvalid_version_re.match(chunks[-1]) + return True + +def isvalid_rev(s): + return s and s[0] == 'r' and s[1:].isdigit() + + +class _native_CPV(object): + + """ + base ebuild package class + + :ivar category: str category + :ivar package: str package + :ivar key: strkey (cat/pkg) + :ivar version: str version + :ivar revision: int revision + :ivar versioned_atom: atom matching this exact version + :ivar unversioned_atom: atom matching all versions of this package + :cvar _get_attr: mapping of attr:callable to generate attributes on the fly + """ + + __slots__ = ("__weakref__", "cpvstr", "key", "category", "package", + "version", "revision", "fullver") + + # if native is being used, forget trying to reuse strings. + def __init__(self, *a, **kwds): + """ + Can be called with one string or with three string args. + + If called with one arg that is the cpv string. (See :obj:`parser` + for allowed syntax). + + If called with three args they are the category, package and + version components of the cpv string respectively. + """ + versioned = True + had_versioned = 'versioned' in kwds + if had_versioned: + versioned = kwds.pop("versioned") + if kwds: + raise TypeError("versioned is the only allowed kwds: %r" % (kwds,)) + l = len(a) + if l == 1: + cpvstr = a[0] + if not had_versioned: + raise TypeError("single arguement invocation requires versioned kwd; %r" + % (cpvstr,)) + elif l == 3: + for x in a: + if not isinstance(x, basestring): + raise TypeError("all args must be strings, got %r" % (a,)) + cpvstr = "%s/%s-%s" % a + versioned = True + else: + raise TypeError("CPV takes 1 arg (cpvstr), or 3 (cat, pkg, ver):" + " got %r" % (a,)) + if not isinstance(cpvstr, basestring): + raise TypeError(self.cpvstr) + + try: + categories, pkgver = cpvstr.rsplit("/", 1) + except ValueError: + # occurs if the rsplit yields only one item + raise InvalidCPV('%s: no package or version components' % cpvstr) + if not isvalid_cat_re.match(categories): + raise InvalidCPV('%s: invalid category name' % cpvstr) + sf = object.__setattr__ + sf(self, 'category', categories) + sf(self, 'cpvstr', cpvstr) + pkg_chunks = pkgver.split("-") + lpkg_chunks = len(pkg_chunks) + if versioned: + if lpkg_chunks == 1: + raise InvalidCPV('%s: missing package version' % cpvstr) + if isvalid_rev(pkg_chunks[-1]): + if lpkg_chunks < 3: + # needs at least ('pkg', 'ver', 'rev') + raise InvalidCPV('%s: missing package name, version, and/or revision' % cpvstr) + rev = int(pkg_chunks.pop(-1)[1:]) + if not rev: + rev = None + # reset the stored cpvstr to drop -r0+ + sf(self, 'cpvstr', "%s/%s" % (categories, + '-'.join(pkg_chunks))) + sf(self, 'revision', rev) + else: + sf(self, 'revision', None) + + if not isvalid_version_re.match(pkg_chunks[-1]): + raise InvalidCPV("%s: invalid version '%s'" % (cpvstr, pkg_chunks[-1])) + sf(self, 'version', pkg_chunks.pop(-1)) + if self.revision: + sf(self, 'fullver', "%s-r%s" % (self.version, self.revision)) + else: + sf(self, 'fullver', self.version) + + if not isvalid_pkg_name(pkg_chunks): + raise InvalidCPV('%s: invalid package name' % cpvstr) + sf(self, 'package', '-'.join(pkg_chunks)) + sf(self, 'key', "%s/%s" % (categories, self.package)) + else: + if not isvalid_pkg_name(pkg_chunks): + raise InvalidCPV('%s: invalid package name' % cpvstr) + sf(self, 'revision', None) + sf(self, 'fullver', None) + sf(self, 'version', None) + sf(self, 'key', cpvstr) + sf(self, 'package', '-'.join(pkg_chunks)) + + def __hash__(self): + return hash(self.cpvstr) + + def __repr__(self): + return '<%s cpvstr=%s @%#8x>' % ( + self.__class__.__name__, getattr(self, 'cpvstr', None), id(self)) + + def __str__(self): + return getattr(self, 'cpvstr', 'None') + + def __cmp__(self, other): + try: + if self.cpvstr == other.cpvstr: + return 0 + + if (self.category and other.category and self.category != other.category): + return cmp(self.category, other.category) + + if self.package and other.package and self.package != other.package: + return cmp(self.package, other.package) + + # note I chucked out valueerror, none checks on versions + # passed in. I suck, I know. + # ~harring + # fails in doing comparison of unversioned atoms against + # versioned atoms + return native_ver_cmp(self.version, self.revision, other.version, + other.revision) + except AttributeError: + return 1 + + +def native_ver_cmp(ver1, rev1, ver2, rev2): + + # If the versions are the same, comparing revisions will suffice. + if ver1 == ver2: + return cmp(rev1, rev2) + + # Split up the versions into dotted strings and lists of suffixes. + parts1 = ver1.split("_") + parts2 = ver2.split("_") + + # If the dotted strings are equal, we can skip doing a detailed comparison. + if parts1[0] != parts2[0]: + + # First split up the dotted strings into their components. + ver_parts1 = parts1[0].split(".") + ver_parts2 = parts2[0].split(".") + + # Pull out any letter suffix on the final components and keep + # them for later. + letters = [] + for ver_parts in (ver_parts1, ver_parts2): + if ver_parts[-1][-1].isalpha(): + letters.append(ord(ver_parts[-1][-1])) + ver_parts[-1] = ver_parts[-1][:-1] + else: + # Using -1 simplifies comparisons later + letters.append(-1) + + # OPT: Pull length calculation out of the loop + ver_parts1_len = len(ver_parts1) + ver_parts2_len = len(ver_parts2) + + # Iterate through the components + for v1, v2 in izip(ver_parts1, ver_parts2): + + # If the string components are equal, the numerical + # components will be equal too. + if v1 == v2: + continue + + # If one of the components begins with a "0" then they + # are compared as floats so that 1.1 > 1.02; else ints. + if v1[0] != "0" and v2[0] != "0": + v1 = int(v1) + v2 = int(v2) + else: + # handle the 0.060 == 0.060 case. + v1 = v1.rstrip("0") + v2 = v2.rstrip("0") + + # If they are not equal, the higher value wins. + c = cmp(v1, v2) + if c: + return c + + if ver_parts1_len > ver_parts2_len: + return 1 + elif ver_parts2_len > ver_parts1_len: + return -1 + + # The dotted components were equal. Let's compare any single + # letter suffixes. + if letters[0] != letters[1]: + return cmp(letters[0], letters[1]) + + # The dotted components were equal, so remove them from our lists + # leaving only suffixes. + del parts1[0] + del parts2[0] + + # OPT: Pull length calculation out of the loop + parts1_len = len(parts1) + parts2_len = len(parts2) + + # Iterate through the suffixes + for x in xrange(max(parts1_len, parts2_len)): + + # If we're at the end of one of our lists, we need to use + # the next suffix from the other list to decide who wins. + if x == parts1_len: + match = suffix_regexp.match(parts2[x]) + val = suffix_value[match.group(1)] + if val: + return cmp(0, val) + return cmp(0, int("0"+match.group(2))) + if x == parts2_len: + match = suffix_regexp.match(parts1[x]) + val = suffix_value[match.group(1)] + if val: + return cmp(val, 0) + return cmp(int("0"+match.group(2)), 0) + + # If the string values are equal, no need to parse them. + # Continue on to the next. + if parts1[x] == parts2[x]: + continue + + # Match against our regular expression to make a split between + # "beta" and "1" in "beta1" + match1 = suffix_regexp.match(parts1[x]) + match2 = suffix_regexp.match(parts2[x]) + + # If our int'ified suffix names are different, use that as the basis + # for comparison. + c = cmp(suffix_value[match1.group(1)], suffix_value[match2.group(1)]) + if c: + return c + + # Otherwise use the digit as the basis for comparison. + c = cmp(int("0"+match1.group(2)), int("0"+match2.group(2))) + if c: + return c + + # Our versions had different strings but ended up being equal. + # The revision holds the final difference. + return cmp(rev1, rev2) + +fake_cat = "fake" +fake_pkg = "pkg" +def cpy_ver_cmp(ver1, rev1, ver2, rev2): + if ver1 == ver2: + return cmp(rev1, rev2) + if ver1 is None: + ver1 = '' + if ver2 is None: + ver2 = '' + c = cmp(cpy_CPV(fake_cat, fake_pkg, ver1, versioned=bool(ver1)), + cpy_CPV(fake_cat, fake_pkg, ver2, versioned=bool(ver2))) + if c != 0: + return c + return cmp(rev1, rev2) + + +def mk_cpv_cls(base_cls): + class CPV(base.base, base_cls): + + """ + base ebuild package class + + :ivar category: str category + :ivar package: str package + :ivar key: strkey (cat/pkg) + :ivar version: str version + :ivar revision: int revision + :ivar versioned_atom: atom matching this exact version + :ivar unversioned_atom: atom matching all versions of this package + :cvar _get_attr: mapping of attr:callable to generate attributes on the fly + """ + + __slots__ = () + + inject_richcmp_methods_from_cmp(locals()) +# __metaclass__ = WeakInstMeta + +# __inst_caching__ = True + + def __repr__(self): + return '<%s cpvstr=%s @%#8x>' % ( + self.__class__.__name__, getattr(self, 'cpvstr', None), + id(self)) + + # for some insane reason, py3k doesn't pick up the __hash__... add it + # manually. + __hash__ = base_cls.__hash__ + + @property + def versioned_atom(self): + return atom.atom("=%s" % self.cpvstr) + + @property + def unversioned_atom(self): + return atom.atom(self.key) + + @classmethod + def versioned(cls, *args): + return cls(versioned=True, *args) + + @classmethod + def unversioned(cls, *args): + return cls(versioned=False, *args) + + def __reduce__(self): + return (self.__class__, (self.cpvstr,), None, None, None) + + return CPV + +native_CPV = mk_cpv_cls(_native_CPV) + +try: + # No name in module + # pylint: disable-msg=E0611 + from pkgcore.ebuild._cpv import CPV as cpy_CPV + CPV_base = cpy_CPV + ver_cmp = cpy_ver_cmp + cpy_builtin = True + cpy_CPV = CPV = mk_cpv_cls(cpy_CPV) +except ImportError: + ver_cmp = native_ver_cmp + cpy_builtin = False + CPV = CPV_base = native_CPV + +versioned_CPV = CPV.versioned +unversioned_CPV = CPV.unversioned + +class versioned_CPV_cls(CPV): + + __slots__ = () + + def __init__(self, *args): + CPV.__init__(self, versioned=True, *args) From e6a1464be6868cea1a29b88eb8ce0585343b22d8 Mon Sep 17 00:00:00 2001 From: Pavlina Moravcova Varekova Date: Thu, 1 Mar 2018 12:14:33 +0100 Subject: [PATCH 010/160] Skip tests using Lua iff Lua is disabled (#264) --- tests/rpmvercmp.at | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/rpmvercmp.at b/tests/rpmvercmp.at index 2a25bdd9..8b32209a 100644 --- a/tests/rpmvercmp.at +++ b/tests/rpmvercmp.at @@ -3,7 +3,9 @@ m4_define([RPMVERCMP],[ AT_SETUP([rpmvercmp($1, $2) = $3]) AT_KEYWORDS([vercmp]) -AT_CHECK([runroot rpm --eval '%{lua: print(rpm.vercmp("$1", "$2"))}'], [0], [$3 +AT_CHECK([ +AT_SKIP_IF([$LUA_DISABLED]) +runroot rpm --eval '%{lua: print(rpm.vercmp("$1", "$2"))}'], [0], [$3 ], []) AT_CLEANUP ]) From 9f567e01012ddc09e853445c34f4dec21d4dbcf6 Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Wed, 11 Apr 2018 05:42:50 -0400 Subject: [PATCH 011/160] move tests to root dir --- tests/ebuild/test_cpv.py | 337 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 337 insertions(+) create mode 100644 tests/ebuild/test_cpv.py diff --git a/tests/ebuild/test_cpv.py b/tests/ebuild/test_cpv.py new file mode 100644 index 00000000..5f3cf233 --- /dev/null +++ b/tests/ebuild/test_cpv.py @@ -0,0 +1,337 @@ +# Copyright: 2006-2011 Brian Harring +# Copyright: 2006 Marien Zwart +# License: BSD/GPL2 + +from random import shuffle + +from snakeoil.compatibility import cmp +from snakeoil.test import TestCase, mk_cpy_loadable_testcase + +from pkgcore.ebuild import cpv + +def generate_misc_sufs(): + simple_good_sufs = ["_alpha", "_beta", "_pre", "_p"] + suf_nums = list(xrange(100)) + shuffle(suf_nums) + + good_sufs = (simple_good_sufs +["%s%i" % (x, suf_nums.pop()) + for x in simple_good_sufs]) + + l = len(good_sufs) + good_sufs = good_sufs + [ + good_sufs[x] + good_sufs[l - x - 1] for x in xrange(l)] + + bad_sufs = ["_a", "_9", "_"] + [x+" " for x in simple_good_sufs] + return good_sufs, bad_sufs + + +class native_CpvTest(TestCase): + + kls = staticmethod(cpv.native_CPV) + + @classmethod + def vkls(cls, *args): + return cls.kls(versioned=True, *args) + + def ukls(cls, *args): + return cls.kls(versioned=False, *args) + + run_cpy_ver_cmp = False + + good_cats = ( + "dev-util", "dev+", "dev-util+", "DEV-UTIL", "aaa0", + "aaa-0", "multi/depth", "cross-dev_idiot.hacks-suck", "a") + bad_cats = (".util", "_dev", "", "dev-util ", "multi//depth") + good_pkgs = ("diffball", "a9", "a9+", "a-100dpi", "diff-mode-") + bad_pkgs = ("diffball ", "diffball-9", "a-3D", "ab--df", "-df", "+dfa") + + good_cp = ( + "bbb-9/foon", "dev-util/diffball", "dev-util/diffball-a9", + "dev-ut-asdf/emacs-cvs", "xfce-base/xfce4", "bah/f-100dpi", + "dev-util/diffball-blah-monkeys") + + good_vers = ("1", "2.3.4", "2.3.4a", "02.3", "2.03", "3d", "3D") + bad_vers = ("2.3a.4", "2.a.3", "2.3_", "2.3 ", "2.3.", "cvs.2") + + good_sufs, bad_sufs = generate_misc_sufs() + good_revs = ("-r1", "-r300", "-r0", "", + "-r1000000000000000000") + bad_revs = ("-r", "-ra", "-r", "-R1") + + testing_secondary_args = False + + def make_inst(self, cat, pkg, fullver=""): + if self.testing_secondary_args: + return self.kls(cat, pkg, fullver, versioned=bool(fullver)) + if fullver: + return self.vkls("%s/%s-%s" % (cat, pkg, fullver)) + return self.ukls("%s/%s" % (cat, pkg)) + + def test_simple_key(self): + self.assertRaises(cpv.InvalidCPV, self.make_inst, "da", "ba-3", "3.3") + for src in [[("dev-util", "diffball", "0.7.1"), "dev-util/diffball"], + ["dev-util/diffball"], + ["dev-perl/mod_perl"], + ["dev-perl/mod_p"], + [("dev-perl", "mod-p", ""), "dev-perl/mod-p"], + ["dev-perl/mod-p-1", "dev-perl/mod-p"],]: + if len(src) == 1: + key = src[0] + else: + key = src[1] + if isinstance(src[0], basestring): + cat, pkgver = src[0].rsplit("/", 1) + vals = pkgver.rsplit("-", 1) + if len(vals) == 1: + pkg = pkgver + ver = '' + else: + pkg, ver = vals + else: + cat, pkg, ver = src[0] + + self.assertEqual(self.make_inst(cat, pkg, ver).key, key) + + def test_init(self): + self.kls("dev-util", "diffball", "0.7.1") + self.kls("dev-util/diffball-0.7.1", versioned=True) + self.assertRaises(TypeError, self.kls, "dev-util", "diffball") + self.assertRaises(TypeError, self.vkls, "dev-util", "diffball", None) + + def test_parsing(self): + # check for gentoo bug 263787 + self.process_pkg(False, 'app-text', 'foo-123-bar') + self.process_ver(False, 'app-text', 'foo-123-bar', '2.0017a_p', '-r5') + self.assertRaises(cpv.InvalidCPV, self.ukls, 'app-text/foo-123') + for cat_ret, cats in [[False, self.good_cats], [True, self.bad_cats]]: + for cat in cats: + for pkg_ret, pkgs in [[False, self.good_pkgs], + [True, self.bad_pkgs]]: + for pkg in pkgs: + self.process_pkg(cat_ret or pkg_ret, cat, pkg) + + for cp in self.good_cp: + cat, pkg = cp.rsplit("/", 1) + for rev_ret, revs in [[False, self.good_revs], + [True, self.bad_revs]]: + for rev in revs: + for ver_ret, vers in [[False, self.good_vers], + [True, self.bad_vers]]: + for ver in vers: + self.process_ver(ver_ret or rev_ret, cat, pkg, + ver, rev) + + for x in (10, 18, 19, 36, 100): + self.assertEqual(self.kls("da", "ba", "1-r0%s" % ("0" * x)).revision, + None) + self.assertEqual(long(self.kls("da", "ba", "1-r1%s1" % ("0" * x)).revision), + long("1%s1" % ("0" * x))) + + + def process_pkg(self, ret, cat, pkg): + if ret: + self.assertRaisesMsg("%s/%s" % (cat,pkg), cpv.InvalidCPV, + self.make_inst, cat, pkg) + else: + c = self.make_inst(cat, pkg) + self.assertEqual(c.cpvstr, "%s/%s" % (cat, pkg)) + self.assertEqual(c.category, cat) + self.assertEqual(c.package, pkg) + self.assertEqual(c.key, "%s/%s" % (cat, pkg)) + self.assertEqual(c.revision, None) + self.assertEqual(c.version, None) + self.assertEqual(c.fullver, None) + + def process_ver(self, ret, cat, pkg, ver, rev): + if ret: + self.assertRaisesMsg("%s/%s-%s%s" % (cat, pkg, ver, rev), + cpv.InvalidCPV, self.make_inst, + cat, pkg, "%s%s" % (ver, rev)) + else: + c = self.make_inst(cat, pkg, ver + rev) + if rev == "" or rev == "-r0": + self.assertEqual(c.cpvstr, "%s/%s-%s" % (cat, pkg, ver)) + self.assertEqual(c.revision, None) + self.assertEqual(c.fullver, ver) + else: + self.assertEqual(c.revision, int(rev.lstrip("-r"))) + self.assertEqual(c.cpvstr, "%s/%s-%s%s" % (cat, pkg, ver, rev)) + self.assertEqual(c.fullver, ver+rev) + self.assertEqual(c.category, cat) + self.assertEqual(c.package, pkg) + self.assertEqual(c.key, "%s/%s" % (cat, pkg)) + self.assertEqual(c.version, ver) + + for suf in self.good_sufs: + self.process_suf(ret, cat, pkg, ver + suf, rev) + for bad_suf in self.bad_sufs: + # double process, front and back. + self.process_suf(True, cat, pkg, suf + bad_suf, rev) + self.process_suf(True, cat, pkg, bad_suf + suf, rev) + + for suf in self.bad_sufs: + # check standalone. + self.process_suf(True, cat, pkg, ver+suf, rev) + + def process_suf(self, ret, cat, pkg, ver, rev): + if ret: + self.assertRaisesMsg("%s/%s-%s%s" % (cat, pkg, ver, rev), + cpv.InvalidCPV, self.make_inst, + cat, pkg, ver+rev) + else: + # redundant in light of process_ver... combine these somehow. + c = self.make_inst(cat, pkg, ver + rev) + if rev == '' or rev == '-r0': + self.assertEqual(c.cpvstr, "%s/%s-%s" % (cat, pkg, ver)) + self.assertEqual(c.revision, None) + self.assertEqual(c.fullver, ver) + else: + self.assertEqual(c.cpvstr, "%s/%s-%s%s" % (cat, pkg, ver, rev)) + self.assertEqual(c.revision, int(rev.lstrip("-r"))) + self.assertEqual(c.fullver, ver + rev) + self.assertEqual(c.category, cat) + self.assertEqual(c.package, pkg) + self.assertEqual(c.key, "%s/%s" % (cat, pkg)) + self.assertEqual(c.version, ver) + + def assertGT(self, obj1, obj2): + self.assertTrue(obj1 > obj2, '%r must be > %r' % (obj1, obj2)) + # swap the ordering, so that it's no longer obj1.__cmp__, but obj2s + self.assertTrue(obj2 < obj1, '%r must be < %r' % (obj2, obj1)) + + if self.run_cpy_ver_cmp and obj1.fullver and obj2.fullver: + self.assertTrue(cpv.cpy_ver_cmp(obj1.version, obj1.revision, + obj2.version, obj2.revision) > 0, + 'cpy_ver_cmp, %r > %r' % (obj1, obj2)) + self.assertTrue(cpv.cpy_ver_cmp(obj2.version, obj2.revision, + obj1.version, obj1.revision) < 0, + 'cpy_ver_cmp, %r < %r' % (obj2, obj1)) + + def test_cmp(self): + ukls, vkls = self.ukls, self.vkls + self.assertTrue( + cmp(vkls("dev-util/diffball-0.1"), + vkls("dev-util/diffball-0.2")) < 0) + base = "dev-util/diffball-0.7.1" + self.assertFalse(cmp(vkls(base), vkls(base))) + for rev in ("", "-r1"): + last = None + for suf in ["_alpha", "_beta", "_pre", "", "_p"]: + if suf == "": + sufs = [suf] + else: + sufs = [suf, suf+"4"] + for x in sufs: + cur = vkls(base+x+rev) + self.assertEqual(cur, vkls(base+x+rev)) + if last is not None: + self.assertGT(cur, last) + + self.assertGT(vkls("da/ba-6a"), vkls("da/ba-6")) + self.assertGT(vkls("da/ba-6a-r1"), vkls("da/ba-6a")) + self.assertGT(vkls("da/ba-6.0"), vkls("da/ba-6")) + self.assertGT(vkls("da/ba-6.0.0"), vkls("da/ba-6.0b")) + self.assertGT(vkls("da/ba-6.02"), vkls("da/ba-6.0.0")) + # float comparison rules. + self.assertGT(vkls("da/ba-6.2"), vkls("da/ba-6.054")) + self.assertEqual(vkls("da/ba-6"), vkls("da/ba-6")) + self.assertGT(ukls("db/ba"), ukls("da/ba")) + self.assertGT(ukls("da/bb"), ukls("da/ba")) + self.assertGT(vkls("da/ba-6.0_alpha0_p1"), vkls("da/ba-6.0_alpha")) + self.assertEqual(vkls("da/ba-6.0_alpha"), vkls("da/ba-6.0_alpha0")) + self.assertGT(vkls("da/ba-6.1"), vkls("da/ba-6.09")) + self.assertGT(vkls("da/ba-6.0.1"), vkls("da/ba-6.0")) + self.assertGT(vkls("da/ba-12.2.5"), vkls("da/ba-12.2b")) + + # test for gentoo bug 287848 + self.assertGT(vkls("dev-lang/erlang-12.2.5"), + vkls("dev-lang/erlang-12.2b")) + self.assertGT(vkls("dev-lang/erlang-12.2.5-r1"), + vkls("dev-lang/erlang-12.2b")) + + self.assertEqual(vkls("da/ba-6.01.0"), vkls("da/ba-6.010.0")) + + for v1, v2 in (("1.001000000000000000001", "1.001000000000000000002"), + ("1.00100000000", "1.0010000000000000001"), + ("1.01", "1.1")): + self.assertGT(vkls("da/ba-%s" % v2), vkls("da/ba-%s" % v1)) + + for x in (18, 36, 100): + s = "0" * x + self.assertGT(vkls("da/ba-10%s1" % s), vkls("da/ba-1%s1" % s)) + + for x in (18, 36, 100): + s = "0" * x + self.assertGT(vkls("da/ba-1-r10%s1" % s), + vkls("da/ba-1-r1%s1" % s)) + + self.assertGT(vkls('sys-apps/net-tools-1.60_p2010081516093'), + vkls('sys-apps/net-tools-1.60_p2009072801401')) + + self.assertGT(vkls('sys-apps/net-tools-1.60_p20100815160931'), + vkls('sys-apps/net-tools-1.60_p20090728014017')) + + self.assertGT(vkls('sys-apps/net-tools-1.60_p20100815160931'), + vkls('sys-apps/net-tools-1.60_p20090728014017-r1')) + + # Regression test: python does comparison slightly differently + # if the classes do not match exactly (it prefers rich + # comparison over __cmp__). + class DummySubclass(self.kls): + pass + self.assertNotEqual( + DummySubclass("da/ba-6.0_alpha0_p1", versioned=True), + vkls("da/ba-6.0_alpha")) + self.assertEqual( + DummySubclass("da/ba-6.0_alpha0", versioned=True), + vkls("da/ba-6.0_alpha")) + + self.assertNotEqual(DummySubclass("da/ba-6.0", versioned=True), + "foon") + self.assertEqual(DummySubclass("da/ba-6.0", versioned=True), + DummySubclass("da/ba-6.0-r0", versioned=True)) + + def test_no_init(self): + """Test if the cpv is in a somewhat sane state if __init__ fails. + + IPython used to segfault when showing a verbose traceback for + a subclass of CPV which raised cpv.InvalidCPV. This checks + if such uninitialized objects survive some basic poking. + """ + uninited = self.kls.__new__(self.kls) + broken = self.kls.__new__(self.kls) + self.assertRaises(cpv.InvalidCPV, broken.__init__, 'broken', versioned=True) + for thing in (uninited, broken): + # the c version returns None, the py version does not have the attr + getattr(thing, 'cpvstr', None) + repr(thing) + str(thing) + # The c version returns a constant, the py version raises + try: + hash(thing) + except AttributeError: + pass + + def test_r0_removal(self): + obj = self.kls("dev-util/diffball-1.0-r0", versioned=True) + self.assertEqual(obj.fullver, "1.0") + self.assertEqual(obj.revision, None) + self.assertEqual(str(obj), "dev-util/diffball-1.0") + + +class CPY_CpvTest(native_CpvTest): + if cpv.cpy_builtin: + kls = staticmethod(cpv.cpy_CPV) + else: + skip = "cpython cpv extension not available" + + run_cpy_ver_cmp = True + + +class CPY_Cpv_OptionalArgsTest(CPY_CpvTest): + + testing_secondary_args = True + +test_cpy_used = mk_cpy_loadable_testcase('pkgcore.ebuild._cpv', + "pkgcore.ebuild.cpv", "CPV_base", "CPV") + From 39eb0be726715fc4c5a5cba834d382e8cd04a79b Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Wed, 11 Apr 2018 10:20:57 -0400 Subject: [PATCH 012/160] more py3 conversions --- src/pkgcore/ebuild/cpv.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index a8bb899c..67320dfe 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -7,8 +7,6 @@ __all__ = ("CPV", "versioned_CPV", "unversioned_CPV") -from itertools import izip - from snakeoil.compatibility import cmp from snakeoil.demandload import demandload, demand_compile_regexp from snakeoil.klass import inject_richcmp_methods_from_cmp @@ -97,14 +95,14 @@ def __init__(self, *a, **kwds): % (cpvstr,)) elif l == 3: for x in a: - if not isinstance(x, basestring): + if not isinstance(x, str): raise TypeError("all args must be strings, got %r" % (a,)) cpvstr = "%s/%s-%s" % a versioned = True else: raise TypeError("CPV takes 1 arg (cpvstr), or 3 (cat, pkg, ver):" " got %r" % (a,)) - if not isinstance(cpvstr, basestring): + if not isinstance(cpvstr, str): raise TypeError(self.cpvstr) try: @@ -222,7 +220,7 @@ def native_ver_cmp(ver1, rev1, ver2, rev2): ver_parts2_len = len(ver_parts2) # Iterate through the components - for v1, v2 in izip(ver_parts1, ver_parts2): + for v1, v2 in zip(ver_parts1, ver_parts2): # If the string components are equal, the numerical # components will be equal too. @@ -264,7 +262,7 @@ def native_ver_cmp(ver1, rev1, ver2, rev2): parts2_len = len(parts2) # Iterate through the suffixes - for x in xrange(max(parts1_len, parts2_len)): + for x in range(max(parts1_len, parts2_len)): # If we're at the end of one of our lists, we need to use # the next suffix from the other list to decide who wins. From bcc63ec1c6d1cdeea06ed0070009d2aeeb0ffc9d Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Mon, 11 Jun 2018 17:34:42 -0400 Subject: [PATCH 013/160] migrate tests to py3 --- tests/ebuild/test_cpv.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/ebuild/test_cpv.py b/tests/ebuild/test_cpv.py index 5f3cf233..c498ea3d 100644 --- a/tests/ebuild/test_cpv.py +++ b/tests/ebuild/test_cpv.py @@ -11,7 +11,7 @@ def generate_misc_sufs(): simple_good_sufs = ["_alpha", "_beta", "_pre", "_p"] - suf_nums = list(xrange(100)) + suf_nums = list(range(100)) shuffle(suf_nums) good_sufs = (simple_good_sufs +["%s%i" % (x, suf_nums.pop()) @@ -19,7 +19,7 @@ def generate_misc_sufs(): l = len(good_sufs) good_sufs = good_sufs + [ - good_sufs[x] + good_sufs[l - x - 1] for x in xrange(l)] + good_sufs[x] + good_sufs[l - x - 1] for x in range(l)] bad_sufs = ["_a", "_9", "_"] + [x+" " for x in simple_good_sufs] return good_sufs, bad_sufs @@ -79,7 +79,7 @@ def test_simple_key(self): key = src[0] else: key = src[1] - if isinstance(src[0], basestring): + if isinstance(src[0], str): cat, pkgver = src[0].rsplit("/", 1) vals = pkgver.rsplit("-", 1) if len(vals) == 1: @@ -124,8 +124,8 @@ def test_parsing(self): for x in (10, 18, 19, 36, 100): self.assertEqual(self.kls("da", "ba", "1-r0%s" % ("0" * x)).revision, None) - self.assertEqual(long(self.kls("da", "ba", "1-r1%s1" % ("0" * x)).revision), - long("1%s1" % ("0" * x))) + self.assertEqual(int(self.kls("da", "ba", "1-r1%s1" % ("0" * x)).revision), + int("1%s1" % ("0" * x))) def process_pkg(self, ret, cat, pkg): From 8eef0246f95034d09d2c8fdf846f93ceca401252 Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Sat, 22 Sep 2018 20:35:11 -0400 Subject: [PATCH 014/160] various f-strings conversions --- src/pkgcore/ebuild/cpv.py | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index 67320dfe..38ecc9cc 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -86,22 +86,20 @@ def __init__(self, *a, **kwds): if had_versioned: versioned = kwds.pop("versioned") if kwds: - raise TypeError("versioned is the only allowed kwds: %r" % (kwds,)) + raise TypeError(f"versioned is the only allowed kwds: {kwds!r}") l = len(a) if l == 1: cpvstr = a[0] if not had_versioned: - raise TypeError("single arguement invocation requires versioned kwd; %r" - % (cpvstr,)) + raise TypeError(f"single argument invocation requires versioned kwd; {cpvstr!r}") elif l == 3: for x in a: if not isinstance(x, str): - raise TypeError("all args must be strings, got %r" % (a,)) - cpvstr = "%s/%s-%s" % a + raise TypeError(f"all args must be strings, got {a!r}") + cpvstr = f"{a[0]}/{a[1]}-{a[2]}" versioned = True else: - raise TypeError("CPV takes 1 arg (cpvstr), or 3 (cat, pkg, ver):" - " got %r" % (a,)) + raise TypeError(f"CPV takes 1 arg (cpvstr), or 3 (cat, pkg, ver): got {a!r}") if not isinstance(cpvstr, str): raise TypeError(self.cpvstr) @@ -109,9 +107,9 @@ def __init__(self, *a, **kwds): categories, pkgver = cpvstr.rsplit("/", 1) except ValueError: # occurs if the rsplit yields only one item - raise InvalidCPV('%s: no package or version components' % cpvstr) + raise InvalidCPV(f'{cpvstr}: no package or version components') if not isvalid_cat_re.match(categories): - raise InvalidCPV('%s: invalid category name' % cpvstr) + raise InvalidCPV(f'{cpvstr}: invalid category name') sf = object.__setattr__ sf(self, 'category', categories) sf(self, 'cpvstr', cpvstr) @@ -119,36 +117,35 @@ def __init__(self, *a, **kwds): lpkg_chunks = len(pkg_chunks) if versioned: if lpkg_chunks == 1: - raise InvalidCPV('%s: missing package version' % cpvstr) + raise InvalidCPV(f'{cpvstr}: missing package version') if isvalid_rev(pkg_chunks[-1]): if lpkg_chunks < 3: # needs at least ('pkg', 'ver', 'rev') - raise InvalidCPV('%s: missing package name, version, and/or revision' % cpvstr) + raise InvalidCPV(f'{cpvstr}: missing package name, version, and/or revision') rev = int(pkg_chunks.pop(-1)[1:]) if not rev: rev = None # reset the stored cpvstr to drop -r0+ - sf(self, 'cpvstr', "%s/%s" % (categories, - '-'.join(pkg_chunks))) + sf(self, 'cpvstr', f"{categories}/{'-'.join(pkg_chunks)}") sf(self, 'revision', rev) else: sf(self, 'revision', None) if not isvalid_version_re.match(pkg_chunks[-1]): - raise InvalidCPV("%s: invalid version '%s'" % (cpvstr, pkg_chunks[-1])) + raise InvalidCPV(f"{cpvstr}: invalid version '{pkg_chunks[-1]}'") sf(self, 'version', pkg_chunks.pop(-1)) if self.revision: - sf(self, 'fullver', "%s-r%s" % (self.version, self.revision)) + sf(self, 'fullver', f"{self.version}-r{self.revision}") else: sf(self, 'fullver', self.version) if not isvalid_pkg_name(pkg_chunks): - raise InvalidCPV('%s: invalid package name' % cpvstr) + raise InvalidCPV(f'{cpvstr}: invalid package name') sf(self, 'package', '-'.join(pkg_chunks)) - sf(self, 'key', "%s/%s" % (categories, self.package)) + sf(self, 'key', f"{categories}/{self.package}") else: if not isvalid_pkg_name(pkg_chunks): - raise InvalidCPV('%s: invalid package name' % cpvstr) + raise InvalidCPV(f'{cpvstr}: invalid package name') sf(self, 'revision', None) sf(self, 'fullver', None) sf(self, 'version', None) @@ -354,7 +351,7 @@ def __repr__(self): @property def versioned_atom(self): - return atom.atom("=%s" % self.cpvstr) + return atom.atom(f"={self.cpvstr}") @property def unversioned_atom(self): From 1a0c4858ce79615668bfecf03493dc74ecf57dd1 Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Mon, 1 Oct 2018 00:13:48 -0400 Subject: [PATCH 015/160] ebuild.cpv: simplify defining ver_cmp() in the cpy extension case --- src/pkgcore/ebuild/cpv.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index 38ecc9cc..189d6fbf 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -301,21 +301,6 @@ def native_ver_cmp(ver1, rev1, ver2, rev2): # The revision holds the final difference. return cmp(rev1, rev2) -fake_cat = "fake" -fake_pkg = "pkg" -def cpy_ver_cmp(ver1, rev1, ver2, rev2): - if ver1 == ver2: - return cmp(rev1, rev2) - if ver1 is None: - ver1 = '' - if ver2 is None: - ver2 = '' - c = cmp(cpy_CPV(fake_cat, fake_pkg, ver1, versioned=bool(ver1)), - cpy_CPV(fake_cat, fake_pkg, ver2, versioned=bool(ver2))) - if c != 0: - return c - return cmp(rev1, rev2) - def mk_cpv_cls(base_cls): class CPV(base.base, base_cls): @@ -376,8 +361,21 @@ def __reduce__(self): # No name in module # pylint: disable-msg=E0611 from pkgcore.ebuild._cpv import CPV as cpy_CPV + + def ver_cmp(ver1, rev1, ver2, rev2): + if ver1 == ver2: + return cmp(rev1, rev2) + if ver1 is None: + ver1 = '' + if ver2 is None: + ver2 = '' + c = cmp(cpy_CPV("fake", "pkg", ver1, versioned=bool(ver1)), + cpy_CPV("fake", "pkg", ver2, versioned=bool(ver2))) + if c != 0: + return c + return cmp(rev1, rev2) + CPV_base = cpy_CPV - ver_cmp = cpy_ver_cmp cpy_builtin = True cpy_CPV = CPV = mk_cpv_cls(cpy_CPV) except ImportError: From daac49e123f82c8daf435e897b568edfe13f9d7e Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Mon, 1 Oct 2018 00:14:39 -0400 Subject: [PATCH 016/160] tests/ebuild/test_cpv: update to use pytest --- tests/ebuild/test_cpv.py | 225 +++++++++++++++++++-------------------- 1 file changed, 108 insertions(+), 117 deletions(-) diff --git a/tests/ebuild/test_cpv.py b/tests/ebuild/test_cpv.py index c498ea3d..2dae0690 100644 --- a/tests/ebuild/test_cpv.py +++ b/tests/ebuild/test_cpv.py @@ -4,8 +4,9 @@ from random import shuffle +import pytest from snakeoil.compatibility import cmp -from snakeoil.test import TestCase, mk_cpy_loadable_testcase +from snakeoil.test import mk_cpy_loadable_testcase from pkgcore.ebuild import cpv @@ -14,8 +15,7 @@ def generate_misc_sufs(): suf_nums = list(range(100)) shuffle(suf_nums) - good_sufs = (simple_good_sufs +["%s%i" % (x, suf_nums.pop()) - for x in simple_good_sufs]) + good_sufs = (simple_good_sufs + [f"{x}{suf_nums.pop()}" for x in simple_good_sufs]) l = len(good_sufs) good_sufs = good_sufs + [ @@ -25,7 +25,7 @@ def generate_misc_sufs(): return good_sufs, bad_sufs -class native_CpvTest(TestCase): +class Test_native_Cpv(object): kls = staticmethod(cpv.native_CPV) @@ -64,11 +64,12 @@ def make_inst(self, cat, pkg, fullver=""): if self.testing_secondary_args: return self.kls(cat, pkg, fullver, versioned=bool(fullver)) if fullver: - return self.vkls("%s/%s-%s" % (cat, pkg, fullver)) - return self.ukls("%s/%s" % (cat, pkg)) + return self.vkls(f"{cat}/{pkg}-{fullver}") + return self.ukls(f"{cat}/{pkg}") def test_simple_key(self): - self.assertRaises(cpv.InvalidCPV, self.make_inst, "da", "ba-3", "3.3") + with pytest.raises(cpv.InvalidCPV): + self.make_inst("da", "ba-3", "3.3") for src in [[("dev-util", "diffball", "0.7.1"), "dev-util/diffball"], ["dev-util/diffball"], ["dev-perl/mod_perl"], @@ -90,19 +91,22 @@ def test_simple_key(self): else: cat, pkg, ver = src[0] - self.assertEqual(self.make_inst(cat, pkg, ver).key, key) + assert self.make_inst(cat, pkg, ver).key == key def test_init(self): self.kls("dev-util", "diffball", "0.7.1") self.kls("dev-util/diffball-0.7.1", versioned=True) - self.assertRaises(TypeError, self.kls, "dev-util", "diffball") - self.assertRaises(TypeError, self.vkls, "dev-util", "diffball", None) + with pytest.raises(TypeError): + self.kls("dev-util", "diffball") + with pytest.raises(TypeError): + self.vkls("dev-util", "diffball", None) def test_parsing(self): # check for gentoo bug 263787 self.process_pkg(False, 'app-text', 'foo-123-bar') self.process_ver(False, 'app-text', 'foo-123-bar', '2.0017a_p', '-r5') - self.assertRaises(cpv.InvalidCPV, self.ukls, 'app-text/foo-123') + with pytest.raises(cpv.InvalidCPV): + self.ukls('app-text/foo-123') for cat_ret, cats in [[False, self.good_cats], [True, self.bad_cats]]: for cat in cats: for pkg_ret, pkgs in [[False, self.good_pkgs], @@ -122,45 +126,42 @@ def test_parsing(self): ver, rev) for x in (10, 18, 19, 36, 100): - self.assertEqual(self.kls("da", "ba", "1-r0%s" % ("0" * x)).revision, - None) - self.assertEqual(int(self.kls("da", "ba", "1-r1%s1" % ("0" * x)).revision), - int("1%s1" % ("0" * x))) - + assert self.kls("da", "ba", f"1-r0{'0' * x}").revision is None + assert \ + int(self.kls("da", "ba", f"1-r1{'0' * x}1").revision) == int(f"1{'0' * x}1") def process_pkg(self, ret, cat, pkg): if ret: - self.assertRaisesMsg("%s/%s" % (cat,pkg), cpv.InvalidCPV, - self.make_inst, cat, pkg) + with pytest.raises(cpv.InvalidCPV): + self.make_inst(cat, pkg) else: c = self.make_inst(cat, pkg) - self.assertEqual(c.cpvstr, "%s/%s" % (cat, pkg)) - self.assertEqual(c.category, cat) - self.assertEqual(c.package, pkg) - self.assertEqual(c.key, "%s/%s" % (cat, pkg)) - self.assertEqual(c.revision, None) - self.assertEqual(c.version, None) - self.assertEqual(c.fullver, None) + assert c.cpvstr == f"{cat}/{pkg}" + assert c.category == cat + assert c.package == pkg + assert c.key == f"{cat}/{pkg}" + assert c.revision is None + assert c.version is None + assert c.fullver is None def process_ver(self, ret, cat, pkg, ver, rev): if ret: - self.assertRaisesMsg("%s/%s-%s%s" % (cat, pkg, ver, rev), - cpv.InvalidCPV, self.make_inst, - cat, pkg, "%s%s" % (ver, rev)) + with pytest.raises(cpv.InvalidCPV): + self.make_inst(cat, pkg, f"{ver}{rev}") else: c = self.make_inst(cat, pkg, ver + rev) if rev == "" or rev == "-r0": - self.assertEqual(c.cpvstr, "%s/%s-%s" % (cat, pkg, ver)) - self.assertEqual(c.revision, None) - self.assertEqual(c.fullver, ver) + assert c.cpvstr == f"{cat}/{pkg}-{ver}" + assert c.revision is None + assert c.fullver == ver else: - self.assertEqual(c.revision, int(rev.lstrip("-r"))) - self.assertEqual(c.cpvstr, "%s/%s-%s%s" % (cat, pkg, ver, rev)) - self.assertEqual(c.fullver, ver+rev) - self.assertEqual(c.category, cat) - self.assertEqual(c.package, pkg) - self.assertEqual(c.key, "%s/%s" % (cat, pkg)) - self.assertEqual(c.version, ver) + assert c.revision == int(rev.lstrip("-r")) + assert c.cpvstr == f"{cat}/{pkg}-{ver}{rev}" + assert c.fullver == ver+rev + assert c.category == cat + assert c.package == pkg + assert c.key == f"{cat}/{pkg}" + assert c.version == ver for suf in self.good_sufs: self.process_suf(ret, cat, pkg, ver + suf, rev) @@ -175,45 +176,42 @@ def process_ver(self, ret, cat, pkg, ver, rev): def process_suf(self, ret, cat, pkg, ver, rev): if ret: - self.assertRaisesMsg("%s/%s-%s%s" % (cat, pkg, ver, rev), - cpv.InvalidCPV, self.make_inst, - cat, pkg, ver+rev) + with pytest.raises(cpv.InvalidCPV): + self.make_inst(cat, pkg, ver+rev) else: # redundant in light of process_ver... combine these somehow. c = self.make_inst(cat, pkg, ver + rev) if rev == '' or rev == '-r0': - self.assertEqual(c.cpvstr, "%s/%s-%s" % (cat, pkg, ver)) - self.assertEqual(c.revision, None) - self.assertEqual(c.fullver, ver) + assert c.cpvstr == f"{cat}/{pkg}-{ver}" + assert c.revision is None + assert c.fullver == ver else: - self.assertEqual(c.cpvstr, "%s/%s-%s%s" % (cat, pkg, ver, rev)) - self.assertEqual(c.revision, int(rev.lstrip("-r"))) - self.assertEqual(c.fullver, ver + rev) - self.assertEqual(c.category, cat) - self.assertEqual(c.package, pkg) - self.assertEqual(c.key, "%s/%s" % (cat, pkg)) - self.assertEqual(c.version, ver) + assert c.cpvstr == f"{cat}/{pkg}-{ver}{rev}" + assert c.revision == int(rev.lstrip("-r")) + assert c.fullver == ver + rev + assert c.category == cat + assert c.package == pkg + assert c.key == f"{cat}/{pkg}" + assert c.version == ver def assertGT(self, obj1, obj2): - self.assertTrue(obj1 > obj2, '%r must be > %r' % (obj1, obj2)) + assert obj1 > obj2, f'{obj1!r} must be > {obj2!r}' # swap the ordering, so that it's no longer obj1.__cmp__, but obj2s - self.assertTrue(obj2 < obj1, '%r must be < %r' % (obj2, obj1)) + assert obj2 < obj1, f'{obj2!r} must be < {obj1!r}' if self.run_cpy_ver_cmp and obj1.fullver and obj2.fullver: - self.assertTrue(cpv.cpy_ver_cmp(obj1.version, obj1.revision, - obj2.version, obj2.revision) > 0, - 'cpy_ver_cmp, %r > %r' % (obj1, obj2)) - self.assertTrue(cpv.cpy_ver_cmp(obj2.version, obj2.revision, - obj1.version, obj1.revision) < 0, - 'cpy_ver_cmp, %r < %r' % (obj2, obj1)) + assert cpv.cpy_ver_cmp( + obj1.version, obj1.revision, obj2.version, obj2.revision) > 0, \ + f'cpy_ver_cmp, {obj1!r} > {obj2!r}' + assert cpv.cpy_ver_cmp( + obj2.version, obj2.revision, obj1.version, obj1.revision) < 0, \ + f'cpy_ver_cmp, {obj2!r} < {obj1!r}' def test_cmp(self): ukls, vkls = self.ukls, self.vkls - self.assertTrue( - cmp(vkls("dev-util/diffball-0.1"), - vkls("dev-util/diffball-0.2")) < 0) + assert cmp(vkls("dev-util/diffball-0.1"), vkls("dev-util/diffball-0.2")) < 0 base = "dev-util/diffball-0.7.1" - self.assertFalse(cmp(vkls(base), vkls(base))) + assert not cmp(vkls(base), vkls(base)) for rev in ("", "-r1"): last = None for suf in ["_alpha", "_beta", "_pre", "", "_p"]: @@ -223,73 +221,66 @@ def test_cmp(self): sufs = [suf, suf+"4"] for x in sufs: cur = vkls(base+x+rev) - self.assertEqual(cur, vkls(base+x+rev)) + assert cur == vkls(base+x+rev) if last is not None: - self.assertGT(cur, last) + assert cur > last - self.assertGT(vkls("da/ba-6a"), vkls("da/ba-6")) - self.assertGT(vkls("da/ba-6a-r1"), vkls("da/ba-6a")) - self.assertGT(vkls("da/ba-6.0"), vkls("da/ba-6")) - self.assertGT(vkls("da/ba-6.0.0"), vkls("da/ba-6.0b")) - self.assertGT(vkls("da/ba-6.02"), vkls("da/ba-6.0.0")) + assert vkls("da/ba-6a") > vkls("da/ba-6") + assert vkls("da/ba-6a-r1") > vkls("da/ba-6a") + assert vkls("da/ba-6.0") > vkls("da/ba-6") + assert vkls("da/ba-6.0.0") > vkls("da/ba-6.0b") + assert vkls("da/ba-6.02") > vkls("da/ba-6.0.0") # float comparison rules. - self.assertGT(vkls("da/ba-6.2"), vkls("da/ba-6.054")) - self.assertEqual(vkls("da/ba-6"), vkls("da/ba-6")) - self.assertGT(ukls("db/ba"), ukls("da/ba")) - self.assertGT(ukls("da/bb"), ukls("da/ba")) - self.assertGT(vkls("da/ba-6.0_alpha0_p1"), vkls("da/ba-6.0_alpha")) - self.assertEqual(vkls("da/ba-6.0_alpha"), vkls("da/ba-6.0_alpha0")) - self.assertGT(vkls("da/ba-6.1"), vkls("da/ba-6.09")) - self.assertGT(vkls("da/ba-6.0.1"), vkls("da/ba-6.0")) - self.assertGT(vkls("da/ba-12.2.5"), vkls("da/ba-12.2b")) + assert vkls("da/ba-6.2") > vkls("da/ba-6.054") + assert vkls("da/ba-6") == vkls("da/ba-6") + assert ukls("db/ba") > ukls("da/ba") + assert ukls("da/bb") > ukls("da/ba") + assert vkls("da/ba-6.0_alpha0_p1") > vkls("da/ba-6.0_alpha") + assert vkls("da/ba-6.0_alpha") == vkls("da/ba-6.0_alpha0") + assert vkls("da/ba-6.1") > vkls("da/ba-6.09") + assert vkls("da/ba-6.0.1") > vkls("da/ba-6.0") + assert vkls("da/ba-12.2.5") > vkls("da/ba-12.2b") # test for gentoo bug 287848 - self.assertGT(vkls("dev-lang/erlang-12.2.5"), - vkls("dev-lang/erlang-12.2b")) - self.assertGT(vkls("dev-lang/erlang-12.2.5-r1"), - vkls("dev-lang/erlang-12.2b")) + assert vkls("dev-lang/erlang-12.2.5") > vkls("dev-lang/erlang-12.2b") + assert vkls("dev-lang/erlang-12.2.5-r1") > vkls("dev-lang/erlang-12.2b") - self.assertEqual(vkls("da/ba-6.01.0"), vkls("da/ba-6.010.0")) + assert vkls("da/ba-6.01.0") == vkls("da/ba-6.010.0") for v1, v2 in (("1.001000000000000000001", "1.001000000000000000002"), ("1.00100000000", "1.0010000000000000001"), ("1.01", "1.1")): - self.assertGT(vkls("da/ba-%s" % v2), vkls("da/ba-%s" % v1)) + assert vkls(f"da/ba-{v2}") > vkls(f"da/ba-{v1}") for x in (18, 36, 100): s = "0" * x - self.assertGT(vkls("da/ba-10%s1" % s), vkls("da/ba-1%s1" % s)) + assert vkls(f"da/ba-10{s}1") > vkls(f"da/ba-1{s}1") for x in (18, 36, 100): s = "0" * x - self.assertGT(vkls("da/ba-1-r10%s1" % s), - vkls("da/ba-1-r1%s1" % s)) + assert vkls(f"da/ba-1-r10{s}1") > vkls(f"da/ba-1-r1{s}1") - self.assertGT(vkls('sys-apps/net-tools-1.60_p2010081516093'), - vkls('sys-apps/net-tools-1.60_p2009072801401')) + assert vkls('sys-apps/net-tools-1.60_p2010081516093') > \ + vkls('sys-apps/net-tools-1.60_p2009072801401') - self.assertGT(vkls('sys-apps/net-tools-1.60_p20100815160931'), - vkls('sys-apps/net-tools-1.60_p20090728014017')) + assert vkls('sys-apps/net-tools-1.60_p20100815160931') > \ + vkls('sys-apps/net-tools-1.60_p20090728014017') - self.assertGT(vkls('sys-apps/net-tools-1.60_p20100815160931'), - vkls('sys-apps/net-tools-1.60_p20090728014017-r1')) + assert vkls('sys-apps/net-tools-1.60_p20100815160931') > \ + vkls('sys-apps/net-tools-1.60_p20090728014017-r1') # Regression test: python does comparison slightly differently # if the classes do not match exactly (it prefers rich # comparison over __cmp__). class DummySubclass(self.kls): pass - self.assertNotEqual( - DummySubclass("da/ba-6.0_alpha0_p1", versioned=True), - vkls("da/ba-6.0_alpha")) - self.assertEqual( - DummySubclass("da/ba-6.0_alpha0", versioned=True), - vkls("da/ba-6.0_alpha")) - - self.assertNotEqual(DummySubclass("da/ba-6.0", versioned=True), - "foon") - self.assertEqual(DummySubclass("da/ba-6.0", versioned=True), - DummySubclass("da/ba-6.0-r0", versioned=True)) + + assert DummySubclass("da/ba-6.0_alpha0_p1", versioned=True) != vkls("da/ba-6.0_alpha") + assert DummySubclass("da/ba-6.0_alpha0", versioned=True) == vkls("da/ba-6.0_alpha") + + assert DummySubclass("da/ba-6.0", versioned=True) != "foon" + assert DummySubclass("da/ba-6.0", versioned=True) == \ + DummySubclass("da/ba-6.0-r0", versioned=True) def test_no_init(self): """Test if the cpv is in a somewhat sane state if __init__ fails. @@ -300,7 +291,8 @@ def test_no_init(self): """ uninited = self.kls.__new__(self.kls) broken = self.kls.__new__(self.kls) - self.assertRaises(cpv.InvalidCPV, broken.__init__, 'broken', versioned=True) + with pytest.raises(cpv.InvalidCPV): + broken.__init__('broken', versioned=True) for thing in (uninited, broken): # the c version returns None, the py version does not have the attr getattr(thing, 'cpvstr', None) @@ -314,24 +306,23 @@ def test_no_init(self): def test_r0_removal(self): obj = self.kls("dev-util/diffball-1.0-r0", versioned=True) - self.assertEqual(obj.fullver, "1.0") - self.assertEqual(obj.revision, None) - self.assertEqual(str(obj), "dev-util/diffball-1.0") + assert obj.fullver == "1.0" + assert obj.revision is None + assert str(obj) == "dev-util/diffball-1.0" -class CPY_CpvTest(native_CpvTest): +@pytest.mark.skipif(not cpv.cpy_builtin, reason="cpython cpv extension not available") +class Test_CPY_Cpv(Test_native_Cpv): if cpv.cpy_builtin: kls = staticmethod(cpv.cpy_CPV) - else: - skip = "cpython cpv extension not available" + run_cpy_ver_cmp = True - run_cpy_ver_cmp = True - -class CPY_Cpv_OptionalArgsTest(CPY_CpvTest): +class Test_CPY_Cpv_OptionalArgs(Test_CPY_Cpv): testing_secondary_args = True -test_cpy_used = mk_cpy_loadable_testcase('pkgcore.ebuild._cpv', - "pkgcore.ebuild.cpv", "CPV_base", "CPV") + +test_cpy_used = mk_cpy_loadable_testcase( + "pkgcore.ebuild._cpv", "pkgcore.ebuild.cpv", "CPV_base", "CPV") From 5235071cf136ee8b49c0914ecf395ca6bd1c617c Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Tue, 2 Oct 2018 12:55:48 -0400 Subject: [PATCH 017/160] remove/fix old comments --- src/pkgcore/ebuild/cpv.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index 189d6fbf..52002ef7 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -321,9 +321,6 @@ class CPV(base.base, base_cls): __slots__ = () inject_richcmp_methods_from_cmp(locals()) -# __metaclass__ = WeakInstMeta - -# __inst_caching__ = True def __repr__(self): return '<%s cpvstr=%s @%#8x>' % ( From 42197aaf747f87c567e962f4afda1db9ff3ffa4c Mon Sep 17 00:00:00 2001 From: Igor Gnatenko Date: Sat, 10 Sep 2016 11:39:23 +0200 Subject: [PATCH 018/160] Add support for sorting caret ('^') higher than base version 1.1^20160101 means 1.1 version (base) and patches which were applied at that date on top of it. * 1.1^201601 > 1.1 * 1.1^201601 < 1.1.1 Having symmetry is also good. Signed-off-by: Igor Gnatenko --- tests/rpmvercmp.at | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/rpmvercmp.at b/tests/rpmvercmp.at index 8b32209a..1e7c960e 100644 --- a/tests/rpmvercmp.at +++ b/tests/rpmvercmp.at @@ -102,6 +102,32 @@ RPMVERCMP(1.0~rc1~git123, 1.0~rc1~git123, 0) RPMVERCMP(1.0~rc1~git123, 1.0~rc1, -1) RPMVERCMP(1.0~rc1, 1.0~rc1~git123, 1) +dnl Basic testcases for caret sorting +RPMVERCMP(1.0^, 1.0^, 0) +RPMVERCMP(1.0^, 1.0, 1) +RPMVERCMP(1.0, 1.0^, -1) +RPMVERCMP(1.0^git1, 1.0^git1, 0) +RPMVERCMP(1.0^git1, 1.0, 1) +RPMVERCMP(1.0, 1.0^git1, -1) +RPMVERCMP(1.0^git1, 1.0^git2, -1) +RPMVERCMP(1.0^git2, 1.0^git1, 1) +RPMVERCMP(1.0^git1, 1.01, -1) +RPMVERCMP(1.01, 1.0^git1, 1) +RPMVERCMP(1.0^20160101, 1.0^20160101, 0) +RPMVERCMP(1.0^20160101, 1.0.1, -1) +RPMVERCMP(1.0.1, 1.0^20160101, 1) +RPMVERCMP(1.0^20160101^git1, 1.0^20160101^git1, 0) +RPMVERCMP(1.0^20160102, 1.0^20160101^git1, 1) +RPMVERCMP(1.0^20160101^git1, 1.0^20160102, -1) + +dnl Basic testcases for tilde and caret sorting +RPMVERCMP(1.0~rc1^git1, 1.0~rc1^git1, 0) +RPMVERCMP(1.0~rc1^git1, 1.0~rc1, 1) +RPMVERCMP(1.0~rc1, 1.0~rc1^git1, -1) +RPMVERCMP(1.0^git1~pre, 1.0^git1~pre, 0) +RPMVERCMP(1.0^git1, 1.0^git1~pre, 1) +RPMVERCMP(1.0^git1~pre, 1.0^git1, -1) + dnl These are included here to document current, arguably buggy behaviors dnl for reference purposes and for easy checking against unintended dnl behavior changes. From 733a9d3249ed51ca9c24ff389d5fbac796dbcd2a Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Wed, 9 Jan 2019 21:28:31 -0600 Subject: [PATCH 019/160] misc f-string and consistency work --- src/pkgcore/ebuild/cpv.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index 52002ef7..1fa5acf2 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -53,9 +53,7 @@ def isvalid_rev(s): class _native_CPV(object): - - """ - base ebuild package class + """base ebuild package class :ivar category: str category :ivar package: str package @@ -121,7 +119,8 @@ def __init__(self, *a, **kwds): if isvalid_rev(pkg_chunks[-1]): if lpkg_chunks < 3: # needs at least ('pkg', 'ver', 'rev') - raise InvalidCPV(f'{cpvstr}: missing package name, version, and/or revision') + raise InvalidCPV( + f'{cpvstr}: missing package name, version, and/or revision') rev = int(pkg_chunks.pop(-1)[1:]) if not rev: rev = None @@ -303,10 +302,9 @@ def native_ver_cmp(ver1, rev1, ver2, rev2): def mk_cpv_cls(base_cls): - class CPV(base.base, base_cls): - """ - base ebuild package class + class CPV(base.base, base_cls): + """base ebuild package class :ivar category: str category :ivar package: str package @@ -388,4 +386,4 @@ class versioned_CPV_cls(CPV): __slots__ = () def __init__(self, *args): - CPV.__init__(self, versioned=True, *args) + super().__init__(versioned=True, *args) From 2d987c49c66b6ca2c7fb791cd663c41b5bf25090 Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Wed, 9 Jan 2019 21:29:28 -0600 Subject: [PATCH 020/160] ebuild.cpv: use better variable name for pkg category --- src/pkgcore/ebuild/cpv.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index 1fa5acf2..1da17526 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -102,14 +102,14 @@ def __init__(self, *a, **kwds): raise TypeError(self.cpvstr) try: - categories, pkgver = cpvstr.rsplit("/", 1) + category, pkgver = cpvstr.rsplit("/", 1) except ValueError: # occurs if the rsplit yields only one item raise InvalidCPV(f'{cpvstr}: no package or version components') - if not isvalid_cat_re.match(categories): + if not isvalid_cat_re.match(category): raise InvalidCPV(f'{cpvstr}: invalid category name') sf = object.__setattr__ - sf(self, 'category', categories) + sf(self, 'category', category) sf(self, 'cpvstr', cpvstr) pkg_chunks = pkgver.split("-") lpkg_chunks = len(pkg_chunks) @@ -125,7 +125,7 @@ def __init__(self, *a, **kwds): if not rev: rev = None # reset the stored cpvstr to drop -r0+ - sf(self, 'cpvstr', f"{categories}/{'-'.join(pkg_chunks)}") + sf(self, 'cpvstr', f"{category}/{'-'.join(pkg_chunks)}") sf(self, 'revision', rev) else: sf(self, 'revision', None) @@ -141,7 +141,7 @@ def __init__(self, *a, **kwds): if not isvalid_pkg_name(pkg_chunks): raise InvalidCPV(f'{cpvstr}: invalid package name') sf(self, 'package', '-'.join(pkg_chunks)) - sf(self, 'key', f"{categories}/{self.package}") + sf(self, 'key', f"{category}/{self.package}") else: if not isvalid_pkg_name(pkg_chunks): raise InvalidCPV(f'{cpvstr}: invalid package name') From 24682084d828f18f2a1c5fba482592dcdb2d7667 Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Sat, 12 Jan 2019 00:04:07 -0600 Subject: [PATCH 021/160] util.parserestrict: parse_match(): add support for globbed targets with version restrictions This lets you do things like: pclean pkg '>=*/alsa*-1.1.7' which coincidentally removes all binpkgs for built alsa pkgs that break audio output allowing one to then run: pmerge -1dOKav alsa-lib alsa-plugins alsa-utils which reinstalls the working 1.1.6 binpkgs of those pkgs. --- src/pkgcore/ebuild/cpv.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index 1da17526..ed84f63f 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -2,7 +2,6 @@ # Copyright: 2005 Jason Stubbs # License: GPL2/BSD - """gentoo ebuild specific base package class""" __all__ = ("CPV", "versioned_CPV", "unversioned_CPV") From 743c4832e178cf9562404448228761fb6f21357f Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Sat, 19 Jan 2019 04:27:29 -0600 Subject: [PATCH 022/160] ebuild.cpv: store missing revisions internally as 0 instead of None Except for unversioned objects. Fixes #225. --- src/pkgcore/ebuild/cpv.py | 17 +++++++++++------ tests/ebuild/test_cpv.py | 8 ++++---- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index ed84f63f..288c1abc 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -64,8 +64,10 @@ class _native_CPV(object): :cvar _get_attr: mapping of attr:callable to generate attributes on the fly """ - __slots__ = ("__weakref__", "cpvstr", "key", "category", "package", - "version", "revision", "fullver") + __slots__ = ( + "__weakref__", "cpvstr", "key", "category", "package", + "version", "revision", "fullver", + ) # if native is being used, forget trying to reuse strings. def __init__(self, *a, **kwds): @@ -122,12 +124,12 @@ def __init__(self, *a, **kwds): f'{cpvstr}: missing package name, version, and/or revision') rev = int(pkg_chunks.pop(-1)[1:]) if not rev: - rev = None + rev = 0 # reset the stored cpvstr to drop -r0+ sf(self, 'cpvstr', f"{category}/{'-'.join(pkg_chunks)}") sf(self, 'revision', rev) else: - sf(self, 'revision', None) + sf(self, 'revision', 0) if not isvalid_version_re.match(pkg_chunks[-1]): raise InvalidCPV(f"{cpvstr}: invalid version '{pkg_chunks[-1]}'") @@ -176,8 +178,8 @@ def __cmp__(self, other): # ~harring # fails in doing comparison of unversioned atoms against # versioned atoms - return native_ver_cmp(self.version, self.revision, other.version, - other.revision) + return native_ver_cmp( + self.version, self.revision, other.version, other.revision) except AttributeError: return 1 @@ -186,6 +188,9 @@ def native_ver_cmp(ver1, rev1, ver2, rev2): # If the versions are the same, comparing revisions will suffice. if ver1 == ver2: + # revisions are equal if 0 or None (versionless cpv) + if not rev1 and not rev2: + return 0 return cmp(rev1, rev2) # Split up the versions into dotted strings and lists of suffixes. diff --git a/tests/ebuild/test_cpv.py b/tests/ebuild/test_cpv.py index 2dae0690..176824b3 100644 --- a/tests/ebuild/test_cpv.py +++ b/tests/ebuild/test_cpv.py @@ -126,7 +126,7 @@ def test_parsing(self): ver, rev) for x in (10, 18, 19, 36, 100): - assert self.kls("da", "ba", f"1-r0{'0' * x}").revision is None + assert self.kls("da", "ba", f"1-r0{'0' * x}").revision == 0 assert \ int(self.kls("da", "ba", f"1-r1{'0' * x}1").revision) == int(f"1{'0' * x}1") @@ -152,7 +152,7 @@ def process_ver(self, ret, cat, pkg, ver, rev): c = self.make_inst(cat, pkg, ver + rev) if rev == "" or rev == "-r0": assert c.cpvstr == f"{cat}/{pkg}-{ver}" - assert c.revision is None + assert c.revision == 0 assert c.fullver == ver else: assert c.revision == int(rev.lstrip("-r")) @@ -183,7 +183,7 @@ def process_suf(self, ret, cat, pkg, ver, rev): c = self.make_inst(cat, pkg, ver + rev) if rev == '' or rev == '-r0': assert c.cpvstr == f"{cat}/{pkg}-{ver}" - assert c.revision is None + assert c.revision == 0 assert c.fullver == ver else: assert c.cpvstr == f"{cat}/{pkg}-{ver}{rev}" @@ -307,7 +307,7 @@ def test_no_init(self): def test_r0_removal(self): obj = self.kls("dev-util/diffball-1.0-r0", versioned=True) assert obj.fullver == "1.0" - assert obj.revision is None + assert obj.revision == 0 assert str(obj) == "dev-util/diffball-1.0" From 485fad302530043d6cef25e6944b756bb00d461f Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Wed, 23 Jan 2019 01:09:43 -0600 Subject: [PATCH 023/160] ebuild.cpv: store explicit -r0 revisions as 0 and missing as None Fixes MissingRevision check for pkgcheck. --- src/pkgcore/ebuild/cpv.py | 5 ++--- tests/ebuild/test_cpv.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index 288c1abc..3706bddb 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -123,13 +123,12 @@ def __init__(self, *a, **kwds): raise InvalidCPV( f'{cpvstr}: missing package name, version, and/or revision') rev = int(pkg_chunks.pop(-1)[1:]) - if not rev: - rev = 0 + if rev == 0: # reset the stored cpvstr to drop -r0+ sf(self, 'cpvstr', f"{category}/{'-'.join(pkg_chunks)}") sf(self, 'revision', rev) else: - sf(self, 'revision', 0) + sf(self, 'revision', None) if not isvalid_version_re.match(pkg_chunks[-1]): raise InvalidCPV(f"{cpvstr}: invalid version '{pkg_chunks[-1]}'") diff --git a/tests/ebuild/test_cpv.py b/tests/ebuild/test_cpv.py index 176824b3..a1b6b070 100644 --- a/tests/ebuild/test_cpv.py +++ b/tests/ebuild/test_cpv.py @@ -152,7 +152,10 @@ def process_ver(self, ret, cat, pkg, ver, rev): c = self.make_inst(cat, pkg, ver + rev) if rev == "" or rev == "-r0": assert c.cpvstr == f"{cat}/{pkg}-{ver}" - assert c.revision == 0 + if rev: + assert c.revision == 0 + else: + assert c.revision is None assert c.fullver == ver else: assert c.revision == int(rev.lstrip("-r")) @@ -183,7 +186,10 @@ def process_suf(self, ret, cat, pkg, ver, rev): c = self.make_inst(cat, pkg, ver + rev) if rev == '' or rev == '-r0': assert c.cpvstr == f"{cat}/{pkg}-{ver}" - assert c.revision == 0 + if rev: + assert c.revision == 0 + else: + assert c.revision is None assert c.fullver == ver else: assert c.cpvstr == f"{cat}/{pkg}-{ver}{rev}" From 14e28556974e9047848fafeba91fa69f98484e33 Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Sun, 14 Jul 2019 02:06:54 -0600 Subject: [PATCH 024/160] ebuild.cpv: store ebuild revisions as strings and compare as integers Fixes handling for leading '0' characters or using explicit '-r0' revisioning. Fixes #261. --- src/pkgcore/ebuild/cpv.py | 57 +++++++++++++++++++++++++++++++++++---- tests/ebuild/test_cpv.py | 35 ++++++++++++++++++++---- 2 files changed, 82 insertions(+), 10 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index 3706bddb..85e5c5c3 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -6,6 +6,8 @@ __all__ = ("CPV", "versioned_CPV", "unversioned_CPV") +from collections import UserString + from snakeoil.compatibility import cmp from snakeoil.demandload import demandload, demand_compile_regexp from snakeoil.klass import inject_richcmp_methods_from_cmp @@ -51,6 +53,49 @@ def isvalid_rev(s): return s and s[0] == 'r' and s[1:].isdigit() +class _Revision(UserString): + """Internal revision class storing revisions as strings and comparing as integers.""" + + # parent __hash__() isn't inherited when __eq__() is defined in the child class + # https://docs.python.org/3/reference/datamodel.html#object.__hash__ + __hash__ = UserString.__hash__ + + def __eq__(self, other): + if isinstance(other, _Revision): + return int(self.data) == int(other.data) + elif isinstance(other, int): + return int(self.data) == other + return self.data == other + + def __lt__(self, other): + if isinstance(other, _Revision): + return int(self.data) < int(other.data) + elif isinstance(other, int): + return int(self.data) < other + return self.data < other + + def __le__(self, other): + if isinstance(other, _Revision): + return int(self.data) <= int(other.data) + elif isinstance(other, int): + return int(self.data) <= other + return self.data <= other + + def __gt__(self, other): + if isinstance(other, _Revision): + return int(self.data) > int(other.data) + elif isinstance(other, int): + return int(self.data) > other + return self.data > other + + def __ge__(self, other): + if isinstance(other, _Revision): + return int(self.data) >= int(other.data) + elif isinstance(other, int): + return int(self.data) >= other + return self.data >= other + + class _native_CPV(object): """base ebuild package class @@ -58,7 +103,7 @@ class _native_CPV(object): :ivar package: str package :ivar key: strkey (cat/pkg) :ivar version: str version - :ivar revision: int revision + :ivar revision: str revision :ivar versioned_atom: atom matching this exact version :ivar unversioned_atom: atom matching all versions of this package :cvar _get_attr: mapping of attr:callable to generate attributes on the fly @@ -122,10 +167,13 @@ def __init__(self, *a, **kwds): # needs at least ('pkg', 'ver', 'rev') raise InvalidCPV( f'{cpvstr}: missing package name, version, and/or revision') - rev = int(pkg_chunks.pop(-1)[1:]) + rev = _Revision(pkg_chunks.pop(-1)[1:]) if rev == 0: - # reset the stored cpvstr to drop -r0+ + # reset stored cpvstr to drop -r0+ sf(self, 'cpvstr', f"{category}/{'-'.join(pkg_chunks)}") + elif rev[0] == '0': + # reset stored cpvstr to drop leading zeroes from revision + sf(self, 'cpvstr', f"{category}/{'-'.join(pkg_chunks)}-r{int(rev)}") sf(self, 'revision', rev) else: sf(self, 'revision', None) @@ -184,7 +232,6 @@ def __cmp__(self, other): def native_ver_cmp(ver1, rev1, ver2, rev2): - # If the versions are the same, comparing revisions will suffice. if ver1 == ver2: # revisions are equal if 0 or None (versionless cpv) @@ -313,7 +360,7 @@ class CPV(base.base, base_cls): :ivar package: str package :ivar key: strkey (cat/pkg) :ivar version: str version - :ivar revision: int revision + :ivar revision: str revision :ivar versioned_atom: atom matching this exact version :ivar unversioned_atom: atom matching all versions of this package :cvar _get_attr: mapping of attr:callable to generate attributes on the fly diff --git a/tests/ebuild/test_cpv.py b/tests/ebuild/test_cpv.py index a1b6b070..5730909a 100644 --- a/tests/ebuild/test_cpv.py +++ b/tests/ebuild/test_cpv.py @@ -154,13 +154,14 @@ def process_ver(self, ret, cat, pkg, ver, rev): assert c.cpvstr == f"{cat}/{pkg}-{ver}" if rev: assert c.revision == 0 + assert c.fullver == ver + rev else: assert c.revision is None - assert c.fullver == ver + assert c.fullver == ver else: assert c.revision == int(rev.lstrip("-r")) assert c.cpvstr == f"{cat}/{pkg}-{ver}{rev}" - assert c.fullver == ver+rev + assert c.fullver == ver + rev assert c.category == cat assert c.package == pkg assert c.key == f"{cat}/{pkg}" @@ -188,9 +189,10 @@ def process_suf(self, ret, cat, pkg, ver, rev): assert c.cpvstr == f"{cat}/{pkg}-{ver}" if rev: assert c.revision == 0 + assert c.fullver == ver + rev else: assert c.revision is None - assert c.fullver == ver + assert c.fullver == ver else: assert c.cpvstr == f"{cat}/{pkg}-{ver}{rev}" assert c.revision == int(rev.lstrip("-r")) @@ -310,11 +312,34 @@ def test_no_init(self): except AttributeError: pass - def test_r0_removal(self): + def test_r0_revisions(self): + # single '0' obj = self.kls("dev-util/diffball-1.0-r0", versioned=True) - assert obj.fullver == "1.0" + assert obj.cpvstr == "dev-util/diffball-1.0" + assert str(obj) == "dev-util/diffball-1.0" + assert obj.fullver == "1.0-r0" assert obj.revision == 0 + + # multiple '0' + obj = self.kls("dev-util/diffball-1.0-r000", versioned=True) + assert obj.cpvstr == "dev-util/diffball-1.0" assert str(obj) == "dev-util/diffball-1.0" + assert obj.fullver == "1.0-r000" + assert obj.revision == 0 + + # single '0' prefix + obj = self.kls("dev-util/diffball-1.0-r01", versioned=True) + assert obj.cpvstr == "dev-util/diffball-1.0-r1" + assert str(obj) == "dev-util/diffball-1.0-r1" + assert obj.fullver == "1.0-r01" + assert obj.revision == 1 + + # multiple '0' prefixes + obj = self.kls("dev-util/diffball-1.0-r0001", versioned=True) + assert obj.cpvstr == "dev-util/diffball-1.0-r1" + assert str(obj) == "dev-util/diffball-1.0-r1" + assert obj.fullver == "1.0-r0001" + assert obj.revision == 1 @pytest.mark.skipif(not cpv.cpy_builtin, reason="cpython cpv extension not available") From f37f5b306da639952d5a885c3242282ab83f2929 Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Sun, 14 Jul 2019 19:33:37 -0600 Subject: [PATCH 025/160] ebuild.cpv: make empty revision store an empty string And set revision to None only for non-versioned CPVs to match other version attrs in that case. --- src/pkgcore/ebuild/cpv.py | 39 ++++++++++++++++++++++++++++----------- tests/ebuild/test_cpv.py | 13 +++++++++---- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index 85e5c5c3..a90d709c 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -60,39 +60,56 @@ class _Revision(UserString): # https://docs.python.org/3/reference/datamodel.html#object.__hash__ __hash__ = UserString.__hash__ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if self.data: + self._revint = int(self.data) + else: + self._revint = 0 + def __eq__(self, other): if isinstance(other, _Revision): - return int(self.data) == int(other.data) + return self._revint == other._revint elif isinstance(other, int): - return int(self.data) == other + return self._revint == other + elif other is None: + return self._revint == 0 return self.data == other def __lt__(self, other): if isinstance(other, _Revision): - return int(self.data) < int(other.data) + return self._revint < other._revint elif isinstance(other, int): - return int(self.data) < other + return self._revint < other + elif other is None: + return self._revint < 0 return self.data < other def __le__(self, other): if isinstance(other, _Revision): - return int(self.data) <= int(other.data) + return self._revint <= other._revint elif isinstance(other, int): - return int(self.data) <= other + return self._revint <= other + elif other is None: + return self._revint <= 0 return self.data <= other def __gt__(self, other): if isinstance(other, _Revision): - return int(self.data) > int(other.data) + return self._revint > other._revint elif isinstance(other, int): - return int(self.data) > other + return self._revint > other + elif other is None: + return self._revint > 0 return self.data > other def __ge__(self, other): if isinstance(other, _Revision): - return int(self.data) >= int(other.data) + return self._revint >= other._revint elif isinstance(other, int): - return int(self.data) >= other + return self._revint >= other + elif other is None: + return self._revint >= 0 return self.data >= other @@ -176,7 +193,7 @@ def __init__(self, *a, **kwds): sf(self, 'cpvstr', f"{category}/{'-'.join(pkg_chunks)}-r{int(rev)}") sf(self, 'revision', rev) else: - sf(self, 'revision', None) + sf(self, 'revision', _Revision('')) if not isvalid_version_re.match(pkg_chunks[-1]): raise InvalidCPV(f"{cpvstr}: invalid version '{pkg_chunks[-1]}'") diff --git a/tests/ebuild/test_cpv.py b/tests/ebuild/test_cpv.py index 5730909a..9b42eff9 100644 --- a/tests/ebuild/test_cpv.py +++ b/tests/ebuild/test_cpv.py @@ -152,11 +152,10 @@ def process_ver(self, ret, cat, pkg, ver, rev): c = self.make_inst(cat, pkg, ver + rev) if rev == "" or rev == "-r0": assert c.cpvstr == f"{cat}/{pkg}-{ver}" + assert c.revision == 0 if rev: - assert c.revision == 0 assert c.fullver == ver + rev else: - assert c.revision is None assert c.fullver == ver else: assert c.revision == int(rev.lstrip("-r")) @@ -187,11 +186,10 @@ def process_suf(self, ret, cat, pkg, ver, rev): c = self.make_inst(cat, pkg, ver + rev) if rev == '' or rev == '-r0': assert c.cpvstr == f"{cat}/{pkg}-{ver}" + assert c.revision == 0 if rev: - assert c.revision == 0 assert c.fullver == ver + rev else: - assert c.revision is None assert c.fullver == ver else: assert c.cpvstr == f"{cat}/{pkg}-{ver}{rev}" @@ -253,7 +251,14 @@ def test_cmp(self): assert vkls("dev-lang/erlang-12.2.5") > vkls("dev-lang/erlang-12.2b") assert vkls("dev-lang/erlang-12.2.5-r1") > vkls("dev-lang/erlang-12.2b") + # equivalent versions assert vkls("da/ba-6.01.0") == vkls("da/ba-6.010.0") + assert vkls("da/ba-6.0.1") == vkls("da/ba-6.000.1") + + # equivalent revisions + assert vkls("da/ba-6.01.0") == vkls("da/ba-6.01.0-r0") + assert vkls("da/ba-6.01.0-r0") == vkls("da/ba-6.01.0-r00") + assert vkls("da/ba-6.01.0-r1") == vkls("da/ba-6.01.0-r001") for v1, v2 in (("1.001000000000000000001", "1.001000000000000000002"), ("1.00100000000", "1.0010000000000000001"), From 8e370252f9ccc7200b75435d49ebd3f231a701d0 Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Sun, 14 Jul 2019 23:07:31 -0600 Subject: [PATCH 026/160] ebuild.cpv: _Revision: add __str__() to return '0' on missing revision --- src/pkgcore/ebuild/cpv.py | 6 ++++++ tests/ebuild/test_cpv.py | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index a90d709c..69ae8350 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -67,6 +67,12 @@ def __init__(self, *args, **kwargs): else: self._revint = 0 + def __str__(self): + if not self.data: + return '0' + else: + return self.data + def __eq__(self, other): if isinstance(other, _Revision): return self._revint == other._revint diff --git a/tests/ebuild/test_cpv.py b/tests/ebuild/test_cpv.py index 9b42eff9..52b70e2c 100644 --- a/tests/ebuild/test_cpv.py +++ b/tests/ebuild/test_cpv.py @@ -156,6 +156,7 @@ def process_ver(self, ret, cat, pkg, ver, rev): if rev: assert c.fullver == ver + rev else: + assert c.revision == "" assert c.fullver == ver else: assert c.revision == int(rev.lstrip("-r")) @@ -190,6 +191,7 @@ def process_suf(self, ret, cat, pkg, ver, rev): if rev: assert c.fullver == ver + rev else: + assert c.revision == "" assert c.fullver == ver else: assert c.cpvstr == f"{cat}/{pkg}-{ver}{rev}" From 395cdc0b1bc46d837a8593cac1c1ab8d97c8d402 Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Tue, 23 Jul 2019 04:48:52 -0600 Subject: [PATCH 027/160] tests: move module unit tests into their own directory To provide separation for future integration and benchmark tests in relation to unit tests. --- tests/ebuild/test_cpv.py | 366 --------------------------------------- 1 file changed, 366 deletions(-) delete mode 100644 tests/ebuild/test_cpv.py diff --git a/tests/ebuild/test_cpv.py b/tests/ebuild/test_cpv.py deleted file mode 100644 index 52b70e2c..00000000 --- a/tests/ebuild/test_cpv.py +++ /dev/null @@ -1,366 +0,0 @@ -# Copyright: 2006-2011 Brian Harring -# Copyright: 2006 Marien Zwart -# License: BSD/GPL2 - -from random import shuffle - -import pytest -from snakeoil.compatibility import cmp -from snakeoil.test import mk_cpy_loadable_testcase - -from pkgcore.ebuild import cpv - -def generate_misc_sufs(): - simple_good_sufs = ["_alpha", "_beta", "_pre", "_p"] - suf_nums = list(range(100)) - shuffle(suf_nums) - - good_sufs = (simple_good_sufs + [f"{x}{suf_nums.pop()}" for x in simple_good_sufs]) - - l = len(good_sufs) - good_sufs = good_sufs + [ - good_sufs[x] + good_sufs[l - x - 1] for x in range(l)] - - bad_sufs = ["_a", "_9", "_"] + [x+" " for x in simple_good_sufs] - return good_sufs, bad_sufs - - -class Test_native_Cpv(object): - - kls = staticmethod(cpv.native_CPV) - - @classmethod - def vkls(cls, *args): - return cls.kls(versioned=True, *args) - - def ukls(cls, *args): - return cls.kls(versioned=False, *args) - - run_cpy_ver_cmp = False - - good_cats = ( - "dev-util", "dev+", "dev-util+", "DEV-UTIL", "aaa0", - "aaa-0", "multi/depth", "cross-dev_idiot.hacks-suck", "a") - bad_cats = (".util", "_dev", "", "dev-util ", "multi//depth") - good_pkgs = ("diffball", "a9", "a9+", "a-100dpi", "diff-mode-") - bad_pkgs = ("diffball ", "diffball-9", "a-3D", "ab--df", "-df", "+dfa") - - good_cp = ( - "bbb-9/foon", "dev-util/diffball", "dev-util/diffball-a9", - "dev-ut-asdf/emacs-cvs", "xfce-base/xfce4", "bah/f-100dpi", - "dev-util/diffball-blah-monkeys") - - good_vers = ("1", "2.3.4", "2.3.4a", "02.3", "2.03", "3d", "3D") - bad_vers = ("2.3a.4", "2.a.3", "2.3_", "2.3 ", "2.3.", "cvs.2") - - good_sufs, bad_sufs = generate_misc_sufs() - good_revs = ("-r1", "-r300", "-r0", "", - "-r1000000000000000000") - bad_revs = ("-r", "-ra", "-r", "-R1") - - testing_secondary_args = False - - def make_inst(self, cat, pkg, fullver=""): - if self.testing_secondary_args: - return self.kls(cat, pkg, fullver, versioned=bool(fullver)) - if fullver: - return self.vkls(f"{cat}/{pkg}-{fullver}") - return self.ukls(f"{cat}/{pkg}") - - def test_simple_key(self): - with pytest.raises(cpv.InvalidCPV): - self.make_inst("da", "ba-3", "3.3") - for src in [[("dev-util", "diffball", "0.7.1"), "dev-util/diffball"], - ["dev-util/diffball"], - ["dev-perl/mod_perl"], - ["dev-perl/mod_p"], - [("dev-perl", "mod-p", ""), "dev-perl/mod-p"], - ["dev-perl/mod-p-1", "dev-perl/mod-p"],]: - if len(src) == 1: - key = src[0] - else: - key = src[1] - if isinstance(src[0], str): - cat, pkgver = src[0].rsplit("/", 1) - vals = pkgver.rsplit("-", 1) - if len(vals) == 1: - pkg = pkgver - ver = '' - else: - pkg, ver = vals - else: - cat, pkg, ver = src[0] - - assert self.make_inst(cat, pkg, ver).key == key - - def test_init(self): - self.kls("dev-util", "diffball", "0.7.1") - self.kls("dev-util/diffball-0.7.1", versioned=True) - with pytest.raises(TypeError): - self.kls("dev-util", "diffball") - with pytest.raises(TypeError): - self.vkls("dev-util", "diffball", None) - - def test_parsing(self): - # check for gentoo bug 263787 - self.process_pkg(False, 'app-text', 'foo-123-bar') - self.process_ver(False, 'app-text', 'foo-123-bar', '2.0017a_p', '-r5') - with pytest.raises(cpv.InvalidCPV): - self.ukls('app-text/foo-123') - for cat_ret, cats in [[False, self.good_cats], [True, self.bad_cats]]: - for cat in cats: - for pkg_ret, pkgs in [[False, self.good_pkgs], - [True, self.bad_pkgs]]: - for pkg in pkgs: - self.process_pkg(cat_ret or pkg_ret, cat, pkg) - - for cp in self.good_cp: - cat, pkg = cp.rsplit("/", 1) - for rev_ret, revs in [[False, self.good_revs], - [True, self.bad_revs]]: - for rev in revs: - for ver_ret, vers in [[False, self.good_vers], - [True, self.bad_vers]]: - for ver in vers: - self.process_ver(ver_ret or rev_ret, cat, pkg, - ver, rev) - - for x in (10, 18, 19, 36, 100): - assert self.kls("da", "ba", f"1-r0{'0' * x}").revision == 0 - assert \ - int(self.kls("da", "ba", f"1-r1{'0' * x}1").revision) == int(f"1{'0' * x}1") - - def process_pkg(self, ret, cat, pkg): - if ret: - with pytest.raises(cpv.InvalidCPV): - self.make_inst(cat, pkg) - else: - c = self.make_inst(cat, pkg) - assert c.cpvstr == f"{cat}/{pkg}" - assert c.category == cat - assert c.package == pkg - assert c.key == f"{cat}/{pkg}" - assert c.revision is None - assert c.version is None - assert c.fullver is None - - def process_ver(self, ret, cat, pkg, ver, rev): - if ret: - with pytest.raises(cpv.InvalidCPV): - self.make_inst(cat, pkg, f"{ver}{rev}") - else: - c = self.make_inst(cat, pkg, ver + rev) - if rev == "" or rev == "-r0": - assert c.cpvstr == f"{cat}/{pkg}-{ver}" - assert c.revision == 0 - if rev: - assert c.fullver == ver + rev - else: - assert c.revision == "" - assert c.fullver == ver - else: - assert c.revision == int(rev.lstrip("-r")) - assert c.cpvstr == f"{cat}/{pkg}-{ver}{rev}" - assert c.fullver == ver + rev - assert c.category == cat - assert c.package == pkg - assert c.key == f"{cat}/{pkg}" - assert c.version == ver - - for suf in self.good_sufs: - self.process_suf(ret, cat, pkg, ver + suf, rev) - for bad_suf in self.bad_sufs: - # double process, front and back. - self.process_suf(True, cat, pkg, suf + bad_suf, rev) - self.process_suf(True, cat, pkg, bad_suf + suf, rev) - - for suf in self.bad_sufs: - # check standalone. - self.process_suf(True, cat, pkg, ver+suf, rev) - - def process_suf(self, ret, cat, pkg, ver, rev): - if ret: - with pytest.raises(cpv.InvalidCPV): - self.make_inst(cat, pkg, ver+rev) - else: - # redundant in light of process_ver... combine these somehow. - c = self.make_inst(cat, pkg, ver + rev) - if rev == '' or rev == '-r0': - assert c.cpvstr == f"{cat}/{pkg}-{ver}" - assert c.revision == 0 - if rev: - assert c.fullver == ver + rev - else: - assert c.revision == "" - assert c.fullver == ver - else: - assert c.cpvstr == f"{cat}/{pkg}-{ver}{rev}" - assert c.revision == int(rev.lstrip("-r")) - assert c.fullver == ver + rev - assert c.category == cat - assert c.package == pkg - assert c.key == f"{cat}/{pkg}" - assert c.version == ver - - def assertGT(self, obj1, obj2): - assert obj1 > obj2, f'{obj1!r} must be > {obj2!r}' - # swap the ordering, so that it's no longer obj1.__cmp__, but obj2s - assert obj2 < obj1, f'{obj2!r} must be < {obj1!r}' - - if self.run_cpy_ver_cmp and obj1.fullver and obj2.fullver: - assert cpv.cpy_ver_cmp( - obj1.version, obj1.revision, obj2.version, obj2.revision) > 0, \ - f'cpy_ver_cmp, {obj1!r} > {obj2!r}' - assert cpv.cpy_ver_cmp( - obj2.version, obj2.revision, obj1.version, obj1.revision) < 0, \ - f'cpy_ver_cmp, {obj2!r} < {obj1!r}' - - def test_cmp(self): - ukls, vkls = self.ukls, self.vkls - assert cmp(vkls("dev-util/diffball-0.1"), vkls("dev-util/diffball-0.2")) < 0 - base = "dev-util/diffball-0.7.1" - assert not cmp(vkls(base), vkls(base)) - for rev in ("", "-r1"): - last = None - for suf in ["_alpha", "_beta", "_pre", "", "_p"]: - if suf == "": - sufs = [suf] - else: - sufs = [suf, suf+"4"] - for x in sufs: - cur = vkls(base+x+rev) - assert cur == vkls(base+x+rev) - if last is not None: - assert cur > last - - assert vkls("da/ba-6a") > vkls("da/ba-6") - assert vkls("da/ba-6a-r1") > vkls("da/ba-6a") - assert vkls("da/ba-6.0") > vkls("da/ba-6") - assert vkls("da/ba-6.0.0") > vkls("da/ba-6.0b") - assert vkls("da/ba-6.02") > vkls("da/ba-6.0.0") - # float comparison rules. - assert vkls("da/ba-6.2") > vkls("da/ba-6.054") - assert vkls("da/ba-6") == vkls("da/ba-6") - assert ukls("db/ba") > ukls("da/ba") - assert ukls("da/bb") > ukls("da/ba") - assert vkls("da/ba-6.0_alpha0_p1") > vkls("da/ba-6.0_alpha") - assert vkls("da/ba-6.0_alpha") == vkls("da/ba-6.0_alpha0") - assert vkls("da/ba-6.1") > vkls("da/ba-6.09") - assert vkls("da/ba-6.0.1") > vkls("da/ba-6.0") - assert vkls("da/ba-12.2.5") > vkls("da/ba-12.2b") - - # test for gentoo bug 287848 - assert vkls("dev-lang/erlang-12.2.5") > vkls("dev-lang/erlang-12.2b") - assert vkls("dev-lang/erlang-12.2.5-r1") > vkls("dev-lang/erlang-12.2b") - - # equivalent versions - assert vkls("da/ba-6.01.0") == vkls("da/ba-6.010.0") - assert vkls("da/ba-6.0.1") == vkls("da/ba-6.000.1") - - # equivalent revisions - assert vkls("da/ba-6.01.0") == vkls("da/ba-6.01.0-r0") - assert vkls("da/ba-6.01.0-r0") == vkls("da/ba-6.01.0-r00") - assert vkls("da/ba-6.01.0-r1") == vkls("da/ba-6.01.0-r001") - - for v1, v2 in (("1.001000000000000000001", "1.001000000000000000002"), - ("1.00100000000", "1.0010000000000000001"), - ("1.01", "1.1")): - assert vkls(f"da/ba-{v2}") > vkls(f"da/ba-{v1}") - - for x in (18, 36, 100): - s = "0" * x - assert vkls(f"da/ba-10{s}1") > vkls(f"da/ba-1{s}1") - - for x in (18, 36, 100): - s = "0" * x - assert vkls(f"da/ba-1-r10{s}1") > vkls(f"da/ba-1-r1{s}1") - - assert vkls('sys-apps/net-tools-1.60_p2010081516093') > \ - vkls('sys-apps/net-tools-1.60_p2009072801401') - - assert vkls('sys-apps/net-tools-1.60_p20100815160931') > \ - vkls('sys-apps/net-tools-1.60_p20090728014017') - - assert vkls('sys-apps/net-tools-1.60_p20100815160931') > \ - vkls('sys-apps/net-tools-1.60_p20090728014017-r1') - - # Regression test: python does comparison slightly differently - # if the classes do not match exactly (it prefers rich - # comparison over __cmp__). - class DummySubclass(self.kls): - pass - - assert DummySubclass("da/ba-6.0_alpha0_p1", versioned=True) != vkls("da/ba-6.0_alpha") - assert DummySubclass("da/ba-6.0_alpha0", versioned=True) == vkls("da/ba-6.0_alpha") - - assert DummySubclass("da/ba-6.0", versioned=True) != "foon" - assert DummySubclass("da/ba-6.0", versioned=True) == \ - DummySubclass("da/ba-6.0-r0", versioned=True) - - def test_no_init(self): - """Test if the cpv is in a somewhat sane state if __init__ fails. - - IPython used to segfault when showing a verbose traceback for - a subclass of CPV which raised cpv.InvalidCPV. This checks - if such uninitialized objects survive some basic poking. - """ - uninited = self.kls.__new__(self.kls) - broken = self.kls.__new__(self.kls) - with pytest.raises(cpv.InvalidCPV): - broken.__init__('broken', versioned=True) - for thing in (uninited, broken): - # the c version returns None, the py version does not have the attr - getattr(thing, 'cpvstr', None) - repr(thing) - str(thing) - # The c version returns a constant, the py version raises - try: - hash(thing) - except AttributeError: - pass - - def test_r0_revisions(self): - # single '0' - obj = self.kls("dev-util/diffball-1.0-r0", versioned=True) - assert obj.cpvstr == "dev-util/diffball-1.0" - assert str(obj) == "dev-util/diffball-1.0" - assert obj.fullver == "1.0-r0" - assert obj.revision == 0 - - # multiple '0' - obj = self.kls("dev-util/diffball-1.0-r000", versioned=True) - assert obj.cpvstr == "dev-util/diffball-1.0" - assert str(obj) == "dev-util/diffball-1.0" - assert obj.fullver == "1.0-r000" - assert obj.revision == 0 - - # single '0' prefix - obj = self.kls("dev-util/diffball-1.0-r01", versioned=True) - assert obj.cpvstr == "dev-util/diffball-1.0-r1" - assert str(obj) == "dev-util/diffball-1.0-r1" - assert obj.fullver == "1.0-r01" - assert obj.revision == 1 - - # multiple '0' prefixes - obj = self.kls("dev-util/diffball-1.0-r0001", versioned=True) - assert obj.cpvstr == "dev-util/diffball-1.0-r1" - assert str(obj) == "dev-util/diffball-1.0-r1" - assert obj.fullver == "1.0-r0001" - assert obj.revision == 1 - - -@pytest.mark.skipif(not cpv.cpy_builtin, reason="cpython cpv extension not available") -class Test_CPY_Cpv(Test_native_Cpv): - if cpv.cpy_builtin: - kls = staticmethod(cpv.cpy_CPV) - run_cpy_ver_cmp = True - - -class Test_CPY_Cpv_OptionalArgs(Test_CPY_Cpv): - - testing_secondary_args = True - - -test_cpy_used = mk_cpy_loadable_testcase( - "pkgcore.ebuild._cpv", "pkgcore.ebuild.cpv", "CPV_base", "CPV") - From 45ac5f923a80050f8e53d2d51d2d68d45e3f337a Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Sat, 17 Aug 2019 13:26:51 -0600 Subject: [PATCH 028/160] package.errors: InvalidPackageName: split params into bad name and optional msg --- src/pkgcore/ebuild/cpv.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index 69ae8350..2bc47406 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -174,9 +174,9 @@ def __init__(self, *a, **kwds): category, pkgver = cpvstr.rsplit("/", 1) except ValueError: # occurs if the rsplit yields only one item - raise InvalidCPV(f'{cpvstr}: no package or version components') + raise InvalidCPV(cpvstr, 'no package or version components') if not isvalid_cat_re.match(category): - raise InvalidCPV(f'{cpvstr}: invalid category name') + raise InvalidCPV(cpvstr, 'invalid category name') sf = object.__setattr__ sf(self, 'category', category) sf(self, 'cpvstr', cpvstr) @@ -184,12 +184,12 @@ def __init__(self, *a, **kwds): lpkg_chunks = len(pkg_chunks) if versioned: if lpkg_chunks == 1: - raise InvalidCPV(f'{cpvstr}: missing package version') + raise InvalidCPV(cpvstr, 'missing package version') if isvalid_rev(pkg_chunks[-1]): if lpkg_chunks < 3: # needs at least ('pkg', 'ver', 'rev') raise InvalidCPV( - f'{cpvstr}: missing package name, version, and/or revision') + cpvstr, 'missing package name, version, and/or revision') rev = _Revision(pkg_chunks.pop(-1)[1:]) if rev == 0: # reset stored cpvstr to drop -r0+ @@ -202,7 +202,7 @@ def __init__(self, *a, **kwds): sf(self, 'revision', _Revision('')) if not isvalid_version_re.match(pkg_chunks[-1]): - raise InvalidCPV(f"{cpvstr}: invalid version '{pkg_chunks[-1]}'") + raise InvalidCPV(cpvstr, f"invalid version '{pkg_chunks[-1]}'") sf(self, 'version', pkg_chunks.pop(-1)) if self.revision: sf(self, 'fullver', f"{self.version}-r{self.revision}") @@ -210,12 +210,12 @@ def __init__(self, *a, **kwds): sf(self, 'fullver', self.version) if not isvalid_pkg_name(pkg_chunks): - raise InvalidCPV(f'{cpvstr}: invalid package name') + raise InvalidCPV(cpvstr, 'invalid package name') sf(self, 'package', '-'.join(pkg_chunks)) sf(self, 'key', f"{category}/{self.package}") else: if not isvalid_pkg_name(pkg_chunks): - raise InvalidCPV(f'{cpvstr}: invalid package name') + raise InvalidCPV(cpvstr, 'invalid package name') sf(self, 'revision', None) sf(self, 'fullver', None) sf(self, 'version', None) From 3d1c059d4c978cf9fd7e0d55dc692a1ac1f4a611 Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Sat, 17 Aug 2019 17:33:01 -0600 Subject: [PATCH 029/160] remove license/copyright file headers These are both covered by the LICENSE file in the repo's root directory. --- src/pkgcore/ebuild/cpv.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index 2bc47406..da135d9d 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -1,7 +1,3 @@ -# Copyright: 2005-2011 Brian Harring -# Copyright: 2005 Jason Stubbs -# License: GPL2/BSD - """gentoo ebuild specific base package class""" __all__ = ("CPV", "versioned_CPV", "unversioned_CPV") From fd02804b58039170f88bfc83d43d4b47def80a8a Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Tue, 20 Aug 2019 03:20:18 -0600 Subject: [PATCH 030/160] move to new snakeoil lazy module importing mechanism A few demandload-ed modules remain to explicitly break cyclical imports. These might be removed with refactoring in the future. --- src/pkgcore/ebuild/cpv.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index da135d9d..4934fd30 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -5,14 +5,13 @@ from collections import UserString from snakeoil.compatibility import cmp -from snakeoil.demandload import demandload, demand_compile_regexp +from snakeoil.demandload import demand_compile_regexp from snakeoil.klass import inject_richcmp_methods_from_cmp +from pkgcore.ebuild import atom from pkgcore.ebuild.errors import InvalidCPV from pkgcore.package import base -# do this to break the cycle. -demandload("pkgcore.ebuild:atom") demand_compile_regexp( 'suffix_regexp', '^(alpha|beta|rc|pre|p)(\\d*)$') From 0b087f011fd0f6e9967cdaa6276620f5c5c228c7 Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Tue, 3 Sep 2019 13:06:34 -0600 Subject: [PATCH 031/160] more various object pickling fixes Unfortunately we have to drop SlottedDict class usage (so memory usage will increase a bit) since the way it dynamically creates classes isn't conducive to usage with multiprocessing. --- src/pkgcore/ebuild/cpv.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index 4934fd30..534f7c1e 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -413,9 +413,6 @@ def versioned(cls, *args): def unversioned(cls, *args): return cls(versioned=False, *args) - def __reduce__(self): - return (self.__class__, (self.cpvstr,), None, None, None) - return CPV native_CPV = mk_cpv_cls(_native_CPV) From 9cfe81ecd1a68ef422a2e42196caef98ecb622a2 Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Mon, 16 Sep 2019 14:18:00 -0600 Subject: [PATCH 032/160] drop explicit object inheritance We don't support py2 anymore so it's not necessary. --- src/pkgcore/ebuild/cpv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index 534f7c1e..d2307cdd 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -114,7 +114,7 @@ def __ge__(self, other): return self.data >= other -class _native_CPV(object): +class _native_CPV: """base ebuild package class :ivar category: str category From a7e947cf7143f85dfd4336989dc7262ee9716c93 Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Mon, 16 Sep 2019 15:49:55 -0600 Subject: [PATCH 033/160] ebuild.cpv: add support for passing (cat, package) for unversioned CPVs --- src/pkgcore/ebuild/cpv.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index d2307cdd..07f34463 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -133,7 +133,7 @@ class _native_CPV: ) # if native is being used, forget trying to reuse strings. - def __init__(self, *a, **kwds): + def __init__(self, *args, **kwargs): """ Can be called with one string or with three string args. @@ -144,26 +144,30 @@ def __init__(self, *a, **kwds): version components of the cpv string respectively. """ versioned = True - had_versioned = 'versioned' in kwds + had_versioned = 'versioned' in kwargs if had_versioned: - versioned = kwds.pop("versioned") - if kwds: - raise TypeError(f"versioned is the only allowed kwds: {kwds!r}") - l = len(a) + versioned = kwargs.pop("versioned") + if kwargs: + raise TypeError(f"versioned is the only allowed kwargs: {kwargs!r}") + + for x in args: + if not isinstance(x, str): + raise TypeError(f"all args must be strings, got {args!r}") + + l = len(args) if l == 1: - cpvstr = a[0] + cpvstr = args[0] if not had_versioned: - raise TypeError(f"single argument invocation requires versioned kwd; {cpvstr!r}") + raise TypeError(f"single argument invocation requires versioned kwarg; {cpvstr!r}") + elif l == 2: + cpvstr = f"{args[0]}/{args[1]}" + versioned = False elif l == 3: - for x in a: - if not isinstance(x, str): - raise TypeError(f"all args must be strings, got {a!r}") - cpvstr = f"{a[0]}/{a[1]}-{a[2]}" + cpvstr = f"{args[0]}/{args[1]}-{args[2]}" versioned = True else: - raise TypeError(f"CPV takes 1 arg (cpvstr), or 3 (cat, pkg, ver): got {a!r}") - if not isinstance(cpvstr, str): - raise TypeError(self.cpvstr) + raise TypeError( + f"CPV takes 1 arg (cpvstr), 2 (cat, pkg), or 3 (cat, pkg, ver): got {args!r}") try: category, pkgver = cpvstr.rsplit("/", 1) From cd5c5b95e416c2cf62e98e06a8c2e3bf8f1a0a36 Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Mon, 16 Sep 2019 16:20:14 -0600 Subject: [PATCH 034/160] ebuild.cpv: drop cpy ext support --- src/pkgcore/ebuild/cpv.py | 340 ++++++++++++++++---------------------- 1 file changed, 142 insertions(+), 198 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index 07f34463..d810647a 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -114,7 +114,127 @@ def __ge__(self, other): return self.data >= other -class _native_CPV: +def ver_cmp(ver1, rev1, ver2, rev2): + # If the versions are the same, comparing revisions will suffice. + if ver1 == ver2: + # revisions are equal if 0 or None (versionless cpv) + if not rev1 and not rev2: + return 0 + return cmp(rev1, rev2) + + # Split up the versions into dotted strings and lists of suffixes. + parts1 = ver1.split("_") + parts2 = ver2.split("_") + + # If the dotted strings are equal, we can skip doing a detailed comparison. + if parts1[0] != parts2[0]: + + # First split up the dotted strings into their components. + ver_parts1 = parts1[0].split(".") + ver_parts2 = parts2[0].split(".") + + # Pull out any letter suffix on the final components and keep + # them for later. + letters = [] + for ver_parts in (ver_parts1, ver_parts2): + if ver_parts[-1][-1].isalpha(): + letters.append(ord(ver_parts[-1][-1])) + ver_parts[-1] = ver_parts[-1][:-1] + else: + # Using -1 simplifies comparisons later + letters.append(-1) + + # OPT: Pull length calculation out of the loop + ver_parts1_len = len(ver_parts1) + ver_parts2_len = len(ver_parts2) + + # Iterate through the components + for v1, v2 in zip(ver_parts1, ver_parts2): + + # If the string components are equal, the numerical + # components will be equal too. + if v1 == v2: + continue + + # If one of the components begins with a "0" then they + # are compared as floats so that 1.1 > 1.02; else ints. + if v1[0] != "0" and v2[0] != "0": + v1 = int(v1) + v2 = int(v2) + else: + # handle the 0.060 == 0.060 case. + v1 = v1.rstrip("0") + v2 = v2.rstrip("0") + + # If they are not equal, the higher value wins. + c = cmp(v1, v2) + if c: + return c + + if ver_parts1_len > ver_parts2_len: + return 1 + elif ver_parts2_len > ver_parts1_len: + return -1 + + # The dotted components were equal. Let's compare any single + # letter suffixes. + if letters[0] != letters[1]: + return cmp(letters[0], letters[1]) + + # The dotted components were equal, so remove them from our lists + # leaving only suffixes. + del parts1[0] + del parts2[0] + + # OPT: Pull length calculation out of the loop + parts1_len = len(parts1) + parts2_len = len(parts2) + + # Iterate through the suffixes + for x in range(max(parts1_len, parts2_len)): + + # If we're at the end of one of our lists, we need to use + # the next suffix from the other list to decide who wins. + if x == parts1_len: + match = suffix_regexp.match(parts2[x]) + val = suffix_value[match.group(1)] + if val: + return cmp(0, val) + return cmp(0, int("0"+match.group(2))) + if x == parts2_len: + match = suffix_regexp.match(parts1[x]) + val = suffix_value[match.group(1)] + if val: + return cmp(val, 0) + return cmp(int("0"+match.group(2)), 0) + + # If the string values are equal, no need to parse them. + # Continue on to the next. + if parts1[x] == parts2[x]: + continue + + # Match against our regular expression to make a split between + # "beta" and "1" in "beta1" + match1 = suffix_regexp.match(parts1[x]) + match2 = suffix_regexp.match(parts2[x]) + + # If our int'ified suffix names are different, use that as the basis + # for comparison. + c = cmp(suffix_value[match1.group(1)], suffix_value[match2.group(1)]) + if c: + return c + + # Otherwise use the digit as the basis for comparison. + c = cmp(int("0"+match1.group(2)), int("0"+match2.group(2))) + if c: + return c + + # Our versions had different strings but ended up being equal. + # The revision holds the final difference. + return cmp(rev1, rev2) + + +class CPV(base.base): """base ebuild package class :ivar category: str category @@ -127,12 +247,9 @@ class _native_CPV: :cvar _get_attr: mapping of attr:callable to generate attributes on the fly """ - __slots__ = ( - "__weakref__", "cpvstr", "key", "category", "package", - "version", "revision", "fullver", - ) + __slots__ = ("cpvstr", "key", "category", "package", "version", "revision", "fullver") + inject_richcmp_methods_from_cmp(locals()) - # if native is being used, forget trying to reuse strings. def __init__(self, *args, **kwargs): """ Can be called with one string or with three string args. @@ -247,212 +364,39 @@ def __cmp__(self, other): # ~harring # fails in doing comparison of unversioned atoms against # versioned atoms - return native_ver_cmp( + return ver_cmp( self.version, self.revision, other.version, other.revision) except AttributeError: return 1 + @property + def versioned_atom(self): + return atom.atom(f"={self.cpvstr}") -def native_ver_cmp(ver1, rev1, ver2, rev2): - # If the versions are the same, comparing revisions will suffice. - if ver1 == ver2: - # revisions are equal if 0 or None (versionless cpv) - if not rev1 and not rev2: - return 0 - return cmp(rev1, rev2) - - # Split up the versions into dotted strings and lists of suffixes. - parts1 = ver1.split("_") - parts2 = ver2.split("_") - - # If the dotted strings are equal, we can skip doing a detailed comparison. - if parts1[0] != parts2[0]: - - # First split up the dotted strings into their components. - ver_parts1 = parts1[0].split(".") - ver_parts2 = parts2[0].split(".") - - # Pull out any letter suffix on the final components and keep - # them for later. - letters = [] - for ver_parts in (ver_parts1, ver_parts2): - if ver_parts[-1][-1].isalpha(): - letters.append(ord(ver_parts[-1][-1])) - ver_parts[-1] = ver_parts[-1][:-1] - else: - # Using -1 simplifies comparisons later - letters.append(-1) - - # OPT: Pull length calculation out of the loop - ver_parts1_len = len(ver_parts1) - ver_parts2_len = len(ver_parts2) - - # Iterate through the components - for v1, v2 in zip(ver_parts1, ver_parts2): - - # If the string components are equal, the numerical - # components will be equal too. - if v1 == v2: - continue - - # If one of the components begins with a "0" then they - # are compared as floats so that 1.1 > 1.02; else ints. - if v1[0] != "0" and v2[0] != "0": - v1 = int(v1) - v2 = int(v2) - else: - # handle the 0.060 == 0.060 case. - v1 = v1.rstrip("0") - v2 = v2.rstrip("0") - - # If they are not equal, the higher value wins. - c = cmp(v1, v2) - if c: - return c - - if ver_parts1_len > ver_parts2_len: - return 1 - elif ver_parts2_len > ver_parts1_len: - return -1 - - # The dotted components were equal. Let's compare any single - # letter suffixes. - if letters[0] != letters[1]: - return cmp(letters[0], letters[1]) - - # The dotted components were equal, so remove them from our lists - # leaving only suffixes. - del parts1[0] - del parts2[0] - - # OPT: Pull length calculation out of the loop - parts1_len = len(parts1) - parts2_len = len(parts2) - - # Iterate through the suffixes - for x in range(max(parts1_len, parts2_len)): + @property + def unversioned_atom(self): + return atom.atom(self.key) - # If we're at the end of one of our lists, we need to use - # the next suffix from the other list to decide who wins. - if x == parts1_len: - match = suffix_regexp.match(parts2[x]) - val = suffix_value[match.group(1)] - if val: - return cmp(0, val) - return cmp(0, int("0"+match.group(2))) - if x == parts2_len: - match = suffix_regexp.match(parts1[x]) - val = suffix_value[match.group(1)] - if val: - return cmp(val, 0) - return cmp(int("0"+match.group(2)), 0) + @classmethod + def versioned(cls, *args): + return cls(versioned=True, *args) - # If the string values are equal, no need to parse them. - # Continue on to the next. - if parts1[x] == parts2[x]: - continue + @classmethod + def unversioned(cls, *args): + return cls(versioned=False, *args) - # Match against our regular expression to make a split between - # "beta" and "1" in "beta1" - match1 = suffix_regexp.match(parts1[x]) - match2 = suffix_regexp.match(parts2[x]) - # If our int'ified suffix names are different, use that as the basis - # for comparison. - c = cmp(suffix_value[match1.group(1)], suffix_value[match2.group(1)]) - if c: - return c - - # Otherwise use the digit as the basis for comparison. - c = cmp(int("0"+match1.group(2)), int("0"+match2.group(2))) - if c: - return c - - # Our versions had different strings but ended up being equal. - # The revision holds the final difference. - return cmp(rev1, rev2) - - -def mk_cpv_cls(base_cls): - - class CPV(base.base, base_cls): - """base ebuild package class - - :ivar category: str category - :ivar package: str package - :ivar key: strkey (cat/pkg) - :ivar version: str version - :ivar revision: str revision - :ivar versioned_atom: atom matching this exact version - :ivar unversioned_atom: atom matching all versions of this package - :cvar _get_attr: mapping of attr:callable to generate attributes on the fly - """ +class versioned_CPV(CPV): - __slots__ = () - - inject_richcmp_methods_from_cmp(locals()) - - def __repr__(self): - return '<%s cpvstr=%s @%#8x>' % ( - self.__class__.__name__, getattr(self, 'cpvstr', None), - id(self)) - - # for some insane reason, py3k doesn't pick up the __hash__... add it - # manually. - __hash__ = base_cls.__hash__ - - @property - def versioned_atom(self): - return atom.atom(f"={self.cpvstr}") - - @property - def unversioned_atom(self): - return atom.atom(self.key) - - @classmethod - def versioned(cls, *args): - return cls(versioned=True, *args) - - @classmethod - def unversioned(cls, *args): - return cls(versioned=False, *args) - - return CPV - -native_CPV = mk_cpv_cls(_native_CPV) - -try: - # No name in module - # pylint: disable-msg=E0611 - from pkgcore.ebuild._cpv import CPV as cpy_CPV - - def ver_cmp(ver1, rev1, ver2, rev2): - if ver1 == ver2: - return cmp(rev1, rev2) - if ver1 is None: - ver1 = '' - if ver2 is None: - ver2 = '' - c = cmp(cpy_CPV("fake", "pkg", ver1, versioned=bool(ver1)), - cpy_CPV("fake", "pkg", ver2, versioned=bool(ver2))) - if c != 0: - return c - return cmp(rev1, rev2) + __slots__ = () - CPV_base = cpy_CPV - cpy_builtin = True - cpy_CPV = CPV = mk_cpv_cls(cpy_CPV) -except ImportError: - ver_cmp = native_ver_cmp - cpy_builtin = False - CPV = CPV_base = native_CPV + def __init__(self, *args): + super().__init__(*args, versioned=True) -versioned_CPV = CPV.versioned -unversioned_CPV = CPV.unversioned -class versioned_CPV_cls(CPV): +class unversioned_CPV(CPV): __slots__ = () def __init__(self, *args): - super().__init__(versioned=True, *args) + super().__init__(*args, versioned=False) From c05082bcfef66a55b83e5e2d777275cf97308b89 Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Tue, 17 Sep 2019 12:26:26 -0600 Subject: [PATCH 035/160] ebuild.cpv: rename versioned/unversioned classes for consistency --- src/pkgcore/ebuild/cpv.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index d810647a..4b34a77a 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -1,7 +1,5 @@ """gentoo ebuild specific base package class""" -__all__ = ("CPV", "versioned_CPV", "unversioned_CPV") - from collections import UserString from snakeoil.compatibility import cmp @@ -386,7 +384,7 @@ def unversioned(cls, *args): return cls(versioned=False, *args) -class versioned_CPV(CPV): +class VersionedCPV(CPV): __slots__ = () @@ -394,7 +392,7 @@ def __init__(self, *args): super().__init__(*args, versioned=True) -class unversioned_CPV(CPV): +class UnversionedCPV(CPV): __slots__ = () From ae48bb77df06c710c4cb899fc2190e04978aae3e Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Tue, 17 Sep 2019 12:27:07 -0600 Subject: [PATCH 036/160] ebuild.cpv: simplify versioned param checking for init --- src/pkgcore/ebuild/cpv.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index 4b34a77a..c6489e84 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -248,7 +248,7 @@ class CPV(base.base): __slots__ = ("cpvstr", "key", "category", "package", "version", "revision", "fullver") inject_richcmp_methods_from_cmp(locals()) - def __init__(self, *args, **kwargs): + def __init__(self, *args, versioned=None): """ Can be called with one string or with three string args. @@ -258,13 +258,6 @@ def __init__(self, *args, **kwargs): If called with three args they are the category, package and version components of the cpv string respectively. """ - versioned = True - had_versioned = 'versioned' in kwargs - if had_versioned: - versioned = kwargs.pop("versioned") - if kwargs: - raise TypeError(f"versioned is the only allowed kwargs: {kwargs!r}") - for x in args: if not isinstance(x, str): raise TypeError(f"all args must be strings, got {args!r}") @@ -272,8 +265,9 @@ def __init__(self, *args, **kwargs): l = len(args) if l == 1: cpvstr = args[0] - if not had_versioned: - raise TypeError(f"single argument invocation requires versioned kwarg; {cpvstr!r}") + if versioned is None: + raise TypeError( + f"single argument invocation requires versioned kwarg; {cpvstr!r}") elif l == 2: cpvstr = f"{args[0]}/{args[1]}" versioned = False From 80f67b3fe229766958568ba245c57e89c0fd8d67 Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Wed, 18 Sep 2019 01:12:26 -0600 Subject: [PATCH 037/160] ebuild.cpv: fix versioned_atom() for unversioned CPV objects Return unversioned_atom() for CPVs lacking versions. --- src/pkgcore/ebuild/cpv.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index c6489e84..558ac109 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -363,7 +363,9 @@ def __cmp__(self, other): @property def versioned_atom(self): - return atom.atom(f"={self.cpvstr}") + if self.version is not None: + return atom.atom(f"={self.cpvstr}") + return self.unversioned_atom @property def unversioned_atom(self): From 1562549068b732f8ea08273473f561f268e3fd9e Mon Sep 17 00:00:00 2001 From: Steven Esser Date: Tue, 22 Sep 2020 15:53:53 -0400 Subject: [PATCH 038/160] Initial commit Signed-off-by: Steven Esser --- .gitignore | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..0abbef13 --- /dev/null +++ b/.gitignore @@ -0,0 +1,69 @@ +# Python compiled files +*.py[cod] + +# virtualenv and other misc bits +*.egg-info +/dist +/build +/bin +/lib +/scripts +/Scripts +/Lib +/pip-selfcheck.json +/tmp +.Python +/include +/Include +/local +*/local/* +/local/ +/share/ +/tcl/ +/.eggs/ + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.cache +.coverage +.coverage.* +nosetests.xml +htmlcov + +# Translations +*.mo + +# IDEs +.project +.pydevproject +.idea +org.eclipse.core.resources.prefs +.vscode +.vs + +# Sphinx +docs/_build +docs/bin +docs/build +docs/include +docs/Lib +doc/pyvenv.cfg +pyvenv.cfg + +# Various junk and temp files +.DS_Store +*~ +.*.sw[po] +.build +.ve +*.bak +/.cache/ + +# pyenv +/.python-version +/man/ +/.pytest_cache/ +lib64 +tcl From 85d17806960eed420cf321f1683f8122f27344c2 Mon Sep 17 00:00:00 2001 From: Steven Esser Date: Tue, 22 Sep 2020 15:54:36 -0400 Subject: [PATCH 039/160] Add initial skeleton files This commit adds the inital skeleton files needed for a bare-bones python library. Includes a simple configure script and setup.py file Signed-off-by: Steven Esser --- README.rst | 3 + apache-2.0.LICENSE | 176 +++++++++++++++++++++++++++++++++++++++++ configure | 43 ++++++++++ requirements-tests.txt | 1 + setup.py | 124 +++++++++++++++++++++++++++++ 5 files changed, 347 insertions(+) create mode 100644 README.rst create mode 100644 apache-2.0.LICENSE create mode 100755 configure create mode 100644 requirements-tests.txt create mode 100644 setup.py diff --git a/README.rst b/README.rst new file mode 100644 index 00000000..83376d53 --- /dev/null +++ b/README.rst @@ -0,0 +1,3 @@ +A Simple Python Project Skeleton +================================ +Note: configure script requires src/ directory to run correctly. diff --git a/apache-2.0.LICENSE b/apache-2.0.LICENSE new file mode 100644 index 00000000..d9a10c0d --- /dev/null +++ b/apache-2.0.LICENSE @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/configure b/configure new file mode 100755 index 00000000..48c26281 --- /dev/null +++ b/configure @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# +# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. +# + +set -e +#set -x + +# source this script for a basic setup and configuration for local development + +CONFIGURE_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + + +if [[ "$1" == "--clean" ]]; then + rm -rf "$CONFIGURE_ROOT_DIR/tmp" + exit +fi + + +if [[ "$PYTHON_EXE" == "" ]]; then + PYTHON_EXE=python3 +fi + + +function setup { + # create a virtualenv on Python + mkdir -p $CONFIGURE_ROOT_DIR/tmp + wget -O $CONFIGURE_ROOT_DIR/tmp/virtualenv.pyz https://bootstrap.pypa.io/virtualenv.pyz + $PYTHON_EXE $CONFIGURE_ROOT_DIR/tmp/virtualenv.pyz $CONFIGURE_ROOT_DIR/tmp + source $CONFIGURE_ROOT_DIR/tmp/bin/activate + $CONFIGURE_ROOT_DIR/tmp/bin/pip install --upgrade pip virtualenv setuptools wheel +} + + +setup + +$CONFIGURE_ROOT_DIR/tmp/bin/pip install -r requirements-tests.txt -e . + +if [ -f "$CONFIGURE_ROOT_DIR/tmp/bin/activate" ]; then + source "$CONFIGURE_ROOT_DIR/tmp/bin/activate" +fi + +set +e diff --git a/requirements-tests.txt b/requirements-tests.txt new file mode 100644 index 00000000..e079f8a6 --- /dev/null +++ b/requirements-tests.txt @@ -0,0 +1 @@ +pytest diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..1c920c10 --- /dev/null +++ b/setup.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +from __future__ import absolute_import +from __future__ import print_function + +import io +from glob import glob +from os.path import basename +from os.path import dirname +from os.path import join +from os.path import splitext +import re +import sys + +from setuptools import find_packages +from setuptools import setup + +version = '0.0.0' + +#### Small hack to force using a plain version number if the option +#### --plain-version is passed to setup.py + +USE_DEFAULT_VERSION = False +try: + sys.argv.remove('--use-default-version') + USE_DEFAULT_VERSION = True +except ValueError: + pass +#### + + +def get_version(default=version, template='{tag}.{distance}.{commit}{dirty}', + use_default=USE_DEFAULT_VERSION): + """ + Return a version collected from git if possible or fall back to an + hard-coded default version otherwise. If `use_default` is True, + always use the default version. + """ + if use_default: + return default + try: + tag, distance, commit, dirty = get_git_version() + if not distance and not dirty: + # we are from a clean Git tag: use tag + return tag + + distance = 'post{}'.format(distance) + if dirty: + time_stamp = get_time_stamp() + dirty = '.dirty.' + get_time_stamp() + else: + dirty = '' + + return template.format(**locals()) + except: + # no git data: use default version + return default + + +def get_time_stamp(): + """ + Return a numeric UTC time stamp without microseconds. + """ + from datetime import datetime + return (datetime.isoformat(datetime.utcnow()).split('.')[0] + .replace('T', '').replace(':', '').replace('-', '')) + + +def get_git_version(): + """ + Return version parts from Git or raise an exception. + """ + from subprocess import check_output, STDOUT + # this may fail with exceptions + cmd = 'git', 'describe', '--tags', '--long', '--dirty', + version = check_output(cmd, stderr=STDOUT).strip() + dirty = version.endswith('-dirty') + tag, distance, commit = version.split('-')[:3] + # lower tag and strip V prefix in tags + tag = tag.lower().lstrip('v ').strip() + # strip leading g from git describe commit + commit = commit.lstrip('g').strip() + return tag, int(distance), commit, dirty + + +def read(*names, **kwargs): + return io.open( + join(dirname(__file__), *names), + encoding=kwargs.get('encoding', 'utf8') + ).read() + + +setup( + name='', + version=get_version(), + license='Apache-2.0', + description='', + long_description=read('README.rst'), + author='nexB. Inc. and others', + author_email='info@aboutcode.org', + url='', + packages=find_packages('src'), + package_dir={'': 'src'}, + py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')], + include_package_data=True, + zip_safe=False, + classifiers=[ + # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Topic :: Software Development', + 'Topic :: Utilities', + ], + keywords=[ + ], + install_requires=[ + ] +) From aa71e06afd7f069202590cf532407f92fcafbf66 Mon Sep 17 00:00:00 2001 From: Steven Esser Date: Tue, 22 Sep 2020 16:21:54 -0400 Subject: [PATCH 040/160] Use pytest-xdist by default for threading support Signed-off-by: Steven Esser --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index e079f8a6..82d8d43d 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1 +1 @@ -pytest +pytest-xdist From 774dc7d6709119442e099c7b706ee3a9f89fa219 Mon Sep 17 00:00:00 2001 From: Steven Esser Date: Wed, 23 Sep 2020 19:24:10 -0400 Subject: [PATCH 041/160] Add additional files * Add AUTHORS.rst * Add CHANGELOG.rst * Add setup.cfg Signed-off-by: Steven Esser --- AUTHORS.rst | 3 +++ CHANGELOG.rst | 5 +++++ setup.cfg | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+) create mode 100644 AUTHORS.rst create mode 100644 CHANGELOG.rst create mode 100644 setup.cfg diff --git a/AUTHORS.rst b/AUTHORS.rst new file mode 100644 index 00000000..51a19cc8 --- /dev/null +++ b/AUTHORS.rst @@ -0,0 +1,3 @@ +The following organizations or individuals have contributed to this repo: + +- diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 00000000..5f8bc8da --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,5 @@ +Release notes +------------- +### Version 0.0.0 + +*xxxx-xx-xx* -- Initial release. diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..24aaefa0 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,50 @@ +[wheel] +universal=1 + +[metadata] +license_files = + README.rst + CHANGELOG.rst + apache-2.0.LICENSE + bsd-new.LICENSE + mit.LICENSE + NOTICE + +[tool:pytest] +norecursedirs = + .git + bin + dist + build + _build + dist + etc + local + ci + docs + man + share + samples + .cache + .settings + Include + include + Lib + lib + lib64 + Lib64 + Scripts + thirdparty + tmp + tests/data + +python_files = *.py + +python_classes=Test +python_functions=test + +addopts = + -rfExXw + --strict + --ignore setup.py + --doctest-modules From 9a56b880389ce929a3c0e0dca8bdda576a3f2945 Mon Sep 17 00:00:00 2001 From: Steven Esser Date: Fri, 25 Sep 2020 13:03:15 -0400 Subject: [PATCH 042/160] Follow modern python packaging and conf practices * Add PEP 517/518 pyproject.toml file * Add setuptools_scm to handle versioning * Add setup.py content to setup.cfg * Update setup.py to act as a shim (so pip install -e works) Addresses: #2 Signed-off-by: Steven Esser --- configure | 2 +- pyproject.toml | 46 +++++++++++++++ requirements-tests.txt | 1 - setup.cfg | 73 ++++++++++-------------- setup.py | 124 +---------------------------------------- 5 files changed, 80 insertions(+), 166 deletions(-) create mode 100644 pyproject.toml delete mode 100644 requirements-tests.txt diff --git a/configure b/configure index 48c26281..a35c8c90 100755 --- a/configure +++ b/configure @@ -34,7 +34,7 @@ function setup { setup -$CONFIGURE_ROOT_DIR/tmp/bin/pip install -r requirements-tests.txt -e . +$CONFIGURE_ROOT_DIR/tmp/bin/pip install -e .[testing] if [ -f "$CONFIGURE_ROOT_DIR/tmp/bin/activate" ]; then source "$CONFIGURE_ROOT_DIR/tmp/bin/activate" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..e75f1cec --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,46 @@ +[build-system] +requires = ["setuptools >= 50", "wheel", "setuptools_scm[toml] >= 4"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] + +[tool.pytest.ini_options] +norecursedirs = [ + ".git", + "bin", + "dist", + "build", + "_build", + "dist", + "etc", + "local", + "ci", + "docs", + "man", + "share", + "samples", + ".cache", + ".settings", + "Include", + "include", + "Lib", + "lib", + "lib64", + "Lib64", + "Scripts", + "thirdparty", + "tmp", + "tests/data", + ".eggs" +] + +python_files = "*.py" + +python_classes="Test" +python_functions="test" + +addopts = [ + "-rfExXw", + "--strict", + "--doctest-modules" +] \ No newline at end of file diff --git a/requirements-tests.txt b/requirements-tests.txt deleted file mode 100644 index 82d8d43d..00000000 --- a/requirements-tests.txt +++ /dev/null @@ -1 +0,0 @@ -pytest-xdist diff --git a/setup.cfg b/setup.cfg index 24aaefa0..b703e571 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,49 +2,36 @@ universal=1 [metadata] -license_files = - README.rst - CHANGELOG.rst - apache-2.0.LICENSE - bsd-new.LICENSE - mit.LICENSE - NOTICE +license_file = apache-2.0.LICENSE +name = skeleton +author = nexB. Inc. and others +author_email = info@aboutcode.org +description = skeleton +long_description = file:README.rst +url = https://github.com/nexB/skeleton +classifiers = + Development Status :: 5 - Production/Stable + Intended Audience :: Developers + Programming Language :: Python :: 3 + Programming Language :: Python :: 3 :: Only + Topic :: Software Development + Topic :: Utilities +keywords = -[tool:pytest] -norecursedirs = - .git - bin - dist - build - _build - dist - etc - local - ci - docs - man - share - samples - .cache - .settings - Include - include - Lib - lib - lib64 - Lib64 - Scripts - thirdparty - tmp - tests/data +[options] +package_dir= + =src +packages=find: +include_package_data = true +zip_safe = false +install_requires = +setup_requires = setuptools_scm[toml] >= 4 -python_files = *.py +[options.packages.find] +where=src -python_classes=Test -python_functions=test - -addopts = - -rfExXw - --strict - --ignore setup.py - --doctest-modules +[options.extras_require] +testing = + # upstream + pytest >= 6 + pytest-xdist >= 2 \ No newline at end of file diff --git a/setup.py b/setup.py index 1c920c10..45f160da 100644 --- a/setup.py +++ b/setup.py @@ -1,124 +1,6 @@ #!/usr/bin/env python -# -*- encoding: utf-8 -*- -from __future__ import absolute_import -from __future__ import print_function +import setuptools -import io -from glob import glob -from os.path import basename -from os.path import dirname -from os.path import join -from os.path import splitext -import re -import sys - -from setuptools import find_packages -from setuptools import setup - -version = '0.0.0' - -#### Small hack to force using a plain version number if the option -#### --plain-version is passed to setup.py - -USE_DEFAULT_VERSION = False -try: - sys.argv.remove('--use-default-version') - USE_DEFAULT_VERSION = True -except ValueError: - pass -#### - - -def get_version(default=version, template='{tag}.{distance}.{commit}{dirty}', - use_default=USE_DEFAULT_VERSION): - """ - Return a version collected from git if possible or fall back to an - hard-coded default version otherwise. If `use_default` is True, - always use the default version. - """ - if use_default: - return default - try: - tag, distance, commit, dirty = get_git_version() - if not distance and not dirty: - # we are from a clean Git tag: use tag - return tag - - distance = 'post{}'.format(distance) - if dirty: - time_stamp = get_time_stamp() - dirty = '.dirty.' + get_time_stamp() - else: - dirty = '' - - return template.format(**locals()) - except: - # no git data: use default version - return default - - -def get_time_stamp(): - """ - Return a numeric UTC time stamp without microseconds. - """ - from datetime import datetime - return (datetime.isoformat(datetime.utcnow()).split('.')[0] - .replace('T', '').replace(':', '').replace('-', '')) - - -def get_git_version(): - """ - Return version parts from Git or raise an exception. - """ - from subprocess import check_output, STDOUT - # this may fail with exceptions - cmd = 'git', 'describe', '--tags', '--long', '--dirty', - version = check_output(cmd, stderr=STDOUT).strip() - dirty = version.endswith('-dirty') - tag, distance, commit = version.split('-')[:3] - # lower tag and strip V prefix in tags - tag = tag.lower().lstrip('v ').strip() - # strip leading g from git describe commit - commit = commit.lstrip('g').strip() - return tag, int(distance), commit, dirty - - -def read(*names, **kwargs): - return io.open( - join(dirname(__file__), *names), - encoding=kwargs.get('encoding', 'utf8') - ).read() - - -setup( - name='', - version=get_version(), - license='Apache-2.0', - description='', - long_description=read('README.rst'), - author='nexB. Inc. and others', - author_email='info@aboutcode.org', - url='', - packages=find_packages('src'), - package_dir={'': 'src'}, - py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')], - include_package_data=True, - zip_safe=False, - classifiers=[ - # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Topic :: Software Development', - 'Topic :: Utilities', - ], - keywords=[ - ], - install_requires=[ - ] -) +if __name__ == "__main__": + setuptools.setup() \ No newline at end of file From 565feee243b68db1bc0608058178a8aa9ba485c9 Mon Sep 17 00:00:00 2001 From: Steven Esser Date: Fri, 25 Sep 2020 14:38:11 -0400 Subject: [PATCH 043/160] Add some minimal documentation Signed-off-by: Steven Esser --- README.rst | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 83376d53..049d38eb 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,31 @@ A Simple Python Project Skeleton ================================ -Note: configure script requires src/ directory to run correctly. +This repo attempts to standardize our python repositories using modern python +packaging and configuration techniques. Using this `blog post`_ as inspiration, this +repository will serve as the base for all new python projects and will be adopted to all +our existing ones as well. + +.. _blog post: https://blog.jaraco.com/a-project-skeleton-for-python-projects/ + +Usage +===== +A brand new project +------------------- +.. code-block:: bash + + git init my-new-repo + cd my-new-repo + git pull git@github.com:nexB/skeleton + +From here, you can make the appropriate changes to the files for your specific project. + +Update an existing project +--------------------------- +.. code-block:: bash + + cd my-existing-project + git remote add skeleton git@github.com:nexB/skeleton + git fetch skeleton + git merge skeleton --allow-unrelated-histories + +This is also the workflow to use when updating the skeleton files in any given repository. \ No newline at end of file From 5febefbe4f7b2cceef99a6b4b9c196fe76a266bf Mon Sep 17 00:00:00 2001 From: Steven Esser Date: Tue, 29 Sep 2020 13:32:52 -0400 Subject: [PATCH 044/160] Fix hanging tag chars in setup.cfg Signed-off-by: Steven Esser --- setup.cfg | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/setup.cfg b/setup.cfg index b703e571..a7ab2fe6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,10 +10,10 @@ description = skeleton long_description = file:README.rst url = https://github.com/nexB/skeleton classifiers = - Development Status :: 5 - Production/Stable - Intended Audience :: Developers - Programming Language :: Python :: 3 - Programming Language :: Python :: 3 :: Only + Development Status :: 5 - Production/Stable + Intended Audience :: Developers + Programming Language :: Python :: 3 + Programming Language :: Python :: 3 :: Only Topic :: Software Development Topic :: Utilities keywords = @@ -32,6 +32,6 @@ where=src [options.extras_require] testing = - # upstream - pytest >= 6 - pytest-xdist >= 2 \ No newline at end of file + # upstream + pytest >= 6 + pytest-xdist >= 2 From 343ff29e14958d31e151310608efac3ce1ad1d9c Mon Sep 17 00:00:00 2001 From: Steven Esser Date: Wed, 30 Sep 2020 13:07:18 -0400 Subject: [PATCH 045/160] Update README instructions with proper git merge conventions Signed-off-by: Steven Esser --- README.rst | 4 ++-- setup.cfg | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 049d38eb..0f1585bc 100644 --- a/README.rst +++ b/README.rst @@ -26,6 +26,6 @@ Update an existing project cd my-existing-project git remote add skeleton git@github.com:nexB/skeleton git fetch skeleton - git merge skeleton --allow-unrelated-histories + git merge skeleton/main --allow-unrelated-histories -This is also the workflow to use when updating the skeleton files in any given repository. \ No newline at end of file +This is also the workflow to use when updating the skeleton files in any given repository. diff --git a/setup.cfg b/setup.cfg index a7ab2fe6..5b665cfa 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,7 +25,7 @@ packages=find: include_package_data = true zip_safe = false install_requires = -setup_requires = setuptools_scm[toml] >= 4 +setup_requires = setuptools_scm >= 4 [options.packages.find] where=src From fa55f68fac8c80413f41bcbc3280d017e6d21198 Mon Sep 17 00:00:00 2001 From: Steven Esser Date: Fri, 16 Oct 2020 11:29:43 -0400 Subject: [PATCH 046/160] Add minimal .travis.yml CI config file Signed-off-by: Steven Esser --- .travis.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..bcc3be8e --- /dev/null +++ b/.travis.yml @@ -0,0 +1,21 @@ +# This is a skeleton Travis CI config file that provides a starting point for adding CI +# to a Python project. Since we primarily develop in python3, this skeleton config file +# will be specific to that language. +# +# See https://config.travis-ci.com/ for a full list of configuration options. + +os: linux + +dist: xenial + +language: python +python: + - "3.6" + - "3.7" + - "3.8" + +# Scripts to run at install stage +install: ./configure + +# Scripts to run at script stage +script: bin/pytest From 0332e7fb6f637336b22d4db3806bb32727c260f0 Mon Sep 17 00:00:00 2001 From: Brian Harring Date: Fri, 28 Aug 2020 23:19:41 -0700 Subject: [PATCH 047/160] Drop dead reference to _get_attr for cpv's --- src/pkgcore/ebuild/cpv.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index 558ac109..8a507f32 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -242,7 +242,6 @@ class CPV(base.base): :ivar revision: str revision :ivar versioned_atom: atom matching this exact version :ivar unversioned_atom: atom matching all versions of this package - :cvar _get_attr: mapping of attr:callable to generate attributes on the fly """ __slots__ = ("cpvstr", "key", "category", "package", "version", "revision", "fullver") From 3296b9fde77cfbe36824d88e403652bcc72f8316 Mon Sep 17 00:00:00 2001 From: Steven Esser Date: Thu, 22 Oct 2020 14:07:23 -0400 Subject: [PATCH 048/160] Update setuptools_scm declaration Signed-off-by: Steven Esser --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 5b665cfa..a7ab2fe6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,7 +25,7 @@ packages=find: include_package_data = true zip_safe = false install_requires = -setup_requires = setuptools_scm >= 4 +setup_requires = setuptools_scm[toml] >= 4 [options.packages.find] where=src From 083bd0483b5869219db51623eed44a9cd711989b Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Mon, 26 Oct 2020 15:30:56 -0700 Subject: [PATCH 049/160] Add azure pipeline config files and templates * Create configure.bat so we can use our skeleton for Windows projects Signed-off-by: Jono Yang --- .travis.yml | 4 +- azure-pipelines.yml | 45 ++++++++++++ configure.bat | 160 +++++++++++++++++++++++++++++++++++++++++ etc/ci/azure-linux.yml | 37 ++++++++++ etc/ci/azure-mac.yml | 36 ++++++++++ etc/ci/azure-win.yml | 36 ++++++++++ 6 files changed, 316 insertions(+), 2 deletions(-) create mode 100644 azure-pipelines.yml create mode 100644 configure.bat create mode 100644 etc/ci/azure-linux.yml create mode 100644 etc/ci/azure-mac.yml create mode 100644 etc/ci/azure-win.yml diff --git a/.travis.yml b/.travis.yml index bcc3be8e..7a342dff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ # This is a skeleton Travis CI config file that provides a starting point for adding CI # to a Python project. Since we primarily develop in python3, this skeleton config file -# will be specific to that language. +# will be specific to that language. # # See https://config.travis-ci.com/ for a full list of configuration options. @@ -18,4 +18,4 @@ python: install: ./configure # Scripts to run at script stage -script: bin/pytest +script: tmp/bin/pytest diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 00000000..904ac906 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,45 @@ + +################################################################################ +# We use Azure to run the full tests suites on Python 3.6 +# on Windows (32 and 64), macOS and Linux (64 various distro) +################################################################################ + +jobs: + +################################################################################ +# These jobs are using VMs and Azure-provided Python 3.6 +################################################################################ + + - template: etc/ci/azure-linux.yml + parameters: + job_name: vm_ubuntu16_py36 + image_name: ubuntu-16.04 + python_versions: ['3.6'] + test_suites: + all: bin/py.test -n 2 -vvs --reruns=3 + + - template: etc/ci/azure-mac.yml + parameters: + job_name: macos1015_py36 + image_name: macos-10.15 + python_versions: ['3.6'] + test_suites: + all: bin/py.test -n 2 -vvs --reruns=3 + + - template: etc/ci/azure-win.yml + parameters: + job_name: Win2016_32_py36 + image_name: vs2017-win2016 + python_versions: ['3.6'] + python_architecture: x86 + test_suites: + all: Scripts\py.test -vvs --reruns=3 + + - template: etc/ci/azure-win.yml + parameters: + job_name: Win2016_64_py36 + image_name: vs2017-win2016 + python_versions: ['3.6'] + python_architecture: x64 + test_suites: + misc: Scripts\py.test -vvs --reruns=3 diff --git a/configure.bat b/configure.bat new file mode 100644 index 00000000..0a2ca00f --- /dev/null +++ b/configure.bat @@ -0,0 +1,160 @@ +@echo OFF +@setlocal +@rem Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. + +@rem ################################ +@rem # A configuration script for Windows +@rem # +@rem # The options and (optional) arguments are: +@rem # --clean : this is exclusive of anything else and cleans the environment +@rem # from built and installed files +@rem # +@rem # --python < path to python.exe> : this must be the first argument and set +@rem # the path to the Python executable to use. If < path to python.exe> is +@rem # set to "path", then the executable will be the python.exe available +@rem # in the PATH. +@rem # +@rem # : this must be the last argument and sets the path to a +@rem # configuration directory to use. +@rem ################################ + +@rem ################################ +@rem # Defaults. Change these variables to customize this script locally +@rem ################################ +@rem # you can define one or more thirdparty dirs, each where the varibale name +@rem # is prefixed with TPP_DIR +set "TPP_DIR=thirdparty" + +@rem # default configurations for dev +set "CONF_DEFAULT=etc/conf/dev" + +@rem # default thirdparty dist for dev +if ""%CONF_DEFAULT%""==""etc/conf/dev"" ( + set "TPP_DIR_DEV=thirdparty/dev" +) + +@rem # default supported version for Python 3 +set SUPPORTED_PYTHON3=3.6 + +@rem ################################# + +@rem python --version +@rem python -c "import sys;print(sys.executable)" + + +@rem Current directory where this .bat files lives +set CFG_ROOT_DIR=%~dp0 + +@rem path where a configured Python should live in the current virtualenv if installed +set CONFIGURED_PYTHON=%CFG_ROOT_DIR%Scripts\python.exe + +set PYTHON_EXECUTABLE= + +@rem parse command line options and arguments +:collectopts +if "%1" EQU "--help" (goto cli_help) +if "%1" EQU "--clean" (call rmdir /s /q "%CFG_ROOT_DIR%tmp") && call exit /b +if "%1" EQU "--python" (set PROVIDED_PYTHON=%~2) && shift && shift && goto collectopts + +@rem We are not cleaning: Either we have a provided configure config path or we use a default. +if ""%1""=="""" ( + set CFG_CMD_LINE_ARGS=%CONF_DEFAULT% +) else ( + set CFG_CMD_LINE_ARGS=%1 +) + +@rem If we have a pre-configured Python in our virtualenv, reuse this as-is and run +if exist ""%CONFIGURED_PYTHON%"" ( + set PYTHON_EXECUTABLE=%CONFIGURED_PYTHON% + goto run +) + +@rem If we have a command arg for Python use this as-is +if ""%PROVIDED_PYTHON%""==""path"" ( + @rem use a bare python available in the PATH + set PYTHON_EXECUTABLE=python + goto run +) +if exist ""%PROVIDED_PYTHON%"" ( + set PYTHON_EXECUTABLE=%PROVIDED_PYTHON% + goto run +) + + +@rem otherwise we search for a suitable Python interpreter +:find_python + +@rem First check the existence of the "py" launcher (available in Python 3) +@rem if we have it, check if we have a py -3 installed with the good version or a py 2.7 +@rem if not, check if we have an old py 2.7 +@rem exist if all fails + +where py >nul 2>nul +if %ERRORLEVEL% == 0 ( + @rem we have a py launcher, check for the availability of our required Python 3 version + py -3.6 --version >nul 2>nul + if %ERRORLEVEL% == 0 ( + set PYTHON_EXECUTABLE=py -3.6 + ) else ( + @rem we have no required python 3, let's try python 2: + py -2 --version >nul 2>nul + if %ERRORLEVEL% == 0 ( + set PYTHON_EXECUTABLE=py -2 + ) else ( + @rem we have py and no python 3 and 2, exit + echo * Unable to find an installation of Python. + exit /b 1 + ) + ) +) else ( + @rem we have no py launcher, check for a default Python 2 installation + if not exist ""%DEFAULT_PYTHON2%"" ( + echo * Unable to find an installation of Python. + exit /b 1 + ) else ( + set PYTHON_EXECUTABLE=%DEFAULT_PYTHON2% + ) +) + +:run + +@rem without this things may not always work on Windows 10, but this makes things slower +set PYTHONDONTWRITEBYTECODE=1 + +call mkdir "%CFG_ROOT_DIR%tmp" +call curl -o "%CFG_ROOT_DIR%tmp\virtualenv.pyz" https://bootstrap.pypa.io/virtualenv.pyz +call %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%tmp\virtualenv.pyz" "%CFG_ROOT_DIR%tmp" +call "%CFG_ROOT_DIR%tmp\Scripts\activate" +call "%CFG_ROOT_DIR%tmp\Scripts\pip" install --upgrade pip virtualenv setuptools wheel + + +@rem Return a proper return code on failure +if %ERRORLEVEL% neq 0 ( + exit /b %ERRORLEVEL% +) +endlocal +goto activate + + +:cli_help +echo A configuration script for Windows +echo usage: configure [options] [path/to/config/directory] +echo. +echo The options and arguments are: +echo [path/to/config/directory] : this optionally sets the path to a +echo configuration directory to use. Defaults to etc/conf/dev if not set +echo. +echo --clean : this is exclusive of anything else and cleans the environment +echo from built and installed files +echo. +echo --python path/to/python.exe : this is set to the path of an alternative +echo Python executable to use. If path/to/python.exe is set to "path", +echo then the executable will be the python.exe available in the PATH. +echo. + + +:activate +@rem Activate the virtualenv +if exist "%CFG_ROOT_DIR%Scripts\activate" ( + "%CFG_ROOT_DIR%Scripts\activate" +) diff --git a/etc/ci/azure-linux.yml b/etc/ci/azure-linux.yml new file mode 100644 index 00000000..2e12e5b0 --- /dev/null +++ b/etc/ci/azure-linux.yml @@ -0,0 +1,37 @@ +parameters: + job_name: '' + image_name: 'ubuntu-16.04' + python_versions: [] + test_suites: {} + python_architecture: x64 + +jobs: + - job: ${{ parameters.job_name }} + + pool: + vmImage: ${{ parameters.image_name }} + + strategy: + matrix: + ${{ each pyver in parameters.python_versions }}: + ${{ each tsuite in parameters.test_suites }}: + ${{ format('py{0} {1}', pyver, tsuite.key) }}: + python_version: ${{ pyver }} + test_suite_label: ${{ tsuite.key }} + test_suite: ${{ tsuite.value }} + + steps: + - checkout: self + fetchDepth: 10 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python_version)' + architecture: '${{ parameters.python_architecture }}' + displayName: 'Install Python $(python_version)' + + - script: ./configure + displayName: 'Run Configure' + + - script: $(test_suite) + displayName: 'Run $(test_suite_label) tests with py$(python_version) on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-mac.yml b/etc/ci/azure-mac.yml new file mode 100644 index 00000000..752ae2e6 --- /dev/null +++ b/etc/ci/azure-mac.yml @@ -0,0 +1,36 @@ +parameters: + job_name: '' + image_name: '' + python_versions: [] + test_suites: {} + python_architecture: x64 + +jobs: + - job: ${{ parameters.job_name }} + + pool: + vmImage: ${{ parameters.image_name }} + + strategy: + matrix: + ${{ each pyver in parameters.python_versions }}: + ${{ each tsuite in parameters.test_suites }}: + ${{ format('py{0} {1}', pyver, tsuite.key) }}: + python_version: ${{ pyver }} + test_suite_label: ${{ tsuite.key }} + test_suite: ${{ tsuite.value }} + steps: + - checkout: self + fetchDepth: 10 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python_version)' + architecture: '${{ parameters.python_architecture }}' + displayName: 'Install Python $(python_version)' + + - script: ./configure + displayName: 'Run Configure' + + - script: $(test_suite) + displayName: 'Run $(test_suite_label) tests with py$(python_version) on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-win.yml b/etc/ci/azure-win.yml new file mode 100644 index 00000000..62208574 --- /dev/null +++ b/etc/ci/azure-win.yml @@ -0,0 +1,36 @@ +parameters: + job_name: '' + image_name: '' + python_versions: [] + test_suites: {} + python_architecture: x86 + +jobs: + - job: ${{ parameters.job_name }} + + pool: + vmImage: ${{ parameters.image_name }} + + strategy: + matrix: + ${{ each pyver in parameters.python_versions }}: + ${{ each tsuite in parameters.test_suites }}: + ${{ format('py{0} {1}', pyver, tsuite.key) }}: + python_version: ${{ pyver }} + test_suite_label: ${{ tsuite.key }} + test_suite: ${{ tsuite.value }} + steps: + - checkout: self + fetchDepth: 10 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python_version)' + architecture: '${{ parameters.python_architecture }}' + displayName: 'Install Python $(python_version)' + + - script: configure --python path + displayName: 'Run Configure' + + - script: $(test_suite) + displayName: 'Run $(test_suite_label) tests with py$(python_version) on ${{ parameters.job_name }}' From 847564ead159cfcebe0f04066daa6f23e1a5a123 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 27 Oct 2020 10:40:38 -0700 Subject: [PATCH 050/160] Fix path to Scripts directory * Remove unused variables and options Signed-off-by: Jono Yang --- configure.bat | 48 ++++-------------------------------------------- 1 file changed, 4 insertions(+), 44 deletions(-) diff --git a/configure.bat b/configure.bat index 0a2ca00f..cbb42446 100644 --- a/configure.bat +++ b/configure.bat @@ -13,41 +13,12 @@ @rem # the path to the Python executable to use. If < path to python.exe> is @rem # set to "path", then the executable will be the python.exe available @rem # in the PATH. -@rem # -@rem # : this must be the last argument and sets the path to a -@rem # configuration directory to use. -@rem ################################ - @rem ################################ -@rem # Defaults. Change these variables to customize this script locally -@rem ################################ -@rem # you can define one or more thirdparty dirs, each where the varibale name -@rem # is prefixed with TPP_DIR -set "TPP_DIR=thirdparty" - -@rem # default configurations for dev -set "CONF_DEFAULT=etc/conf/dev" - -@rem # default thirdparty dist for dev -if ""%CONF_DEFAULT%""==""etc/conf/dev"" ( - set "TPP_DIR_DEV=thirdparty/dev" -) - -@rem # default supported version for Python 3 -set SUPPORTED_PYTHON3=3.6 - -@rem ################################# - -@rem python --version -@rem python -c "import sys;print(sys.executable)" - @rem Current directory where this .bat files lives set CFG_ROOT_DIR=%~dp0 - @rem path where a configured Python should live in the current virtualenv if installed -set CONFIGURED_PYTHON=%CFG_ROOT_DIR%Scripts\python.exe - +set CONFIGURED_PYTHON=%CFG_ROOT_DIR%tmp\Scripts\python.exe set PYTHON_EXECUTABLE= @rem parse command line options and arguments @@ -56,13 +27,6 @@ if "%1" EQU "--help" (goto cli_help) if "%1" EQU "--clean" (call rmdir /s /q "%CFG_ROOT_DIR%tmp") && call exit /b if "%1" EQU "--python" (set PROVIDED_PYTHON=%~2) && shift && shift && goto collectopts -@rem We are not cleaning: Either we have a provided configure config path or we use a default. -if ""%1""=="""" ( - set CFG_CMD_LINE_ARGS=%CONF_DEFAULT% -) else ( - set CFG_CMD_LINE_ARGS=%1 -) - @rem If we have a pre-configured Python in our virtualenv, reuse this as-is and run if exist ""%CONFIGURED_PYTHON%"" ( set PYTHON_EXECUTABLE=%CONFIGURED_PYTHON% @@ -83,7 +47,6 @@ if exist ""%PROVIDED_PYTHON%"" ( @rem otherwise we search for a suitable Python interpreter :find_python - @rem First check the existence of the "py" launcher (available in Python 3) @rem if we have it, check if we have a py -3 installed with the good version or a py 2.7 @rem if not, check if we have an old py 2.7 @@ -116,8 +79,8 @@ if %ERRORLEVEL% == 0 ( ) ) -:run +:run @rem without this things may not always work on Windows 10, but this makes things slower set PYTHONDONTWRITEBYTECODE=1 @@ -141,9 +104,6 @@ echo A configuration script for Windows echo usage: configure [options] [path/to/config/directory] echo. echo The options and arguments are: -echo [path/to/config/directory] : this optionally sets the path to a -echo configuration directory to use. Defaults to etc/conf/dev if not set -echo. echo --clean : this is exclusive of anything else and cleans the environment echo from built and installed files echo. @@ -155,6 +115,6 @@ echo. :activate @rem Activate the virtualenv -if exist "%CFG_ROOT_DIR%Scripts\activate" ( - "%CFG_ROOT_DIR%Scripts\activate" +if exist "%CFG_ROOT_DIR%tmp\Scripts\activate" ( + "%CFG_ROOT_DIR%tmp\Scripts\activate" ) From 0f293cb11374d96df99757ce8dc6bed4730c9751 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 27 Oct 2020 10:45:16 -0700 Subject: [PATCH 051/160] Install current project in configure.bat Signed-off-by: Jono Yang --- configure.bat | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configure.bat b/configure.bat index cbb42446..958f5bf8 100644 --- a/configure.bat +++ b/configure.bat @@ -21,6 +21,7 @@ set CFG_ROOT_DIR=%~dp0 set CONFIGURED_PYTHON=%CFG_ROOT_DIR%tmp\Scripts\python.exe set PYTHON_EXECUTABLE= + @rem parse command line options and arguments :collectopts if "%1" EQU "--help" (goto cli_help) @@ -89,7 +90,7 @@ call curl -o "%CFG_ROOT_DIR%tmp\virtualenv.pyz" https://bootstrap.pypa.io/virtua call %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%tmp\virtualenv.pyz" "%CFG_ROOT_DIR%tmp" call "%CFG_ROOT_DIR%tmp\Scripts\activate" call "%CFG_ROOT_DIR%tmp\Scripts\pip" install --upgrade pip virtualenv setuptools wheel - +call "%CFG_ROOT_DIR%tmp\Scripts\pip" install -e .[testing] @rem Return a proper return code on failure if %ERRORLEVEL% neq 0 ( From 63f6946e1b3b070924b156a86b0ed0c3da6b7a48 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 27 Oct 2020 11:41:51 -0700 Subject: [PATCH 052/160] Call pytest from proper location * Remove rerun option from azure-pipelines.yml Signed-off-by: Jono Yang --- azure-pipelines.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 904ac906..cf84da29 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -16,7 +16,7 @@ jobs: image_name: ubuntu-16.04 python_versions: ['3.6'] test_suites: - all: bin/py.test -n 2 -vvs --reruns=3 + all: tmp/bin/pytest -n 2 -vvs - template: etc/ci/azure-mac.yml parameters: @@ -24,7 +24,7 @@ jobs: image_name: macos-10.15 python_versions: ['3.6'] test_suites: - all: bin/py.test -n 2 -vvs --reruns=3 + all: tmp/bin/pytest -n 2 -vvs - template: etc/ci/azure-win.yml parameters: @@ -33,7 +33,7 @@ jobs: python_versions: ['3.6'] python_architecture: x86 test_suites: - all: Scripts\py.test -vvs --reruns=3 + all: tmp\Scripts\pytest -vvs - template: etc/ci/azure-win.yml parameters: @@ -42,4 +42,4 @@ jobs: python_versions: ['3.6'] python_architecture: x64 test_suites: - misc: Scripts\py.test -vvs --reruns=3 + misc: tmp\Scripts\pytest -vvs From 7fd250691c0f9db772e289d3cfb2224d70f3dcd0 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 27 Oct 2020 12:43:10 -0700 Subject: [PATCH 053/160] Use newer VM images on Azure Signed-off-by: Jono Yang --- azure-pipelines.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cf84da29..fad6928d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -12,8 +12,8 @@ jobs: - template: etc/ci/azure-linux.yml parameters: - job_name: vm_ubuntu16_py36 - image_name: ubuntu-16.04 + job_name: ubuntu18_py36 + image_name: ubuntu-18.04 python_versions: ['3.6'] test_suites: all: tmp/bin/pytest -n 2 -vvs @@ -28,8 +28,8 @@ jobs: - template: etc/ci/azure-win.yml parameters: - job_name: Win2016_32_py36 - image_name: vs2017-win2016 + job_name: win2019_32_py36 + image_name: windows-2019 python_versions: ['3.6'] python_architecture: x86 test_suites: @@ -37,9 +37,9 @@ jobs: - template: etc/ci/azure-win.yml parameters: - job_name: Win2016_64_py36 - image_name: vs2017-win2016 + job_name: win2019_64_py36 + image_name: windows-2019 python_versions: ['3.6'] python_architecture: x64 test_suites: - misc: tmp\Scripts\pytest -vvs + all: tmp\Scripts\pytest -vvs From bceb8f98633bda15982c667848dc86be19ee6f97 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 28 Oct 2020 17:03:27 -0700 Subject: [PATCH 054/160] Add .gitattributes * We have this to ensure the line ending of configure.bat is always CRLF Signed-off-by: Jono Yang --- .gitattributes | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..2d555b20 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Set configure.bat's line ending to CRLF. Sometimes batch scripts don't work +# properly on Windows if the line ending is LF and not CRLF +configure.bat eol=crlf From e772fe670da017a9ff51e3b7119996f12169e79c Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 3 Nov 2020 19:32:39 -0800 Subject: [PATCH 055/160] Clean template Signed-off-by: Jono Yang --- etc/ci/azure-linux.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/etc/ci/azure-linux.yml b/etc/ci/azure-linux.yml index 2e12e5b0..752ae2e6 100644 --- a/etc/ci/azure-linux.yml +++ b/etc/ci/azure-linux.yml @@ -1,6 +1,6 @@ parameters: job_name: '' - image_name: 'ubuntu-16.04' + image_name: '' python_versions: [] test_suites: {} python_architecture: x64 @@ -19,7 +19,6 @@ jobs: python_version: ${{ pyver }} test_suite_label: ${{ tsuite.key }} test_suite: ${{ tsuite.value }} - steps: - checkout: self fetchDepth: 10 From 629abedc035290810b4efa30b3e2cc47951f2344 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 5 Nov 2020 16:36:20 +0530 Subject: [PATCH 056/160] Update .gitignore to ignore Jupyter temp files Signed-off-by: Ayan Sinha Mahapatra --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 0abbef13..68de2d22 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,6 @@ pyvenv.cfg /.pytest_cache/ lib64 tcl + +# Ignore Jupyter Notebook related temp files +.ipynb_checkpoints/ From ef210cd813de2961fe8ca4a5ec7f14532ea6e9f8 Mon Sep 17 00:00:00 2001 From: Steven Esser Date: Mon, 16 Nov 2020 15:29:48 -0500 Subject: [PATCH 057/160] Quick doc update Signed-off-by: Steven Esser --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 0f1585bc..a0e682f3 100644 --- a/README.rst +++ b/README.rst @@ -17,6 +17,9 @@ A brand new project cd my-new-repo git pull git@github.com:nexB/skeleton + # Create the new repo on GitHub, then update your remote + git remote set-url origin git@github.com:nexB/your-new-repo.git + From here, you can make the appropriate changes to the files for your specific project. Update an existing project From 0b6caf92fe394bec7f9f43a338dc0e2d3a32d5df Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 10 Dec 2020 14:17:12 +0530 Subject: [PATCH 058/160] Add RTD docs configuration file Adds a RTD configuration file (v2) to customize builds. Signed-off-by: Ayan Sinha Mahapatra --- .readthedocs.yml | 18 ++++++++++++++++++ setup.cfg | 4 ++++ 2 files changed, 22 insertions(+) create mode 100644 .readthedocs.yml diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..1b71cd9e --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,18 @@ +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Where the Sphinx conf.py file is located +sphinx: + configuration: docs/source/conf.py + +# Setting the python version and doc build requirements +python: + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/setup.cfg b/setup.cfg index a7ab2fe6..e4274bb0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,3 +35,7 @@ testing = # upstream pytest >= 6 pytest-xdist >= 2 +docs= + Sphinx>=3.3.1 + sphinx-rtd-theme>=0.5.0 + doc8>=0.8.1 \ No newline at end of file From f2c1400e39aa99b2a53392910d01509a0a8114f7 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 10 Dec 2020 14:19:15 +0530 Subject: [PATCH 059/160] Add basic RTD documentaion #4 Signed-off-by: Ayan Sinha Mahapatra --- docs/Makefile | 20 +++++++++++ docs/make.bat | 35 +++++++++++++++++++ docs/source/conf.py | 63 ++++++++++++++++++++++++++++++++++ docs/source/index.rst | 15 ++++++++ docs/source/skeleton/index.rst | 15 ++++++++ 5 files changed, 148 insertions(+) create mode 100644 docs/Makefile create mode 100644 docs/make.bat create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst create mode 100644 docs/source/skeleton/index.rst diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..d0c3cbf1 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 00000000..6247f7e2 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 00000000..529cae39 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,63 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'nexb-skeleton' +copyright = 'nexb Inc.' +author = 'nexb Inc.' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +html_context = { + 'css_files': [ + '_static/theme_overrides.css', # override wide tables in RTD theme + ], + "display_github": True, + "github_user": "nexB", + "github_repo": "nexb-skeleton", + "github_version": "develop", # branch + "conf_py_path": "/docs/source/", # path in the checkout to the docs root + } \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 00000000..67fcf213 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,15 @@ +Welcome to nexb-skeleton's documentation! +========================================= + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + skeleton/index + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/source/skeleton/index.rst b/docs/source/skeleton/index.rst new file mode 100644 index 00000000..7dfc6cb4 --- /dev/null +++ b/docs/source/skeleton/index.rst @@ -0,0 +1,15 @@ +# Docs Structure Guide +# Rst docs - https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html +# +# 1. Place docs in folders under source for different sections +# 2. Link them by adding individual index files in each section +# to the main index, and then files for each section to their +# respective index files. +# 3. Use `.. include` statements to include other .rst files +# or part of them, or use hyperlinks to a section of the docs, +# to get rid of repetition. +# https://docutils.sourceforge.io/docs/ref/rst/directives.html#including-an-external-document-fragment +# +# Note: Replace these guide/placeholder docs + +.. include:: ../../../README.rst From e7d19903edae48c176baab11a9a5b7393ae74854 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 10 Dec 2020 14:20:49 +0530 Subject: [PATCH 060/160] Add RTD requirements file and test scripts Signed-off-by: Ayan Sinha Mahapatra --- docs/scripts/doc8_style_check.sh | 5 +++++ docs/scripts/sphinx_build_link_check.sh | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 docs/scripts/doc8_style_check.sh create mode 100644 docs/scripts/sphinx_build_link_check.sh diff --git a/docs/scripts/doc8_style_check.sh b/docs/scripts/doc8_style_check.sh new file mode 100644 index 00000000..94163239 --- /dev/null +++ b/docs/scripts/doc8_style_check.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# halt script on error +set -e +# Check for Style Code Violations +doc8 --max-line-length 100 source --ignore D000 --quiet \ No newline at end of file diff --git a/docs/scripts/sphinx_build_link_check.sh b/docs/scripts/sphinx_build_link_check.sh new file mode 100644 index 00000000..c5426863 --- /dev/null +++ b/docs/scripts/sphinx_build_link_check.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# halt script on error +set -e +# Build locally, and then check links +sphinx-build -E -W -b linkcheck source build \ No newline at end of file From 393b348d305a924af7566600f54941ea28e24e2d Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Mon, 14 Dec 2020 14:46:48 -0700 Subject: [PATCH 061/160] CPV: implement rich comparison methods directly --- src/pkgcore/ebuild/cpv.py | 54 +++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index 8a507f32..1a7f2257 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -4,7 +4,6 @@ from snakeoil.compatibility import cmp from snakeoil.demandload import demand_compile_regexp -from snakeoil.klass import inject_richcmp_methods_from_cmp from pkgcore.ebuild import atom from pkgcore.ebuild.errors import InvalidCPV @@ -245,7 +244,6 @@ class CPV(base.base): """ __slots__ = ("cpvstr", "key", "category", "package", "version", "revision", "fullver") - inject_richcmp_methods_from_cmp(locals()) def __init__(self, *args, versioned=None): """ @@ -339,26 +337,50 @@ def __repr__(self): def __str__(self): return getattr(self, 'cpvstr', 'None') - def __cmp__(self, other): + def __eq__(self, other): try: if self.cpvstr == other.cpvstr: - return 0 + return True + if self.category == other.category and self.package == other.package: + return ver_cmp(self.version, self.revision, other.version, other.revision) == 0 + except AttributeError: + pass + return False - if (self.category and other.category and self.category != other.category): - return cmp(self.category, other.category) + def __ne__(self, other): + return not self.__eq__(other) - if self.package and other.package and self.package != other.package: - return cmp(self.package, other.package) + def __lt__(self, other): + try: + if self.category == other.category: + if self.package == other.package: + return ver_cmp(self.version, self.revision, other.version, other.revision) < 0 + return self.package < other.package + return self.category < other.category + except AttributeError: + raise TypeError( + "'<' not supported between instances of " + f"{self.__class__.__name__!r} and {other.__class__.__name__!r}" + ) - # note I chucked out valueerror, none checks on versions - # passed in. I suck, I know. - # ~harring - # fails in doing comparison of unversioned atoms against - # versioned atoms - return ver_cmp( - self.version, self.revision, other.version, other.revision) + def __le__(self, other): + return self.__lt__(other) or self.__eq__(other) + + def __gt__(self, other): + try: + if self.category == other.category: + if self.package == other.package: + return ver_cmp(self.version, self.revision, other.version, other.revision) > 0 + return self.package > other.package + return self.category > other.category except AttributeError: - return 1 + raise TypeError( + "'>' not supported between instances of " + f"{self.__class__.__name__!r} and {other.__class__.__name__!r}" + ) + + def __ge__(self, other): + return self.__gt__(other) or self.__eq__(other) @property def versioned_atom(self): From a70f9a0a33c53f402ac68df3d20053595cc7fdf2 Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Mon, 14 Dec 2020 15:27:02 -0700 Subject: [PATCH 062/160] CPV: fully implement __le__() and __ge__() --- src/pkgcore/ebuild/cpv.py | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index 1a7f2257..b0fb5794 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -352,11 +352,9 @@ def __ne__(self, other): def __lt__(self, other): try: - if self.category == other.category: - if self.package == other.package: - return ver_cmp(self.version, self.revision, other.version, other.revision) < 0 - return self.package < other.package - return self.category < other.category + if self.category == other.category and self.package == other.package: + return ver_cmp(self.version, self.revision, other.version, other.revision) < 0 + return self.package < other.package or self.category < other.category except AttributeError: raise TypeError( "'<' not supported between instances of " @@ -364,15 +362,21 @@ def __lt__(self, other): ) def __le__(self, other): - return self.__lt__(other) or self.__eq__(other) + try: + if self.category > other.category and self.package > other.package: + return ver_cmp(self.version, self.revision, other.version, other.revision) <= 0 + return True + except AttributeError: + raise TypeError( + "'<=' not supported between instances of " + f"{self.__class__.__name__!r} and {other.__class__.__name__!r}" + ) def __gt__(self, other): try: - if self.category == other.category: - if self.package == other.package: - return ver_cmp(self.version, self.revision, other.version, other.revision) > 0 - return self.package > other.package - return self.category > other.category + if self.category == other.category and self.package == other.package: + return ver_cmp(self.version, self.revision, other.version, other.revision) > 0 + return self.package > other.package or self.category > other.category except AttributeError: raise TypeError( "'>' not supported between instances of " @@ -380,7 +384,15 @@ def __gt__(self, other): ) def __ge__(self, other): - return self.__gt__(other) or self.__eq__(other) + try: + if self.category < other.category and self.package < other.package: + return ver_cmp(self.version, self.revision, other.version, other.revision) >= 0 + return True + except AttributeError: + raise TypeError( + "'>=' not supported between instances of " + f"{self.__class__.__name__!r} and {other.__class__.__name__!r}" + ) @property def versioned_atom(self): From bd545fe8f26ad6ec40af050ffe4a672371a000c5 Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Mon, 14 Dec 2020 19:40:31 -0700 Subject: [PATCH 063/160] use relative imports --- src/pkgcore/ebuild/cpv.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index b0fb5794..49c9e88a 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -5,9 +5,9 @@ from snakeoil.compatibility import cmp from snakeoil.demandload import demand_compile_regexp -from pkgcore.ebuild import atom -from pkgcore.ebuild.errors import InvalidCPV -from pkgcore.package import base +from . import atom +from .errors import InvalidCPV +from ..package import base demand_compile_regexp( From 9617d3cdfc9828738c2efa4e3403aac965f25148 Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Mon, 14 Dec 2020 21:05:36 -0700 Subject: [PATCH 064/160] sort imports with isort --- src/pkgcore/ebuild/cpv.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index 49c9e88a..692101d4 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -5,10 +5,9 @@ from snakeoil.compatibility import cmp from snakeoil.demandload import demand_compile_regexp +from ..package import base from . import atom from .errors import InvalidCPV -from ..package import base - demand_compile_regexp( 'suffix_regexp', '^(alpha|beta|rc|pre|p)(\\d*)$') From 7d37af0b6636ab23780503a0e8de8287ab282b66 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 16 Dec 2020 19:45:34 +0530 Subject: [PATCH 065/160] Add `src` folder to pass CI tests and RTD builds Signed-off-by: Ayan Sinha Mahapatra --- src/README.rst | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 src/README.rst diff --git a/src/README.rst b/src/README.rst new file mode 100644 index 00000000..efb1a14a --- /dev/null +++ b/src/README.rst @@ -0,0 +1,5 @@ +Package Module +-------------- + +Put your python modules in this directory. + From 3f891951a11ac276e75fea3d183c4bf7316d0e53 Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Wed, 16 Dec 2020 22:11:47 -0700 Subject: [PATCH 066/160] ebuild.cpv: fix sorting logic --- src/pkgcore/ebuild/cpv.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index 692101d4..6b6e7dd1 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -351,9 +351,11 @@ def __ne__(self, other): def __lt__(self, other): try: - if self.category == other.category and self.package == other.package: - return ver_cmp(self.version, self.revision, other.version, other.revision) < 0 - return self.package < other.package or self.category < other.category + if self.category == other.category: + if self.package == other.package: + return ver_cmp(self.version, self.revision, other.version, other.revision) < 0 + return self.package < other.package + return self.category < other.category except AttributeError: raise TypeError( "'<' not supported between instances of " @@ -373,9 +375,11 @@ def __le__(self, other): def __gt__(self, other): try: - if self.category == other.category and self.package == other.package: - return ver_cmp(self.version, self.revision, other.version, other.revision) > 0 - return self.package > other.package or self.category > other.category + if self.category == other.category: + if self.package == other.package: + return ver_cmp(self.version, self.revision, other.version, other.revision) > 0 + return self.package > other.package + return self.category > other.category except AttributeError: raise TypeError( "'>' not supported between instances of " From 8ed14469dca245b198e2886e5120f0142167cecb Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Thu, 17 Dec 2020 10:37:15 -0700 Subject: [PATCH 067/160] ebuild.cpv: fix __le__() and __ge__() --- src/pkgcore/ebuild/cpv.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index 6b6e7dd1..7541240d 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -364,9 +364,11 @@ def __lt__(self, other): def __le__(self, other): try: - if self.category > other.category and self.package > other.package: - return ver_cmp(self.version, self.revision, other.version, other.revision) <= 0 - return True + if self.category == other.category: + if self.package == other.package: + return ver_cmp(self.version, self.revision, other.version, other.revision) <= 0 + return self.package < other.package + return self.category < other.category except AttributeError: raise TypeError( "'<=' not supported between instances of " @@ -388,9 +390,11 @@ def __gt__(self, other): def __ge__(self, other): try: - if self.category < other.category and self.package < other.package: - return ver_cmp(self.version, self.revision, other.version, other.revision) >= 0 - return True + if self.category == other.category: + if self.package == other.package: + return ver_cmp(self.version, self.revision, other.version, other.revision) >= 0 + return self.package > other.package + return self.category > other.category except AttributeError: raise TypeError( "'>=' not supported between instances of " From 7b8d3d5e204dc206782e9061b0016ac2c2898a0a Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Fri, 25 Dec 2020 14:25:35 -0700 Subject: [PATCH 068/160] ebuild.cpv: mark Revision as a public class Since it's used in pkgcheck. --- src/pkgcore/ebuild/cpv.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index 7541240d..883851dd 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -44,7 +44,7 @@ def isvalid_rev(s): return s and s[0] == 'r' and s[1:].isdigit() -class _Revision(UserString): +class Revision(UserString): """Internal revision class storing revisions as strings and comparing as integers.""" # parent __hash__() isn't inherited when __eq__() is defined in the child class @@ -65,7 +65,7 @@ def __str__(self): return self.data def __eq__(self, other): - if isinstance(other, _Revision): + if isinstance(other, Revision): return self._revint == other._revint elif isinstance(other, int): return self._revint == other @@ -74,7 +74,7 @@ def __eq__(self, other): return self.data == other def __lt__(self, other): - if isinstance(other, _Revision): + if isinstance(other, Revision): return self._revint < other._revint elif isinstance(other, int): return self._revint < other @@ -83,7 +83,7 @@ def __lt__(self, other): return self.data < other def __le__(self, other): - if isinstance(other, _Revision): + if isinstance(other, Revision): return self._revint <= other._revint elif isinstance(other, int): return self._revint <= other @@ -92,7 +92,7 @@ def __le__(self, other): return self.data <= other def __gt__(self, other): - if isinstance(other, _Revision): + if isinstance(other, Revision): return self._revint > other._revint elif isinstance(other, int): return self._revint > other @@ -101,7 +101,7 @@ def __gt__(self, other): return self.data > other def __ge__(self, other): - if isinstance(other, _Revision): + if isinstance(other, Revision): return self._revint >= other._revint elif isinstance(other, int): return self._revint >= other @@ -294,7 +294,7 @@ def __init__(self, *args, versioned=None): # needs at least ('pkg', 'ver', 'rev') raise InvalidCPV( cpvstr, 'missing package name, version, and/or revision') - rev = _Revision(pkg_chunks.pop(-1)[1:]) + rev = Revision(pkg_chunks.pop(-1)[1:]) if rev == 0: # reset stored cpvstr to drop -r0+ sf(self, 'cpvstr', f"{category}/{'-'.join(pkg_chunks)}") @@ -303,7 +303,7 @@ def __init__(self, *args, versioned=None): sf(self, 'cpvstr', f"{category}/{'-'.join(pkg_chunks)}-r{int(rev)}") sf(self, 'revision', rev) else: - sf(self, 'revision', _Revision('')) + sf(self, 'revision', Revision('')) if not isvalid_version_re.match(pkg_chunks[-1]): raise InvalidCPV(cpvstr, f"invalid version '{pkg_chunks[-1]}'") From 03ffc8a23606b22ad728bb093e52264bfe1af658 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 6 Jan 2021 18:54:31 +0100 Subject: [PATCH 069/160] Ensure we use official full text of Apache 2.0 Taken from https://www.apache.org/licenses/LICENSE-2.0.txt Signed-off-by: Philippe Ombredanne --- apache-2.0.LICENSE | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/apache-2.0.LICENSE b/apache-2.0.LICENSE index d9a10c0d..261eeb9e 100644 --- a/apache-2.0.LICENSE +++ b/apache-2.0.LICENSE @@ -174,3 +174,28 @@ of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From 5e386d9ff0bfe77c170e37abde84313aa45b3a3f Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 14 Jan 2021 17:04:37 +0100 Subject: [PATCH 070/160] Never ever let Git convert line delimiters Signed-off-by: Philippe Ombredanne --- .gitattributes | 5 +- configure.bat | 242 ++++++++++++++++++++++++------------------------- 2 files changed, 123 insertions(+), 124 deletions(-) diff --git a/.gitattributes b/.gitattributes index 2d555b20..c446d380 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,2 @@ -# Set configure.bat's line ending to CRLF. Sometimes batch scripts don't work -# properly on Windows if the line ending is LF and not CRLF -configure.bat eol=crlf +# Ignore all Git auto CR/LF line endings conversions +* binary diff --git a/configure.bat b/configure.bat index 958f5bf8..f03ea07c 100644 --- a/configure.bat +++ b/configure.bat @@ -1,121 +1,121 @@ -@echo OFF -@setlocal -@rem Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. - -@rem ################################ -@rem # A configuration script for Windows -@rem # -@rem # The options and (optional) arguments are: -@rem # --clean : this is exclusive of anything else and cleans the environment -@rem # from built and installed files -@rem # -@rem # --python < path to python.exe> : this must be the first argument and set -@rem # the path to the Python executable to use. If < path to python.exe> is -@rem # set to "path", then the executable will be the python.exe available -@rem # in the PATH. -@rem ################################ - -@rem Current directory where this .bat files lives -set CFG_ROOT_DIR=%~dp0 -@rem path where a configured Python should live in the current virtualenv if installed -set CONFIGURED_PYTHON=%CFG_ROOT_DIR%tmp\Scripts\python.exe -set PYTHON_EXECUTABLE= - - -@rem parse command line options and arguments -:collectopts -if "%1" EQU "--help" (goto cli_help) -if "%1" EQU "--clean" (call rmdir /s /q "%CFG_ROOT_DIR%tmp") && call exit /b -if "%1" EQU "--python" (set PROVIDED_PYTHON=%~2) && shift && shift && goto collectopts - -@rem If we have a pre-configured Python in our virtualenv, reuse this as-is and run -if exist ""%CONFIGURED_PYTHON%"" ( - set PYTHON_EXECUTABLE=%CONFIGURED_PYTHON% - goto run -) - -@rem If we have a command arg for Python use this as-is -if ""%PROVIDED_PYTHON%""==""path"" ( - @rem use a bare python available in the PATH - set PYTHON_EXECUTABLE=python - goto run -) -if exist ""%PROVIDED_PYTHON%"" ( - set PYTHON_EXECUTABLE=%PROVIDED_PYTHON% - goto run -) - - -@rem otherwise we search for a suitable Python interpreter -:find_python -@rem First check the existence of the "py" launcher (available in Python 3) -@rem if we have it, check if we have a py -3 installed with the good version or a py 2.7 -@rem if not, check if we have an old py 2.7 -@rem exist if all fails - -where py >nul 2>nul -if %ERRORLEVEL% == 0 ( - @rem we have a py launcher, check for the availability of our required Python 3 version - py -3.6 --version >nul 2>nul - if %ERRORLEVEL% == 0 ( - set PYTHON_EXECUTABLE=py -3.6 - ) else ( - @rem we have no required python 3, let's try python 2: - py -2 --version >nul 2>nul - if %ERRORLEVEL% == 0 ( - set PYTHON_EXECUTABLE=py -2 - ) else ( - @rem we have py and no python 3 and 2, exit - echo * Unable to find an installation of Python. - exit /b 1 - ) - ) -) else ( - @rem we have no py launcher, check for a default Python 2 installation - if not exist ""%DEFAULT_PYTHON2%"" ( - echo * Unable to find an installation of Python. - exit /b 1 - ) else ( - set PYTHON_EXECUTABLE=%DEFAULT_PYTHON2% - ) -) - - -:run -@rem without this things may not always work on Windows 10, but this makes things slower -set PYTHONDONTWRITEBYTECODE=1 - -call mkdir "%CFG_ROOT_DIR%tmp" -call curl -o "%CFG_ROOT_DIR%tmp\virtualenv.pyz" https://bootstrap.pypa.io/virtualenv.pyz -call %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%tmp\virtualenv.pyz" "%CFG_ROOT_DIR%tmp" -call "%CFG_ROOT_DIR%tmp\Scripts\activate" -call "%CFG_ROOT_DIR%tmp\Scripts\pip" install --upgrade pip virtualenv setuptools wheel -call "%CFG_ROOT_DIR%tmp\Scripts\pip" install -e .[testing] - -@rem Return a proper return code on failure -if %ERRORLEVEL% neq 0 ( - exit /b %ERRORLEVEL% -) -endlocal -goto activate - - -:cli_help -echo A configuration script for Windows -echo usage: configure [options] [path/to/config/directory] -echo. -echo The options and arguments are: -echo --clean : this is exclusive of anything else and cleans the environment -echo from built and installed files -echo. -echo --python path/to/python.exe : this is set to the path of an alternative -echo Python executable to use. If path/to/python.exe is set to "path", -echo then the executable will be the python.exe available in the PATH. -echo. - - -:activate -@rem Activate the virtualenv -if exist "%CFG_ROOT_DIR%tmp\Scripts\activate" ( - "%CFG_ROOT_DIR%tmp\Scripts\activate" -) +@echo OFF +@setlocal +@rem Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. + +@rem ################################ +@rem # A configuration script for Windows +@rem # +@rem # The options and (optional) arguments are: +@rem # --clean : this is exclusive of anything else and cleans the environment +@rem # from built and installed files +@rem # +@rem # --python < path to python.exe> : this must be the first argument and set +@rem # the path to the Python executable to use. If < path to python.exe> is +@rem # set to "path", then the executable will be the python.exe available +@rem # in the PATH. +@rem ################################ + +@rem Current directory where this .bat files lives +set CFG_ROOT_DIR=%~dp0 +@rem path where a configured Python should live in the current virtualenv if installed +set CONFIGURED_PYTHON=%CFG_ROOT_DIR%tmp\Scripts\python.exe +set PYTHON_EXECUTABLE= + + +@rem parse command line options and arguments +:collectopts +if "%1" EQU "--help" (goto cli_help) +if "%1" EQU "--clean" (call rmdir /s /q "%CFG_ROOT_DIR%tmp") && call exit /b +if "%1" EQU "--python" (set PROVIDED_PYTHON=%~2) && shift && shift && goto collectopts + +@rem If we have a pre-configured Python in our virtualenv, reuse this as-is and run +if exist ""%CONFIGURED_PYTHON%"" ( + set PYTHON_EXECUTABLE=%CONFIGURED_PYTHON% + goto run +) + +@rem If we have a command arg for Python use this as-is +if ""%PROVIDED_PYTHON%""==""path"" ( + @rem use a bare python available in the PATH + set PYTHON_EXECUTABLE=python + goto run +) +if exist ""%PROVIDED_PYTHON%"" ( + set PYTHON_EXECUTABLE=%PROVIDED_PYTHON% + goto run +) + + +@rem otherwise we search for a suitable Python interpreter +:find_python +@rem First check the existence of the "py" launcher (available in Python 3) +@rem if we have it, check if we have a py -3 installed with the good version or a py 2.7 +@rem if not, check if we have an old py 2.7 +@rem exist if all fails + +where py >nul 2>nul +if %ERRORLEVEL% == 0 ( + @rem we have a py launcher, check for the availability of our required Python 3 version + py -3.6 --version >nul 2>nul + if %ERRORLEVEL% == 0 ( + set PYTHON_EXECUTABLE=py -3.6 + ) else ( + @rem we have no required python 3, let's try python 2: + py -2 --version >nul 2>nul + if %ERRORLEVEL% == 0 ( + set PYTHON_EXECUTABLE=py -2 + ) else ( + @rem we have py and no python 3 and 2, exit + echo * Unable to find an installation of Python. + exit /b 1 + ) + ) +) else ( + @rem we have no py launcher, check for a default Python 2 installation + if not exist ""%DEFAULT_PYTHON2%"" ( + echo * Unable to find an installation of Python. + exit /b 1 + ) else ( + set PYTHON_EXECUTABLE=%DEFAULT_PYTHON2% + ) +) + + +:run +@rem without this things may not always work on Windows 10, but this makes things slower +set PYTHONDONTWRITEBYTECODE=1 + +call mkdir "%CFG_ROOT_DIR%tmp" +call curl -o "%CFG_ROOT_DIR%tmp\virtualenv.pyz" https://bootstrap.pypa.io/virtualenv.pyz +call %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%tmp\virtualenv.pyz" "%CFG_ROOT_DIR%tmp" +call "%CFG_ROOT_DIR%tmp\Scripts\activate" +call "%CFG_ROOT_DIR%tmp\Scripts\pip" install --upgrade pip virtualenv setuptools wheel +call "%CFG_ROOT_DIR%tmp\Scripts\pip" install -e .[testing] + +@rem Return a proper return code on failure +if %ERRORLEVEL% neq 0 ( + exit /b %ERRORLEVEL% +) +endlocal +goto activate + + +:cli_help +echo A configuration script for Windows +echo usage: configure [options] [path/to/config/directory] +echo. +echo The options and arguments are: +echo --clean : this is exclusive of anything else and cleans the environment +echo from built and installed files +echo. +echo --python path/to/python.exe : this is set to the path of an alternative +echo Python executable to use. If path/to/python.exe is set to "path", +echo then the executable will be the python.exe available in the PATH. +echo. + + +:activate +@rem Activate the virtualenv +if exist "%CFG_ROOT_DIR%tmp\Scripts\activate" ( + "%CFG_ROOT_DIR%tmp\Scripts\activate" +) From 2e48dca222b6f5889a3874ed5cda5476a7e9ff9b Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 14 Jan 2021 17:05:30 +0100 Subject: [PATCH 071/160] Run tests on more OSes and Python versions Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 55 ++++++++++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fad6928d..9a4c950c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,45 +1,64 @@ ################################################################################ -# We use Azure to run the full tests suites on Python 3.6 -# on Windows (32 and 64), macOS and Linux (64 various distro) +# We use Azure to run the full tests suites on multiple Python 3.x +# on multiple Windows, macOS and Linux versions all on 64 bits +# These jobs are using VMs with Azure-provided Python builds ################################################################################ jobs: -################################################################################ -# These jobs are using VMs and Azure-provided Python 3.6 -################################################################################ + - template: etc/ci/azure-linux.yml + parameters: + job_name: ubuntu16_cpython + image_name: ubuntu-16.04 + python_versions: ['3.6', '3.7', '3.8', '3.9'] + test_suites: + all: tmp/bin/pytest -vvs - template: etc/ci/azure-linux.yml parameters: - job_name: ubuntu18_py36 + job_name: ubuntu18_cpython image_name: ubuntu-18.04 - python_versions: ['3.6'] + python_versions: ['3.6', '3.7', '3.8', '3.9'] + test_suites: + all: tmp/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-linux.yml + parameters: + job_name: ubuntu20_cpython + image_name: ubuntu-20.04 + python_versions: ['3.6', '3.7', '3.8', '3.9'] + test_suites: + all: tmp/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-mac.yml + parameters: + job_name: macos1014_cpython + image_name: macos-10.14 + python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: all: tmp/bin/pytest -n 2 -vvs - template: etc/ci/azure-mac.yml parameters: - job_name: macos1015_py36 + job_name: macos1015_cpython image_name: macos-10.15 - python_versions: ['3.6'] + python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: all: tmp/bin/pytest -n 2 -vvs - template: etc/ci/azure-win.yml parameters: - job_name: win2019_32_py36 - image_name: windows-2019 - python_versions: ['3.6'] - python_architecture: x86 + job_name: win2016_cpython + image_name: vs2017-win2016 + python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: - all: tmp\Scripts\pytest -vvs + all: tmp\Scripts\pytest -n 2 -vvs - template: etc/ci/azure-win.yml parameters: - job_name: win2019_64_py36 + job_name: win2019_cpython image_name: windows-2019 - python_versions: ['3.6'] - python_architecture: x64 + python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: - all: tmp\Scripts\pytest -vvs + all: tmp\Scripts\pytest -n 2 -vvs From b959539450069bb573653a3d1265a95cb6c6f563 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 14 Jan 2021 17:11:01 +0100 Subject: [PATCH 072/160] Do not make wheel universal Also include more license files Signed-off-by: Philippe Ombredanne --- setup.cfg | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index e4274bb0..f7910848 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,11 +1,15 @@ -[wheel] -universal=1 - [metadata] -license_file = apache-2.0.LICENSE +license_files = + apache-2.0.LICENSE + NOTICE + AUTHORS.rst + CHANGELOG.rst name = skeleton author = nexB. Inc. and others author_email = info@aboutcode.org +license = Apache-2.0 + +# description must be on ONE line https://github.com/pypa/setuptools/issues/1390 description = skeleton long_description = file:README.rst url = https://github.com/nexB/skeleton @@ -17,6 +21,7 @@ classifiers = Topic :: Software Development Topic :: Utilities keywords = + utilities [options] package_dir= From 98641a067009e2b1c08682e99cbae84d30f59d15 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 14 Jan 2021 17:12:00 +0100 Subject: [PATCH 073/160] Format and add license Signed-off-by: Philippe Ombredanne --- configure | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configure b/configure index a35c8c90..8f3a68e3 100755 --- a/configure +++ b/configure @@ -1,6 +1,7 @@ #!/usr/bin/env bash # -# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 # set -e From 9fa3f315b7cc1495889b95d50a15cc738d48887f Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 14 Jan 2021 17:12:13 +0100 Subject: [PATCH 074/160] Format Signed-off-by: Philippe Ombredanne --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 45f160da..bac24a43 100644 --- a/setup.py +++ b/setup.py @@ -3,4 +3,4 @@ import setuptools if __name__ == "__main__": - setuptools.setup() \ No newline at end of file + setuptools.setup() From 0b8cd65580db4ad56b31fc67dd438a006cc66a6b Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 14 Jan 2021 17:12:27 +0100 Subject: [PATCH 075/160] Improve documentation Signed-off-by: Philippe Ombredanne --- src/README.rst | 5 +---- tests/README.rst | 2 ++ 2 files changed, 3 insertions(+), 4 deletions(-) create mode 100644 tests/README.rst diff --git a/src/README.rst b/src/README.rst index efb1a14a..ec651fcf 100644 --- a/src/README.rst +++ b/src/README.rst @@ -1,5 +1,2 @@ -Package Module --------------- - -Put your python modules in this directory. +Put your Python source code (and installable data) in this directory. diff --git a/tests/README.rst b/tests/README.rst new file mode 100644 index 00000000..d94783e6 --- /dev/null +++ b/tests/README.rst @@ -0,0 +1,2 @@ +Put your Python test modules in this directory. + From 300769149b530e283a4e4a6aa7f338ca72e5df5e Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 14 Jan 2021 17:12:48 +0100 Subject: [PATCH 076/160] Format for spaces Signed-off-by: Philippe Ombredanne --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e75f1cec..55fb92c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,11 +36,11 @@ norecursedirs = [ python_files = "*.py" -python_classes="Test" -python_functions="test" +python_classes = "Test" +python_functions = "test" addopts = [ "-rfExXw", "--strict", "--doctest-modules" -] \ No newline at end of file +] From b49895c69abd9713461de61032645cc2c492e73e Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 14 Jan 2021 17:14:47 +0100 Subject: [PATCH 077/160] Add manifest for source distributions Signed-off-by: Philippe Ombredanne --- MANIFEST.in | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..ef3721e8 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,15 @@ +graft src + +include *.LICENSE +include NOTICE +include *.ABOUT +include *.toml +include *.yml +include *.rst +include setup.* +include configure* +include requirements* +include .git* + +global-exclude *.py[co] __pycache__ *.*~ + From d3e2d28d9f6cde0ded3d8450107d14fb4da05c4e Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 14 Jan 2021 17:15:06 +0100 Subject: [PATCH 078/160] Add Apache license NOTICE Signed-off-by: Philippe Ombredanne --- NOTICE | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 NOTICE diff --git a/NOTICE b/NOTICE new file mode 100644 index 00000000..65936b2b --- /dev/null +++ b/NOTICE @@ -0,0 +1,19 @@ +# +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 +# +# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# ScanCode is a trademark of nexB Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# From f46bc48f7b056670f87cbc25e51a0979c7fff8db Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 24 Jan 2021 12:32:27 +0100 Subject: [PATCH 079/160] Default to 64 bits windows on CI Signed-off-by: Philippe Ombredanne --- etc/ci/azure-win.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/ci/azure-win.yml b/etc/ci/azure-win.yml index 62208574..afe1686f 100644 --- a/etc/ci/azure-win.yml +++ b/etc/ci/azure-win.yml @@ -3,7 +3,7 @@ parameters: image_name: '' python_versions: [] test_suites: {} - python_architecture: x86 + python_architecture: x64 jobs: - job: ${{ parameters.job_name }} From 182532f69052c75d3621ff214ede196cbeed16e7 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 25 Jan 2021 12:54:13 +0100 Subject: [PATCH 080/160] Use wheels embedded in virtualenv.pyz Signed-off-by: Philippe Ombredanne --- configure | 2 +- configure.bat | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 8f3a68e3..d41bf8e5 100755 --- a/configure +++ b/configure @@ -27,7 +27,7 @@ function setup { # create a virtualenv on Python mkdir -p $CONFIGURE_ROOT_DIR/tmp wget -O $CONFIGURE_ROOT_DIR/tmp/virtualenv.pyz https://bootstrap.pypa.io/virtualenv.pyz - $PYTHON_EXE $CONFIGURE_ROOT_DIR/tmp/virtualenv.pyz $CONFIGURE_ROOT_DIR/tmp + $PYTHON_EXE $CONFIGURE_ROOT_DIR/tmp/virtualenv.pyz --wheel embed --pip embed --setuptools embed --seeder pip $CONFIGURE_ROOT_DIR/tmp source $CONFIGURE_ROOT_DIR/tmp/bin/activate $CONFIGURE_ROOT_DIR/tmp/bin/pip install --upgrade pip virtualenv setuptools wheel } diff --git a/configure.bat b/configure.bat index f03ea07c..ee68f9e6 100644 --- a/configure.bat +++ b/configure.bat @@ -87,7 +87,7 @@ set PYTHONDONTWRITEBYTECODE=1 call mkdir "%CFG_ROOT_DIR%tmp" call curl -o "%CFG_ROOT_DIR%tmp\virtualenv.pyz" https://bootstrap.pypa.io/virtualenv.pyz -call %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%tmp\virtualenv.pyz" "%CFG_ROOT_DIR%tmp" +call %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%tmp\virtualenv.pyz" --wheel embed --pip embed --setuptools embed --seeder pip "%CFG_ROOT_DIR%tmp" call "%CFG_ROOT_DIR%tmp\Scripts\activate" call "%CFG_ROOT_DIR%tmp\Scripts\pip" install --upgrade pip virtualenv setuptools wheel call "%CFG_ROOT_DIR%tmp\Scripts\pip" install -e .[testing] From cd4e87beb91ea5e9380dfeb19c3530c0a92ff192 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 25 Jan 2021 12:56:18 +0100 Subject: [PATCH 081/160] Do not force an upgrade on virtualenv.pyz embeds Signed-off-by: Philippe Ombredanne --- configure | 1 - configure.bat | 1 - 2 files changed, 2 deletions(-) diff --git a/configure b/configure index d41bf8e5..78e74983 100755 --- a/configure +++ b/configure @@ -29,7 +29,6 @@ function setup { wget -O $CONFIGURE_ROOT_DIR/tmp/virtualenv.pyz https://bootstrap.pypa.io/virtualenv.pyz $PYTHON_EXE $CONFIGURE_ROOT_DIR/tmp/virtualenv.pyz --wheel embed --pip embed --setuptools embed --seeder pip $CONFIGURE_ROOT_DIR/tmp source $CONFIGURE_ROOT_DIR/tmp/bin/activate - $CONFIGURE_ROOT_DIR/tmp/bin/pip install --upgrade pip virtualenv setuptools wheel } diff --git a/configure.bat b/configure.bat index ee68f9e6..00cb1011 100644 --- a/configure.bat +++ b/configure.bat @@ -89,7 +89,6 @@ call mkdir "%CFG_ROOT_DIR%tmp" call curl -o "%CFG_ROOT_DIR%tmp\virtualenv.pyz" https://bootstrap.pypa.io/virtualenv.pyz call %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%tmp\virtualenv.pyz" --wheel embed --pip embed --setuptools embed --seeder pip "%CFG_ROOT_DIR%tmp" call "%CFG_ROOT_DIR%tmp\Scripts\activate" -call "%CFG_ROOT_DIR%tmp\Scripts\pip" install --upgrade pip virtualenv setuptools wheel call "%CFG_ROOT_DIR%tmp\Scripts\pip" install -e .[testing] @rem Return a proper return code on failure From 4b1ebce09e2d06ac97dda5cfd21df89a9c5023c5 Mon Sep 17 00:00:00 2001 From: Panu Matilainen Date: Thu, 4 Feb 2021 15:49:39 +0200 Subject: [PATCH 082/160] Make Lua a hard requirement for rpm More and more macros, scriptlets and other functionality has been getting built around Lua, to the point that it has in practice been required for several years now. Maintaining the pretense of being optional comes at a cost in holding back developments and having to check for that theoretical special case. Lets make it a hard requirement and embrace it some more! --- tests/rpmvercmp.at | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/rpmvercmp.at b/tests/rpmvercmp.at index 1e7c960e..a245a115 100644 --- a/tests/rpmvercmp.at +++ b/tests/rpmvercmp.at @@ -4,7 +4,6 @@ m4_define([RPMVERCMP],[ AT_SETUP([rpmvercmp($1, $2) = $3]) AT_KEYWORDS([vercmp]) AT_CHECK([ -AT_SKIP_IF([$LUA_DISABLED]) runroot rpm --eval '%{lua: print(rpm.vercmp("$1", "$2"))}'], [0], [$3 ], []) AT_CLEANUP From 51510cbdb2f2d066d6652695aed40175a37d88a4 Mon Sep 17 00:00:00 2001 From: Steven Esser Date: Thu, 11 Feb 2021 15:56:55 -0500 Subject: [PATCH 083/160] Fix .gitattributes Signed-off-by: Steven Esser --- .gitattributes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index c446d380..b79df5cd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,2 @@ # Ignore all Git auto CR/LF line endings conversions -* binary +* -text From 2a7bc10c7dd1c1f00d7435c61cad07a9763b4d12 Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Thu, 4 Mar 2021 13:41:41 -0700 Subject: [PATCH 084/160] tests: move to repo root dir --- tests/ebuild/test_cpv.py | 333 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 333 insertions(+) create mode 100644 tests/ebuild/test_cpv.py diff --git a/tests/ebuild/test_cpv.py b/tests/ebuild/test_cpv.py new file mode 100644 index 00000000..ccfe4279 --- /dev/null +++ b/tests/ebuild/test_cpv.py @@ -0,0 +1,333 @@ +from random import shuffle + +import pytest +from snakeoil.compatibility import cmp + +from pkgcore.ebuild import cpv + + +def generate_misc_sufs(): + simple_good_sufs = ["_alpha", "_beta", "_pre", "_p"] + suf_nums = list(range(100)) + shuffle(suf_nums) + + good_sufs = (simple_good_sufs + [f"{x}{suf_nums.pop()}" for x in simple_good_sufs]) + + l = len(good_sufs) + good_sufs = good_sufs + [ + good_sufs[x] + good_sufs[l - x - 1] for x in range(l)] + + bad_sufs = ["_a", "_9", "_"] + [x+" " for x in simple_good_sufs] + return good_sufs, bad_sufs + + +class TestCPV: + + good_cats = ( + "dev-util", "dev+", "dev-util+", "DEV-UTIL", "aaa0", + "aaa-0", "multi/depth", "cross-dev_idiot.hacks-suck", "a") + bad_cats = (".util", "_dev", "", "dev-util ", "multi//depth") + good_pkgs = ("diffball", "a9", "a9+", "a-100dpi", "diff-mode-") + bad_pkgs = ("diffball ", "diffball-9", "a-3D", "ab--df", "-df", "+dfa") + + good_cp = ( + "bbb-9/foon", "dev-util/diffball", "dev-util/diffball-a9", + "dev-ut-asdf/emacs-cvs", "xfce-base/xfce4", "bah/f-100dpi", + "dev-util/diffball-blah-monkeys") + + good_vers = ("1", "2.3.4", "2.3.4a", "02.3", "2.03", "3d", "3D") + bad_vers = ("2.3a.4", "2.a.3", "2.3_", "2.3 ", "2.3.", "cvs.2") + + good_sufs, bad_sufs = generate_misc_sufs() + good_revs = ("-r1", "-r300", "-r0", "", + "-r1000000000000000000") + bad_revs = ("-r", "-ra", "-r", "-R1") + + testing_secondary_args = False + + def make_inst(self, cat, pkg, fullver=""): + if self.testing_secondary_args: + return cpv.CPV(cat, pkg, fullver, versioned=bool(fullver)) + if fullver: + return cpv.VersionedCPV(f"{cat}/{pkg}-{fullver}") + return cpv.UnversionedCPV(f"{cat}/{pkg}") + + def test_simple_key(self): + with pytest.raises(cpv.InvalidCPV): + self.make_inst("da", "ba-3", "3.3") + for src in [[("dev-util", "diffball", "0.7.1"), "dev-util/diffball"], + ["dev-util/diffball"], + ["dev-perl/mod_perl"], + ["dev-perl/mod_p"], + [("dev-perl", "mod-p", ""), "dev-perl/mod-p"], + ["dev-perl/mod-p-1", "dev-perl/mod-p"],]: + if len(src) == 1: + key = src[0] + else: + key = src[1] + if isinstance(src[0], str): + cat, pkgver = src[0].rsplit("/", 1) + vals = pkgver.rsplit("-", 1) + if len(vals) == 1: + pkg = pkgver + ver = '' + else: + pkg, ver = vals + else: + cat, pkg, ver = src[0] + + assert self.make_inst(cat, pkg, ver).key == key + + def test_init(self): + cpv.CPV("dev-util", "diffball", "0.7.1") + cpv.CPV("dev-util", "diffball") + cpv.CPV("dev-util/diffball-0.7.1", versioned=True) + with pytest.raises(TypeError): + cpv.VersionedCPV("dev-util", "diffball", None) + + def test_parsing(self): + # check for gentoo bug 263787 + self.process_pkg(False, 'app-text', 'foo-123-bar') + self.process_ver(False, 'app-text', 'foo-123-bar', '2.0017a_p', '-r5') + with pytest.raises(cpv.InvalidCPV): + cpv.UnversionedCPV('app-text/foo-123') + for cat_ret, cats in [[False, self.good_cats], [True, self.bad_cats]]: + for cat in cats: + for pkg_ret, pkgs in [[False, self.good_pkgs], + [True, self.bad_pkgs]]: + for pkg in pkgs: + self.process_pkg(cat_ret or pkg_ret, cat, pkg) + + for cp in self.good_cp: + cat, pkg = cp.rsplit("/", 1) + for rev_ret, revs in [[False, self.good_revs], + [True, self.bad_revs]]: + for rev in revs: + for ver_ret, vers in [[False, self.good_vers], + [True, self.bad_vers]]: + for ver in vers: + self.process_ver(ver_ret or rev_ret, cat, pkg, + ver, rev) + + for x in (10, 18, 19, 36, 100): + assert cpv.CPV("da", "ba", f"1-r0{'0' * x}").revision == 0 + assert \ + int(cpv.CPV("da", "ba", f"1-r1{'0' * x}1").revision) == int(f"1{'0' * x}1") + + def process_pkg(self, ret, cat, pkg): + if ret: + with pytest.raises(cpv.InvalidCPV): + self.make_inst(cat, pkg) + else: + c = self.make_inst(cat, pkg) + assert c.cpvstr == f"{cat}/{pkg}" + assert c.category == cat + assert c.package == pkg + assert c.key == f"{cat}/{pkg}" + assert c.revision is None + assert c.version is None + assert c.fullver is None + + def process_ver(self, ret, cat, pkg, ver, rev): + if ret: + with pytest.raises(cpv.InvalidCPV): + self.make_inst(cat, pkg, f"{ver}{rev}") + else: + c = self.make_inst(cat, pkg, ver + rev) + if rev == "" or rev == "-r0": + assert c.cpvstr == f"{cat}/{pkg}-{ver}" + assert c.revision == 0 + if rev: + assert c.fullver == ver + rev + else: + assert c.revision == "" + assert c.fullver == ver + else: + assert c.revision == int(rev.lstrip("-r")) + assert c.cpvstr == f"{cat}/{pkg}-{ver}{rev}" + assert c.fullver == ver + rev + assert c.category == cat + assert c.package == pkg + assert c.key == f"{cat}/{pkg}" + assert c.version == ver + + for suf in self.good_sufs: + self.process_suf(ret, cat, pkg, ver + suf, rev) + for bad_suf in self.bad_sufs: + # double process, front and back. + self.process_suf(True, cat, pkg, suf + bad_suf, rev) + self.process_suf(True, cat, pkg, bad_suf + suf, rev) + + for suf in self.bad_sufs: + # check standalone. + self.process_suf(True, cat, pkg, ver+suf, rev) + + def process_suf(self, ret, cat, pkg, ver, rev): + if ret: + with pytest.raises(cpv.InvalidCPV): + self.make_inst(cat, pkg, ver+rev) + else: + # redundant in light of process_ver... combine these somehow. + c = self.make_inst(cat, pkg, ver + rev) + if rev == '' or rev == '-r0': + assert c.cpvstr == f"{cat}/{pkg}-{ver}" + assert c.revision == 0 + if rev: + assert c.fullver == ver + rev + else: + assert c.revision == "" + assert c.fullver == ver + else: + assert c.cpvstr == f"{cat}/{pkg}-{ver}{rev}" + assert c.revision == int(rev.lstrip("-r")) + assert c.fullver == ver + rev + assert c.category == cat + assert c.package == pkg + assert c.key == f"{cat}/{pkg}" + assert c.version == ver + + def test_cmp(self): + ukls, vkls = cpv.UnversionedCPV, cpv.VersionedCPV + assert vkls("dev-util/diffball-0.1") < vkls("dev-util/diffball-0.2") + base = "dev-util/diffball-0.7.1" + assert vkls(base) == vkls(base) + for rev in ("", "-r1"): + last = None + for suf in ["_alpha", "_beta", "_pre", "", "_p"]: + if suf == "": + sufs = [suf] + else: + sufs = [suf, f'{suf}4'] + for x in sufs: + cur = vkls(f'{base}{x}{rev}') + assert cur == vkls(f'{base}{x}{rev}') + if last is not None: + assert cur > last + + assert vkls("da/ba-6a") > vkls("da/ba-6") + assert vkls("da/ba-6a-r1") > vkls("da/ba-6a") + assert vkls("da/ba-6.0") > vkls("da/ba-6") + assert vkls("da/ba-6.0.0") > vkls("da/ba-6.0b") + assert vkls("da/ba-6.02") > vkls("da/ba-6.0.0") + # float comparison rules. + assert vkls("da/ba-6.2") > vkls("da/ba-6.054") + assert vkls("da/ba-6") == vkls("da/ba-6") + assert ukls("db/ba") > ukls("da/ba") + assert ukls("da/bb") > ukls("da/ba") + assert vkls("da/ba-6.0_alpha0_p1") > vkls("da/ba-6.0_alpha") + assert vkls("da/ba-6.0_alpha") == vkls("da/ba-6.0_alpha0") + assert vkls("da/ba-6.1") > vkls("da/ba-6.09") + assert vkls("da/ba-6.0.1") > vkls("da/ba-6.0") + assert vkls("da/ba-12.2.5") > vkls("da/ba-12.2b") + + # test for gentoo bug 287848 + assert vkls("dev-lang/erlang-12.2.5") > vkls("dev-lang/erlang-12.2b") + assert vkls("dev-lang/erlang-12.2.5-r1") > vkls("dev-lang/erlang-12.2b") + + # equivalent versions + assert vkls("da/ba-6.01.0") == vkls("da/ba-6.010.0") + assert vkls("da/ba-6.0.1") == vkls("da/ba-6.000.1") + + # equivalent revisions + assert vkls("da/ba-6.01.0") == vkls("da/ba-6.01.0-r0") + assert vkls("da/ba-6.01.0-r0") == vkls("da/ba-6.01.0-r00") + assert vkls("da/ba-6.01.0-r1") == vkls("da/ba-6.01.0-r001") + + for v1, v2 in (("1.001000000000000000001", "1.001000000000000000002"), + ("1.00100000000", "1.0010000000000000001"), + ("1.01", "1.1")): + assert vkls(f"da/ba-{v2}") > vkls(f"da/ba-{v1}") + + for x in (18, 36, 100): + s = "0" * x + assert vkls(f"da/ba-10{s}1") > vkls(f"da/ba-1{s}1") + + for x in (18, 36, 100): + s = "0" * x + assert vkls(f"da/ba-1-r10{s}1") > vkls(f"da/ba-1-r1{s}1") + + assert vkls('sys-apps/net-tools-1.60_p2010081516093') > \ + vkls('sys-apps/net-tools-1.60_p2009072801401') + + assert vkls('sys-apps/net-tools-1.60_p20100815160931') > \ + vkls('sys-apps/net-tools-1.60_p20090728014017') + + assert vkls('sys-apps/net-tools-1.60_p20100815160931') > \ + vkls('sys-apps/net-tools-1.60_p20090728014017-r1') + + # Regression test: python does comparison slightly differently + # if the classes do not match exactly (it prefers rich + # comparison over __cmp__). + class DummySubclass(cpv.CPV): + pass + + assert DummySubclass("da/ba-6.0_alpha0_p1", versioned=True) != vkls("da/ba-6.0_alpha") + assert DummySubclass("da/ba-6.0_alpha0", versioned=True) == vkls("da/ba-6.0_alpha") + + assert DummySubclass("da/ba-6.0", versioned=True) != "foon" + assert DummySubclass("da/ba-6.0", versioned=True) == \ + DummySubclass("da/ba-6.0-r0", versioned=True) + + def test_no_init(self): + """Test if the cpv is in a somewhat sane state if __init__ fails. + + IPython used to segfault when showing a verbose traceback for + a subclass of CPV which raised cpv.InvalidCPV. This checks + if such uninitialized objects survive some basic poking. + """ + uninited = cpv.CPV.__new__(cpv.CPV) + broken = cpv.CPV.__new__(cpv.CPV) + with pytest.raises(cpv.InvalidCPV): + broken.__init__('broken', versioned=True) + for thing in (uninited, broken): + # the c version returns None, the py version does not have the attr + getattr(thing, 'cpvstr', None) + repr(thing) + str(thing) + # The c version returns a constant, the py version raises + try: + hash(thing) + except AttributeError: + pass + + def test_r0_revisions(self): + # single '0' + obj = cpv.CPV("dev-util/diffball-1.0-r0", versioned=True) + assert obj.cpvstr == "dev-util/diffball-1.0" + assert str(obj) == "dev-util/diffball-1.0" + assert obj.fullver == "1.0-r0" + assert obj.revision == 0 + + # multiple '0' + obj = cpv.CPV("dev-util/diffball-1.0-r000", versioned=True) + assert obj.cpvstr == "dev-util/diffball-1.0" + assert str(obj) == "dev-util/diffball-1.0" + assert obj.fullver == "1.0-r000" + assert obj.revision == 0 + + # single '0' prefix + obj = cpv.CPV("dev-util/diffball-1.0-r01", versioned=True) + assert obj.cpvstr == "dev-util/diffball-1.0-r1" + assert str(obj) == "dev-util/diffball-1.0-r1" + assert obj.fullver == "1.0-r01" + assert obj.revision == 1 + + # multiple '0' prefixes + obj = cpv.CPV("dev-util/diffball-1.0-r0001", versioned=True) + assert obj.cpvstr == "dev-util/diffball-1.0-r1" + assert str(obj) == "dev-util/diffball-1.0-r1" + assert obj.fullver == "1.0-r0001" + assert obj.revision == 1 + + def test_attribute_errors(self): + obj = cpv.VersionedCPV("foo/bar-0") + assert not obj == 0 + assert obj != 0 + with pytest.raises(TypeError): + assert obj < 0 + with pytest.raises(TypeError): + assert obj <= 0 + with pytest.raises(TypeError): + assert obj > 0 + with pytest.raises(TypeError): + assert obj >= 0 From d6fe59fd2e832075905ecb27235640a2776dad7a Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 7 May 2021 14:56:42 +0200 Subject: [PATCH 085/160] Update markers syntax for pytest Signed-off-by: Philippe Ombredanne --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 55fb92c3..a3bda449 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,6 @@ python_functions = "test" addopts = [ "-rfExXw", - "--strict", + "--strict-markers", "--doctest-modules" ] From ca6ab2189a6ff6fd093dc9424aa17183a05e6988 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 7 May 2021 14:59:17 +0200 Subject: [PATCH 086/160] Add fallback version for setuptools_scm This will work even from a git archive or when git is not installed. Signed-off-by: Philippe Ombredanne --- .gitattributes | 1 + pyproject.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/.gitattributes b/.gitattributes index b79df5cd..96c89ceb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ # Ignore all Git auto CR/LF line endings conversions * -text +pyproject.toml export-subst diff --git a/pyproject.toml b/pyproject.toml index a3bda449..52caac46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,6 +3,7 @@ requires = ["setuptools >= 50", "wheel", "setuptools_scm[toml] >= 4"] build-backend = "setuptools.build_meta" [tool.setuptools_scm] +fallback_version = "v9999.$Format:%h-%cs$" [tool.pytest.ini_options] norecursedirs = [ From 1364bbbb9c399bd535686ea4ec6bfc241eb0e689 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 11 May 2021 10:57:19 +0200 Subject: [PATCH 087/160] Add note for setuptools_scam fallback version Signed-off-by: Philippe Ombredanne --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 52caac46..8eebe91a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,6 +3,8 @@ requires = ["setuptools >= 50", "wheel", "setuptools_scm[toml] >= 4"] build-backend = "setuptools.build_meta" [tool.setuptools_scm] +# this is used populated when creating a git archive +# and when there is .git dir and/or there is no git installed fallback_version = "v9999.$Format:%h-%cs$" [tool.pytest.ini_options] From be851b017a6e5c98ad85a84cda8b3f070e7acf34 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 11 May 2021 11:00:26 +0200 Subject: [PATCH 088/160] Use azure-posix.yml for linux and macOS Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 10 +++--- etc/ci/azure-mac.yml | 36 --------------------- etc/ci/{azure-linux.yml => azure-posix.yml} | 0 3 files changed, 5 insertions(+), 41 deletions(-) delete mode 100644 etc/ci/azure-mac.yml rename etc/ci/{azure-linux.yml => azure-posix.yml} (100%) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9a4c950c..31ef36f0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -7,7 +7,7 @@ jobs: - - template: etc/ci/azure-linux.yml + - template: etc/ci/azure-posix.yml parameters: job_name: ubuntu16_cpython image_name: ubuntu-16.04 @@ -15,7 +15,7 @@ jobs: test_suites: all: tmp/bin/pytest -vvs - - template: etc/ci/azure-linux.yml + - template: etc/ci/azure-posix.yml parameters: job_name: ubuntu18_cpython image_name: ubuntu-18.04 @@ -23,7 +23,7 @@ jobs: test_suites: all: tmp/bin/pytest -n 2 -vvs - - template: etc/ci/azure-linux.yml + - template: etc/ci/azure-posix.yml parameters: job_name: ubuntu20_cpython image_name: ubuntu-20.04 @@ -31,7 +31,7 @@ jobs: test_suites: all: tmp/bin/pytest -n 2 -vvs - - template: etc/ci/azure-mac.yml + - template: etc/ci/azure-posix.yml parameters: job_name: macos1014_cpython image_name: macos-10.14 @@ -39,7 +39,7 @@ jobs: test_suites: all: tmp/bin/pytest -n 2 -vvs - - template: etc/ci/azure-mac.yml + - template: etc/ci/azure-posix.yml parameters: job_name: macos1015_cpython image_name: macos-10.15 diff --git a/etc/ci/azure-mac.yml b/etc/ci/azure-mac.yml deleted file mode 100644 index 752ae2e6..00000000 --- a/etc/ci/azure-mac.yml +++ /dev/null @@ -1,36 +0,0 @@ -parameters: - job_name: '' - image_name: '' - python_versions: [] - test_suites: {} - python_architecture: x64 - -jobs: - - job: ${{ parameters.job_name }} - - pool: - vmImage: ${{ parameters.image_name }} - - strategy: - matrix: - ${{ each pyver in parameters.python_versions }}: - ${{ each tsuite in parameters.test_suites }}: - ${{ format('py{0} {1}', pyver, tsuite.key) }}: - python_version: ${{ pyver }} - test_suite_label: ${{ tsuite.key }} - test_suite: ${{ tsuite.value }} - steps: - - checkout: self - fetchDepth: 10 - - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python_version)' - architecture: '${{ parameters.python_architecture }}' - displayName: 'Install Python $(python_version)' - - - script: ./configure - displayName: 'Run Configure' - - - script: $(test_suite) - displayName: 'Run $(test_suite_label) tests with py$(python_version) on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-linux.yml b/etc/ci/azure-posix.yml similarity index 100% rename from etc/ci/azure-linux.yml rename to etc/ci/azure-posix.yml From 4f0aecf4f2a01c71b8d0f54987cd68de5f7922c2 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 11 May 2021 11:14:23 +0200 Subject: [PATCH 089/160] Adopt new configure script derived from ScanCode Signed-off-by: Philippe Ombredanne --- configure | 164 ++++++++++++++++++++++++---- configure.bat | 238 ++++++++++++++++++++++++++--------------- etc/ci/azure-posix.yml | 7 +- etc/ci/azure-win.yml | 5 +- 4 files changed, 304 insertions(+), 110 deletions(-) diff --git a/configure b/configure index 78e74983..25ab0ce9 100755 --- a/configure +++ b/configure @@ -1,43 +1,169 @@ #!/usr/bin/env bash # -# Copyright (c) nexB Inc. and others. +# Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/ for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # set -e #set -x -# source this script for a basic setup and configuration for local development +################################ +# A configuration script to set things up: +# create a virtualenv and install or update thirdparty packages. +# Source this script for initial configuration +# Use configure --help for details +# +# This script will search for a virtualenv.pyz app in etc/thirdparty/virtualenv.pyz +# Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default +################################ +CLI_ARGS=$1 -CONFIGURE_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +################################ +# Defaults. Change these variables to customize this script +################################ +# Requirement arguments passed to pip and used by default or with --dev. +REQUIREMENTS="--editable ." +DEV_REQUIREMENTS="--editable .[testing]" -if [[ "$1" == "--clean" ]]; then - rm -rf "$CONFIGURE_ROOT_DIR/tmp" - exit +# where we create a virtualenv +VIRTUALENV_DIR=tmp + +# Cleanable files and directories with the --clean option +CLEANABLE=" + build + tmp" + +# extra arguments passed to pip +PIP_EXTRA_ARGS=" " + +# the URL to download virtualenv.pyz if needed +VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz +################################ + + +################################ +# Current directory where this script lives +CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin + + +################################ +# Set the quiet flag to empty if not defined +if [[ "$CFG_QUIET" == "" ]]; then + CFG_QUIET=" " fi -if [[ "$PYTHON_EXE" == "" ]]; then - PYTHON_EXE=python3 +################################ +# find a proper Python to run +# Use environment variables or a file if available. +# Otherwise the latest Python by default. +if [[ "$PYTHON_EXECUTABLE" == "" ]]; then + # check for a file named PYTHON_EXECUTABLE + if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then + PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE") + else + PYTHON_EXECUTABLE=python3 + fi fi -function setup { - # create a virtualenv on Python - mkdir -p $CONFIGURE_ROOT_DIR/tmp - wget -O $CONFIGURE_ROOT_DIR/tmp/virtualenv.pyz https://bootstrap.pypa.io/virtualenv.pyz - $PYTHON_EXE $CONFIGURE_ROOT_DIR/tmp/virtualenv.pyz --wheel embed --pip embed --setuptools embed --seeder pip $CONFIGURE_ROOT_DIR/tmp - source $CONFIGURE_ROOT_DIR/tmp/bin/activate +################################ +cli_help() { + echo An initial configuration script + echo " usage: ./configure [options]" + echo + echo The default is to configure for regular use. Use --dev for development. + echo + echo The options are: + echo " --clean: clean built and installed files and exit." + echo " --dev: configure the environment for development." + echo " --help: display this help message and exit." + echo + echo By default, the python interpreter version found in the path is used. + echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to + echo configure another Python executable interpreter to use. If this is not + echo set, a file named PYTHON_EXECUTABLE containing a single line with the + echo path of the Python executable to use will be checked last. + set +e + exit } -setup +clean() { + # Remove cleanable file and directories and files from the root dir. + echo "* Cleaning ..." + for cln in $CLEANABLE; + do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}"; + done + set +e + exit +} -$CONFIGURE_ROOT_DIR/tmp/bin/pip install -e .[testing] -if [ -f "$CONFIGURE_ROOT_DIR/tmp/bin/activate" ]; then - source "$CONFIGURE_ROOT_DIR/tmp/bin/activate" -fi +create_virtualenv() { + # create a virtualenv for Python + # Note: we do not use the bundled Python 3 "venv" because its behavior and + # presence is not consistent across Linux distro and sometimes pip is not + # included either by default. The virtualenv.pyz app cures all these issues. + + VENV_DIR="$1" + if [ ! -f "$CFG_BIN_DIR/python" ]; then + + mkdir -p "$CFG_ROOT_DIR/$VENV_DIR" + + if [ -f "$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz" ]; then + VIRTUALENV_PYZ="$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz" + else + VIRTUALENV_PYZ="$CFG_ROOT_DIR/$VENV_DIR/virtualenv.pyz" + wget -O "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL" + fi + + $PYTHON_EXECUTABLE "$VIRTUALENV_PYZ" \ + --wheel embed --pip embed --setuptools embed \ + --seeder pip \ + --never-download \ + --no-periodic-update \ + --no-vcs-ignore \ + $CFG_QUIET \ + "$CFG_ROOT_DIR/$VENV_DIR" + fi +} + + +install_packages() { + # install requirements in virtualenv + # note: --no-build-isolation means that pip/wheel/setuptools will not + # be reinstalled a second time and reused from the virtualenv and this + # speeds up the installation. + # We always have the PEP517 build dependencies installed already. + + "$CFG_BIN_DIR/pip" install \ + --upgrade \ + --no-build-isolation \ + $CFG_QUIET \ + $PIP_EXTRA_ARGS \ + $1 +} + + +################################ +# Main command line entry point +CFG_DEV_MODE=0 +CFG_REQUIREMENTS=$REQUIREMENTS + +case "$CLI_ARGS" in + --help) cli_help;; + --clean) clean;; + --dev) CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;; +esac + +create_virtualenv "$VIRTUALENV_DIR" +install_packages "$CFG_REQUIREMENTS" +. "$CFG_BIN_DIR/activate" set +e diff --git a/configure.bat b/configure.bat index 00cb1011..8c497ba0 100644 --- a/configure.bat +++ b/configure.bat @@ -1,120 +1,180 @@ @echo OFF @setlocal -@rem Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. + +@rem Copyright (c) nexB Inc. and others. All rights reserved. +@rem SPDX-License-Identifier: Apache-2.0 +@rem See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +@rem See https://github.com/nexB/ for support or download. +@rem See https://aboutcode.org for more information about nexB OSS projects. + @rem ################################ -@rem # A configuration script for Windows -@rem # -@rem # The options and (optional) arguments are: -@rem # --clean : this is exclusive of anything else and cleans the environment -@rem # from built and installed files -@rem # -@rem # --python < path to python.exe> : this must be the first argument and set -@rem # the path to the Python executable to use. If < path to python.exe> is -@rem # set to "path", then the executable will be the python.exe available -@rem # in the PATH. +@rem # A configuration script to set things up: +@rem # create a virtualenv and install or update thirdparty packages. +@rem # Source this script for initial configuration +@rem # Use configure --help for details + +@rem # This script will search for a virtualenv.pyz app in etc\thirdparty\virtualenv.pyz +@rem # Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default @rem ################################ -@rem Current directory where this .bat files lives -set CFG_ROOT_DIR=%~dp0 -@rem path where a configured Python should live in the current virtualenv if installed -set CONFIGURED_PYTHON=%CFG_ROOT_DIR%tmp\Scripts\python.exe -set PYTHON_EXECUTABLE= +@rem ################################ +@rem # Defaults. Change these variables to customize this script +@rem ################################ + +@rem # Requirement arguments passed to pip and used by default or with --dev. +set "REQUIREMENTS=--editable ." +set "DEV_REQUIREMENTS=--editable .[testing]" + +@rem # where we create a virtualenv +set "VIRTUALENV_DIR=tmp" + +@rem # Cleanable files and directories to delete with the --clean option +set "CLEANABLE=build tmp" -@rem parse command line options and arguments -:collectopts -if "%1" EQU "--help" (goto cli_help) -if "%1" EQU "--clean" (call rmdir /s /q "%CFG_ROOT_DIR%tmp") && call exit /b -if "%1" EQU "--python" (set PROVIDED_PYTHON=%~2) && shift && shift && goto collectopts +@rem # extra arguments passed to pip +set "PIP_EXTRA_ARGS= " -@rem If we have a pre-configured Python in our virtualenv, reuse this as-is and run -if exist ""%CONFIGURED_PYTHON%"" ( - set PYTHON_EXECUTABLE=%CONFIGURED_PYTHON% - goto run +@rem # the URL to download virtualenv.pyz if needed +set VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz +@rem ################################ + + +@rem ################################ +@rem # Current directory where this script lives +set CFG_ROOT_DIR=%~dp0 +set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" + + +@rem ################################ +@rem # Set the quiet flag to empty if not defined +if not defined CFG_QUIET ( + set "CFG_QUIET= " ) -@rem If we have a command arg for Python use this as-is -if ""%PROVIDED_PYTHON%""==""path"" ( - @rem use a bare python available in the PATH - set PYTHON_EXECUTABLE=python - goto run + +@rem ################################ +@rem # Main command line entry point +set CFG_DEV_MODE=0 +set "CFG_REQUIREMENTS=%REQUIREMENTS%" + +if "%1" EQU "--help" (goto cli_help) +if "%1" EQU "--clean" (goto clean) +if "%1" EQU "--dev" ( + set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" + set CFG_DEV_MODE=1 ) -if exist ""%PROVIDED_PYTHON%"" ( - set PYTHON_EXECUTABLE=%PROVIDED_PYTHON% - goto run +if "%1" EQU "--python" ( + echo "The --python is now DEPRECATED. Use the PYTHON_EXECUTABLE environment + echo "variable instead. Run configure --help for details." + exit /b 0 ) +@rem ################################ +@rem # find a proper Python to run +@rem # Use environment variables or a file if available. +@rem # Otherwise the latest Python by default. +if not defined PYTHON_EXECUTABLE ( + @rem # check for a file named PYTHON_EXECUTABLE + if exist ""%CFG_ROOT_DIR%\PYTHON_EXECUTABLE"" ( + set /p PYTHON_EXECUTABLE=<""%CFG_ROOT_DIR%\PYTHON_EXECUTABLE"" + ) else ( + set "PYTHON_EXECUTABLE=py" + ) +) -@rem otherwise we search for a suitable Python interpreter -:find_python -@rem First check the existence of the "py" launcher (available in Python 3) -@rem if we have it, check if we have a py -3 installed with the good version or a py 2.7 -@rem if not, check if we have an old py 2.7 -@rem exist if all fails +:create_virtualenv +@rem # create a virtualenv for Python +@rem # Note: we do not use the bundled Python 3 "venv" because its behavior and +@rem # presence is not consistent across Linux distro and sometimes pip is not +@rem # included either by default. The virtualenv.pyz app cures all these issues. -where py >nul 2>nul -if %ERRORLEVEL% == 0 ( - @rem we have a py launcher, check for the availability of our required Python 3 version - py -3.6 --version >nul 2>nul - if %ERRORLEVEL% == 0 ( - set PYTHON_EXECUTABLE=py -3.6 - ) else ( - @rem we have no required python 3, let's try python 2: - py -2 --version >nul 2>nul - if %ERRORLEVEL% == 0 ( - set PYTHON_EXECUTABLE=py -2 - ) else ( - @rem we have py and no python 3 and 2, exit - echo * Unable to find an installation of Python. - exit /b 1 - ) +if not exist ""%CFG_BIN_DIR%\python.exe"" ( + if not exist "%CFG_BIN_DIR%" ( + mkdir %CFG_BIN_DIR% ) -) else ( - @rem we have no py launcher, check for a default Python 2 installation - if not exist ""%DEFAULT_PYTHON2%"" ( - echo * Unable to find an installation of Python. - exit /b 1 + + if exist ""%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz"" ( + %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" ^ + --wheel embed --pip embed --setuptools embed ^ + --seeder pip ^ + --never-download ^ + --no-periodic-update ^ + --no-vcs-ignore ^ + %CFG_QUIET% ^ + %CFG_ROOT_DIR%\%VIRTUALENV_DIR% ) else ( - set PYTHON_EXECUTABLE=%DEFAULT_PYTHON2% + if not exist ""%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz"" ( + curl -o "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" %VIRTUALENV_PYZ_URL% + + if %ERRORLEVEL% neq 0 ( + exit /b %ERRORLEVEL% + ) + ) + %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" ^ + --wheel embed --pip embed --setuptools embed ^ + --seeder pip ^ + --never-download ^ + --no-periodic-update ^ + --no-vcs-ignore ^ + %CFG_QUIET% ^ + %CFG_ROOT_DIR%\%VIRTUALENV_DIR% ) ) +if %ERRORLEVEL% neq 0 ( + exit /b %ERRORLEVEL% +) + -:run -@rem without this things may not always work on Windows 10, but this makes things slower -set PYTHONDONTWRITEBYTECODE=1 +:install_packages +@rem # install requirements in virtualenv +@rem # note: --no-build-isolation means that pip/wheel/setuptools will not +@rem # be reinstalled a second time and reused from the virtualenv and this +@rem # speeds up the installation. +@rem # We always have the PEP517 build dependencies installed already. -call mkdir "%CFG_ROOT_DIR%tmp" -call curl -o "%CFG_ROOT_DIR%tmp\virtualenv.pyz" https://bootstrap.pypa.io/virtualenv.pyz -call %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%tmp\virtualenv.pyz" --wheel embed --pip embed --setuptools embed --seeder pip "%CFG_ROOT_DIR%tmp" -call "%CFG_ROOT_DIR%tmp\Scripts\activate" -call "%CFG_ROOT_DIR%tmp\Scripts\pip" install -e .[testing] +%CFG_BIN_DIR%\pip install ^ + --upgrade ^ + --no-build-isolation ^ + %CFG_QUIET% ^ + %PIP_EXTRA_ARGS% ^ + %CFG_REQUIREMENTS% -@rem Return a proper return code on failure if %ERRORLEVEL% neq 0 ( exit /b %ERRORLEVEL% ) -endlocal -goto activate +exit /b 0 + + +@rem ################################ :cli_help -echo A configuration script for Windows -echo usage: configure [options] [path/to/config/directory] -echo. -echo The options and arguments are: -echo --clean : this is exclusive of anything else and cleans the environment -echo from built and installed files -echo. -echo --python path/to/python.exe : this is set to the path of an alternative -echo Python executable to use. If path/to/python.exe is set to "path", -echo then the executable will be the python.exe available in the PATH. -echo. - - -:activate -@rem Activate the virtualenv -if exist "%CFG_ROOT_DIR%tmp\Scripts\activate" ( - "%CFG_ROOT_DIR%tmp\Scripts\activate" + echo An initial configuration script + echo " usage: configure [options]" + echo " " + echo The default is to configure for regular use. Use --dev for development. + echo " " + echo The options are: + echo " --clean: clean built and installed files and exit." + echo " --dev: configure the environment for development." + echo " --help: display this help message and exit." + echo " " + echo By default, the python interpreter version found in the path is used. + echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to + echo configure another Python executable interpreter to use. If this is not + echo set, a file named PYTHON_EXECUTABLE containing a single line with the + echo path of the Python executable to use will be checked last. + exit /b 0 + + +:clean +@rem # Remove cleanable file and directories and files from the root dir. +echo "* Cleaning ..." +for %%F in (%CLEANABLE%) do ( + rmdir /s /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1 + del /f /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1 ) +exit /b 0 diff --git a/etc/ci/azure-posix.yml b/etc/ci/azure-posix.yml index 752ae2e6..0921d9b3 100644 --- a/etc/ci/azure-posix.yml +++ b/etc/ci/azure-posix.yml @@ -19,6 +19,7 @@ jobs: python_version: ${{ pyver }} test_suite_label: ${{ tsuite.key }} test_suite: ${{ tsuite.value }} + steps: - checkout: self fetchDepth: 10 @@ -29,7 +30,11 @@ jobs: architecture: '${{ parameters.python_architecture }}' displayName: 'Install Python $(python_version)' - - script: ./configure + - script: | + python3 --version + python$(python_version) --version + echo "python$(python_version)" > PYTHON_EXECUTABLE + ./configure --dev displayName: 'Run Configure' - script: $(test_suite) diff --git a/etc/ci/azure-win.yml b/etc/ci/azure-win.yml index afe1686f..03d89274 100644 --- a/etc/ci/azure-win.yml +++ b/etc/ci/azure-win.yml @@ -29,7 +29,10 @@ jobs: architecture: '${{ parameters.python_architecture }}' displayName: 'Install Python $(python_version)' - - script: configure --python path + - script: | + python --version + echo | set /p=python> PYTHON_EXECUTABLE + configure --dev displayName: 'Run Configure' - script: $(test_suite) From aa04429ae6e5d05ef8ee2a0fbad9872014463a25 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 11 May 2021 11:17:09 +0200 Subject: [PATCH 090/160] Add notes on customization Signed-off-by: Philippe Ombredanne --- README.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.rst b/README.rst index a0e682f3..a2911736 100644 --- a/README.rst +++ b/README.rst @@ -32,3 +32,12 @@ Update an existing project git merge skeleton/main --allow-unrelated-histories This is also the workflow to use when updating the skeleton files in any given repository. + + +Customizing +----------- + +You typically want to perform these customizations: + +- remove or update the src/README.rst and tests/README.rst files +- check the configure and configure.bat defaults From 56ada8fffacac14140bf016fd3f6bee4f4615fcc Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 11 May 2021 11:19:12 +0200 Subject: [PATCH 091/160] Adopt new configure --dev convention Signed-off-by: Philippe Ombredanne --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7a342dff..1b52eb20 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ python: - "3.8" # Scripts to run at install stage -install: ./configure +install: ./configure --dev # Scripts to run at script stage script: tmp/bin/pytest From 0dbcdc9f6c929b3d030910a69e5566c149e15d7a Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 11 May 2021 11:21:48 +0200 Subject: [PATCH 092/160] Clarify CHANGELOG to be Rst Signed-off-by: Philippe Ombredanne --- CHANGELOG.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5f8bc8da..fc2b6e32 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,5 +1,8 @@ -Release notes -------------- -### Version 0.0.0 +Changelog +========= + + +v0.0.0 +------ *xxxx-xx-xx* -- Initial release. From d21aef35a61675289bbebf963030b539c10a7b28 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 11 May 2021 11:22:22 +0200 Subject: [PATCH 093/160] Add skeleton release notes to README.rst This was they do not end up in the template CHANGELOG.rst Signed-off-by: Philippe Ombredanne --- README.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.rst b/README.rst index a2911736..b84a0491 100644 --- a/README.rst +++ b/README.rst @@ -41,3 +41,10 @@ You typically want to perform these customizations: - remove or update the src/README.rst and tests/README.rst files - check the configure and configure.bat defaults + + +Release Notes +------------- + +- 2021-05-11: adopt new configure scripts from ScanCode TK that allows correct + configuration of which Python version is used. From fa048f45bdb384f610125c201c983f2b0c8388a6 Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Fri, 28 May 2021 01:14:34 -0600 Subject: [PATCH 094/160] use isort to sort imports --- tests/ebuild/test_cpv.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/ebuild/test_cpv.py b/tests/ebuild/test_cpv.py index ccfe4279..83282abe 100644 --- a/tests/ebuild/test_cpv.py +++ b/tests/ebuild/test_cpv.py @@ -1,9 +1,8 @@ from random import shuffle import pytest -from snakeoil.compatibility import cmp - from pkgcore.ebuild import cpv +from snakeoil.compatibility import cmp def generate_misc_sufs(): From 3aeb2ec68d313b75430539d9e4d2e57c53ef6998 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 31 May 2021 11:24:39 +0200 Subject: [PATCH 095/160] Update format Signed-off-by: Philippe Ombredanne --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index f7910848..f192f220 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,7 +20,7 @@ classifiers = Programming Language :: Python :: 3 :: Only Topic :: Software Development Topic :: Utilities -keywords = +keywords = utilities [options] @@ -43,4 +43,4 @@ testing = docs= Sphinx>=3.3.1 sphinx-rtd-theme>=0.5.0 - doc8>=0.8.1 \ No newline at end of file + doc8>=0.8.1 From 2c412e8222d4d615384a24e2ddc472b0c9703916 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 31 May 2021 11:24:57 +0200 Subject: [PATCH 096/160] Add Python 3.9 to Travis Signed-off-by: Philippe Ombredanne --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 1b52eb20..1a90a385 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,7 @@ python: - "3.6" - "3.7" - "3.8" + - "3.9" # Scripts to run at install stage install: ./configure --dev From 69eec23792d59dbdc3a3acb1711884560cf27073 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 31 May 2021 11:27:35 +0200 Subject: [PATCH 097/160] Format and remove spurious spaces From https://github.com/nexB/typecode/pull/20 Reported-by: Pierre Tardy Signed-off-by: Philippe Ombredanne --- configure.bat | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/configure.bat b/configure.bat index 8c497ba0..80d0a43b 100644 --- a/configure.bat +++ b/configure.bat @@ -9,7 +9,7 @@ @rem ################################ -@rem # A configuration script to set things up: +@rem # A configuration script to set things up: @rem # create a virtualenv and install or update thirdparty packages. @rem # Source this script for initial configuration @rem # Use configure --help for details @@ -48,7 +48,7 @@ set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" @rem ################################ -@rem # Set the quiet flag to empty if not defined +@rem # Set the quiet flag to empty if not defined if not defined CFG_QUIET ( set "CFG_QUIET= " ) @@ -65,8 +65,8 @@ if "%1" EQU "--dev" ( set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" set CFG_DEV_MODE=1 ) -if "%1" EQU "--python" ( - echo "The --python is now DEPRECATED. Use the PYTHON_EXECUTABLE environment +if "%1" EQU "--python"( + echo "The --python option is now DEPRECATED. Use the PYTHON_EXECUTABLE environment" echo "variable instead. Run configure --help for details." exit /b 0 ) @@ -76,7 +76,7 @@ if "%1" EQU "--python" ( @rem # Use environment variables or a file if available. @rem # Otherwise the latest Python by default. if not defined PYTHON_EXECUTABLE ( - @rem # check for a file named PYTHON_EXECUTABLE + @rem # check for a file named PYTHON_EXECUTABLE if exist ""%CFG_ROOT_DIR%\PYTHON_EXECUTABLE"" ( set /p PYTHON_EXECUTABLE=<""%CFG_ROOT_DIR%\PYTHON_EXECUTABLE"" ) else ( From 0e09ad9eb77ca0b580d71baa428955a0a56d19f1 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 31 May 2021 19:17:43 +0200 Subject: [PATCH 098/160] Bump to more modern version of setuptools_scm And remove v prefix from fallback version Signed-off-by: Philippe Ombredanne --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8eebe91a..852f0fce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,11 +1,11 @@ [build-system] -requires = ["setuptools >= 50", "wheel", "setuptools_scm[toml] >= 4"] +requires = ["setuptools >= 50", "wheel", "setuptools_scm[toml] >= 6"] build-backend = "setuptools.build_meta" [tool.setuptools_scm] # this is used populated when creating a git archive # and when there is .git dir and/or there is no git installed -fallback_version = "v9999.$Format:%h-%cs$" +fallback_version = "9999.$Format:%h-%cs$" [tool.pytest.ini_options] norecursedirs = [ From e339a70e1a46b613fa73b9d0a9273fe7640acb8d Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 31 May 2021 19:18:09 +0200 Subject: [PATCH 099/160] Add space for correct syntax Signed-off-by: Philippe Ombredanne --- configure.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.bat b/configure.bat index 80d0a43b..c12f937f 100644 --- a/configure.bat +++ b/configure.bat @@ -65,7 +65,7 @@ if "%1" EQU "--dev" ( set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" set CFG_DEV_MODE=1 ) -if "%1" EQU "--python"( +if "%1" EQU "--python" ( echo "The --python option is now DEPRECATED. Use the PYTHON_EXECUTABLE environment" echo "variable instead. Run configure --help for details." exit /b 0 From 9dff54a1f827ee2d40761fae72cd0c1b69489818 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Thu, 17 Jun 2021 17:59:32 +0800 Subject: [PATCH 100/160] Create junction from Scripts to bin * This is handy for windows to have the same path as linux Signed-off-by: Chin Yeung Li --- configure.bat | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configure.bat b/configure.bat index c12f937f..bafa126e 100644 --- a/configure.bat +++ b/configure.bat @@ -142,6 +142,9 @@ if %ERRORLEVEL% neq 0 ( %PIP_EXTRA_ARGS% ^ %CFG_REQUIREMENTS% +@rem # Create junction to bin to have the same directory between linux and windows +mklink /J %CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin %CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts + if %ERRORLEVEL% neq 0 ( exit /b %ERRORLEVEL% ) From 235f8431b73a1cec523e300e10635ad7cfd657ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Sat, 14 Aug 2021 21:48:49 +0200 Subject: [PATCH 101/160] pkgcore.ebuild.cpv: fix isvalid_pkg_name() logic Fix the package name verification logic not to incorrectly reject packages that resemble a version number or that contain a series of hyphens. Document the algorithm better and add more tests for corner cases. --- src/pkgcore/ebuild/cpv.py | 13 +++++++++---- tests/ebuild/test_cpv.py | 11 +++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index 883851dd..c49ac8e0 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -33,13 +33,18 @@ def isvalid_pkg_name(chunks): if not chunks[0] or chunks[0][0] == '+': # this means a leading -; additionally, '+asdf' is disallowed return False - mf = _pkg_re.match - if not all(mf(s) for s in chunks[:-1]): + # all remaining chunks can either be empty (meaning multiple + # hyphens) or must be valid chars + if not all(not s or _pkg_re.match(s) for s in chunks): + return False + # the package name must not end with a hyphen followed by anything that + # looks like a version -- need to ensure that we've gotten more than one + # chunk, i.e. at least one hyphen + if len(chunks) > 1 and isvalid_version_re.match(chunks[-1]): return False - if chunks[-1]: - return mf(chunks[-1]) and not isvalid_version_re.match(chunks[-1]) return True + def isvalid_rev(s): return s and s[0] == 'r' and s[1:].isdigit() diff --git a/tests/ebuild/test_cpv.py b/tests/ebuild/test_cpv.py index 83282abe..7bd1f67d 100644 --- a/tests/ebuild/test_cpv.py +++ b/tests/ebuild/test_cpv.py @@ -24,15 +24,18 @@ class TestCPV: good_cats = ( "dev-util", "dev+", "dev-util+", "DEV-UTIL", "aaa0", - "aaa-0", "multi/depth", "cross-dev_idiot.hacks-suck", "a") + "aaa-0", "multi/depth", "cross-dev_idiot.hacks-suck", "a", + "foo---", "multi--hyphen") bad_cats = (".util", "_dev", "", "dev-util ", "multi//depth") - good_pkgs = ("diffball", "a9", "a9+", "a-100dpi", "diff-mode-") - bad_pkgs = ("diffball ", "diffball-9", "a-3D", "ab--df", "-df", "+dfa") + good_pkgs = ("diffball", "a9", "a9+", "a-100dpi", "diff-mode-", + "multi--hyphen", "timidity--", "frob---", "diffball-9-") + bad_pkgs = ("diffball ", "diffball-9", "a-3D", "-df", "+dfa", + "timidity--9f", "ormaybe---13_beta") good_cp = ( "bbb-9/foon", "dev-util/diffball", "dev-util/diffball-a9", "dev-ut-asdf/emacs-cvs", "xfce-base/xfce4", "bah/f-100dpi", - "dev-util/diffball-blah-monkeys") + "dev-util/diffball-blah-monkeys", "virtual/7z") good_vers = ("1", "2.3.4", "2.3.4a", "02.3", "2.03", "3d", "3D") bad_vers = ("2.3a.4", "2.a.3", "2.3_", "2.3 ", "2.3.", "cvs.2") From d6f83029ca65cfd901ccb06fc850d23aaf173d69 Mon Sep 17 00:00:00 2001 From: Tim Harder Date: Thu, 26 Aug 2021 01:37:56 -0600 Subject: [PATCH 102/160] ebuild.cpv: explicitly handle invalid revisions To avoid crashes when overriding the raw pkg class used when iterating over package restrictions. In most cases, this was already handled internally but pkgcheck overrides the pkg class for certain reasons in some cases causing an unhandled crash. --- src/pkgcore/ebuild/cpv.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pkgcore/ebuild/cpv.py b/src/pkgcore/ebuild/cpv.py index c49ac8e0..1b2f8703 100644 --- a/src/pkgcore/ebuild/cpv.py +++ b/src/pkgcore/ebuild/cpv.py @@ -59,7 +59,10 @@ class Revision(UserString): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if self.data: - self._revint = int(self.data) + try: + self._revint = int(self.data) + except ValueError: + raise InvalidCPV(self.data, 'invalid revision') else: self._revint = 0 From 77ce5e4068eaa64b876ca267d09e1689fe67ae8f Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Mon, 30 Aug 2021 17:40:27 -0700 Subject: [PATCH 103/160] Check for deps in local thirdparty directory #31 Signed-off-by: Jono Yang --- configure | 8 +++++--- configure.bat | 6 ++++++ thirdparty/README.rst | 2 ++ 3 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 thirdparty/README.rst diff --git a/configure b/configure index 25ab0ce9..99bdf573 100755 --- a/configure +++ b/configure @@ -11,7 +11,7 @@ set -e #set -x ################################ -# A configuration script to set things up: +# A configuration script to set things up: # create a virtualenv and install or update thirdparty packages. # Source this script for initial configuration # Use configure --help for details @@ -50,9 +50,11 @@ VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin +# Find packages from the local thirdparty directory or from pypi +PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty" ################################ -# Set the quiet flag to empty if not defined +# Set the quiet flag to empty if not defined if [[ "$CFG_QUIET" == "" ]]; then CFG_QUIET=" " fi @@ -63,7 +65,7 @@ fi # Use environment variables or a file if available. # Otherwise the latest Python by default. if [[ "$PYTHON_EXECUTABLE" == "" ]]; then - # check for a file named PYTHON_EXECUTABLE + # check for a file named PYTHON_EXECUTABLE if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE") else diff --git a/configure.bat b/configure.bat index bafa126e..be8f5793 100644 --- a/configure.bat +++ b/configure.bat @@ -47,6 +47,12 @@ set CFG_ROOT_DIR=%~dp0 set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" +@rem ################################ +@rem # Thirdparty package locations and index handling +set "PIP_EXTRA_ARGS=--find-links %CFG_ROOT_DIR%\thirdparty" +@rem ################################ + + @rem ################################ @rem # Set the quiet flag to empty if not defined if not defined CFG_QUIET ( diff --git a/thirdparty/README.rst b/thirdparty/README.rst new file mode 100644 index 00000000..b31482f8 --- /dev/null +++ b/thirdparty/README.rst @@ -0,0 +1,2 @@ +Put your Python dependency wheels to be vendored in this directory. + From 1bcaaa574d4430ae363e66a3ace74ef3e4e8981b Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 1 Sep 2021 15:59:49 -0700 Subject: [PATCH 104/160] Enforce use of requirements.txt #34 Signed-off-by: Jono Yang --- configure | 12 ++++++++---- configure.bat | 11 ++++++++--- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/configure b/configure index 99bdf573..66d939aa 100755 --- a/configure +++ b/configure @@ -26,8 +26,8 @@ CLI_ARGS=$1 ################################ # Requirement arguments passed to pip and used by default or with --dev. -REQUIREMENTS="--editable ." -DEV_REQUIREMENTS="--editable .[testing]" +REQUIREMENTS="--editable . --constraint requirements.txt" +DEV_REQUIREMENTS="--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" # where we create a virtualenv VIRTUALENV_DIR=tmp @@ -50,8 +50,12 @@ VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin -# Find packages from the local thirdparty directory or from pypi -PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty" +# Find packages from the local thirdparty directory or from thirdparty.aboutcode.org +PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty --find-links https://thirdparty.aboutcode.org/pypi" + +if [[ -f "$CFG_ROOT_DIR/requirements.txt" ]] && [[ -f "$CFG_ROOT_DIR/requirements-dev.txt" ]]; then + PIP_EXTRA_ARGS+=" --no-index" +fi ################################ # Set the quiet flag to empty if not defined diff --git a/configure.bat b/configure.bat index be8f5793..75cab5fc 100644 --- a/configure.bat +++ b/configure.bat @@ -24,8 +24,8 @@ @rem ################################ @rem # Requirement arguments passed to pip and used by default or with --dev. -set "REQUIREMENTS=--editable ." -set "DEV_REQUIREMENTS=--editable .[testing]" +set "REQUIREMENTS=--editable . --constraint requirements.txt" +set "DEV_REQUIREMENTS=--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" @rem # where we create a virtualenv set "VIRTUALENV_DIR=tmp" @@ -49,7 +49,12 @@ set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" @rem ################################ @rem # Thirdparty package locations and index handling -set "PIP_EXTRA_ARGS=--find-links %CFG_ROOT_DIR%\thirdparty" +if exist ""%CFG_ROOT_DIR%\requirements.txt"" if exist ""%CFG_ROOT_DIR%\requirements-dev.txt"" ( + set "INDEX_ARG= --no-index" +) else ( + set "INDEX_ARG= " +) +set "PIP_EXTRA_ARGS=--find-links %CFG_ROOT_DIR%\thirdparty --find-links https://thirdparty.aboutcode.org/pypi" & %INDEX_ARG% @rem ################################ From e9067c81d14d07ec2dafc732292a078d0519c885 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 1 Sep 2021 18:04:25 -0700 Subject: [PATCH 105/160] Add scripts from scancode-toolkit/etc/release/ #33 Signed-off-by: Jono Yang --- etc/scripts/bootstrap.py | 212 ++ etc/scripts/build_wheels.py | 97 + etc/scripts/check_thirdparty.py | 32 + etc/scripts/fetch_requirements.py | 145 + etc/scripts/fix_thirdparty.py | 81 + etc/scripts/gen_requirements.py | 43 + etc/scripts/gen_requirements_dev.py | 55 + .../test_utils_pip_compatibility_tags.py | 128 + ...test_utils_pip_compatibility_tags.py.ABOUT | 14 + etc/scripts/test_utils_pypi_supported_tags.py | 91 + .../test_utils_pypi_supported_tags.py.ABOUT | 17 + etc/scripts/utils_dejacode.py | 213 ++ etc/scripts/utils_pip_compatibility_tags.py | 192 ++ .../utils_pip_compatibility_tags.py.ABOUT | 14 + etc/scripts/utils_pypi_supported_tags.py | 109 + .../utils_pypi_supported_tags.py.ABOUT | 17 + etc/scripts/utils_requirements.py | 103 + etc/scripts/utils_thirdparty.py | 2940 +++++++++++++++++ etc/scripts/utils_thirdparty.py.ABOUT | 15 + 19 files changed, 4518 insertions(+) create mode 100644 etc/scripts/bootstrap.py create mode 100644 etc/scripts/build_wheels.py create mode 100644 etc/scripts/check_thirdparty.py create mode 100644 etc/scripts/fetch_requirements.py create mode 100644 etc/scripts/fix_thirdparty.py create mode 100644 etc/scripts/gen_requirements.py create mode 100644 etc/scripts/gen_requirements_dev.py create mode 100644 etc/scripts/test_utils_pip_compatibility_tags.py create mode 100644 etc/scripts/test_utils_pip_compatibility_tags.py.ABOUT create mode 100644 etc/scripts/test_utils_pypi_supported_tags.py create mode 100644 etc/scripts/test_utils_pypi_supported_tags.py.ABOUT create mode 100644 etc/scripts/utils_dejacode.py create mode 100644 etc/scripts/utils_pip_compatibility_tags.py create mode 100644 etc/scripts/utils_pip_compatibility_tags.py.ABOUT create mode 100644 etc/scripts/utils_pypi_supported_tags.py create mode 100644 etc/scripts/utils_pypi_supported_tags.py.ABOUT create mode 100644 etc/scripts/utils_requirements.py create mode 100644 etc/scripts/utils_thirdparty.py create mode 100644 etc/scripts/utils_thirdparty.py.ABOUT diff --git a/etc/scripts/bootstrap.py b/etc/scripts/bootstrap.py new file mode 100644 index 00000000..54701f63 --- /dev/null +++ b/etc/scripts/bootstrap.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import itertools + +import click + +import utils_thirdparty +from utils_thirdparty import Environment +from utils_thirdparty import PypiPackage + + +@click.command() + +@click.option('-r', '--requirements-file', + type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), + metavar='FILE', + multiple=True, + default=['requirements.txt'], + show_default=True, + help='Path to the requirements file(s) to use for thirdparty packages.', +) +@click.option('-d', '--thirdparty-dir', + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), + metavar='DIR', + default=utils_thirdparty.THIRDPARTY_DIR, + show_default=True, + help='Path to the thirdparty directory where wheels are built and ' + 'sources, ABOUT and LICENSE files fetched.', +) +@click.option('-p', '--python-version', + type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), + metavar='PYVER', + default=utils_thirdparty.PYTHON_VERSIONS, + show_default=True, + multiple=True, + help='Python version(s) to use for this build.', +) +@click.option('-o', '--operating-system', + type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), + metavar='OS', + default=tuple(utils_thirdparty.PLATFORMS_BY_OS), + multiple=True, + show_default=True, + help='OS(ses) to use for this build: one of linux, mac or windows.', +) +@click.option('-l', '--latest-version', + is_flag=True, + help='Get the latest version of all packages, ignoring version specifiers.', +) +@click.option('--sync-dejacode', + is_flag=True, + help='Synchronize packages with DejaCode.', +) +@click.option('--with-deps', + is_flag=True, + help='Also include all dependent wheels.', +) +@click.help_option('-h', '--help') +def bootstrap( + requirements_file, + thirdparty_dir, + python_version, + operating_system, + with_deps, + latest_version, + sync_dejacode, + build_remotely=False, +): + """ + Boostrap a thirdparty Python packages directory from pip requirements. + + Fetch or build to THIRDPARTY_DIR all the wheels and source distributions for + the pip ``--requirement-file`` requirements FILE(s). Build wheels compatible + with all the provided ``--python-version`` PYVER(s) and ```--operating_system`` + OS(s) defaulting to all supported combinations. Create or fetch .ABOUT and + .LICENSE files. + + Optionally ignore version specifiers and use the ``--latest-version`` + of everything. + + Sources and wheels are fetched with attempts first from PyPI, then our remote repository. + If missing wheels are built as needed. + """ + # rename variables for clarity since these are lists + requirements_files = requirements_file + python_versions = python_version + operating_systems = operating_system + + # create the environments we need + evts = itertools.product(python_versions, operating_systems) + environments = [Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] + + # collect all packages to process from requirements files + # this will fail with an exception if there are packages we cannot find + + required_name_versions = set() + + for req_file in requirements_files: + nvs = utils_thirdparty.load_requirements( + requirements_file=req_file, force_pinned=False) + required_name_versions.update(nvs) + if latest_version: + required_name_versions = set((name, None) for name, _ver in required_name_versions) + + print(f'PROCESSING {len(required_name_versions)} REQUIREMENTS in {len(requirements_files)} FILES') + + # fetch all available wheels, keep track of missing + # start with local, then remote, then PyPI + + print('==> COLLECTING ALREADY LOCALLY AVAILABLE REQUIRED WHEELS') + # list of all the wheel filenames either pre-existing, fetched or built + # updated as we progress + available_wheel_filenames = [] + + local_packages_by_namever = { + (p.name, p.version): p + for p in utils_thirdparty.get_local_packages(directory=thirdparty_dir) + } + + # list of (name, version, environment) not local and to fetch + name_version_envt_to_fetch = [] + + # start with a local check + for (name, version), envt in itertools.product(required_name_versions, environments): + local_pack = local_packages_by_namever.get((name, version,)) + if local_pack: + supported_wheels = list(local_pack.get_supported_wheels(environment=envt)) + if supported_wheels: + available_wheel_filenames.extend(w.filename for w in supported_wheels) + print(f'====> No fetch or build needed. ' + f'Local wheel already available for {name}=={version} ' + f'on os: {envt.operating_system} for Python: {envt.python_version}') + continue + + name_version_envt_to_fetch.append((name, version, envt,)) + + print(f'==> TRYING TO FETCH #{len(name_version_envt_to_fetch)} REQUIRED WHEELS') + + # list of (name, version, environment) not fetch and to build + name_version_envt_to_build = [] + + # then check if the wheel can be fetched without building from remote and Pypi + for name, version, envt in name_version_envt_to_fetch: + + fetched_fwn = utils_thirdparty.fetch_package_wheel( + name=name, + version=version, + environment=envt, + dest_dir=thirdparty_dir, + ) + + if fetched_fwn: + available_wheel_filenames.append(fetched_fwn) + else: + name_version_envt_to_build.append((name, version, envt,)) + + # At this stage we have all the wheels we could obtain without building + for name, version, envt in name_version_envt_to_build: + print(f'====> Need to build wheels for {name}=={version} on os: ' + f'{envt.operating_system} for Python: {envt.python_version}') + + packages_and_envts_to_build = [ + (PypiPackage(name, version), envt) + for name, version, envt in name_version_envt_to_build + ] + + print(f'==> BUILDING #{len(packages_and_envts_to_build)} MISSING WHEELS') + + package_envts_not_built, wheel_filenames_built = utils_thirdparty.build_missing_wheels( + packages_and_envts=packages_and_envts_to_build, + build_remotely=build_remotely, + with_deps=with_deps, + dest_dir=thirdparty_dir, +) + if wheel_filenames_built: + available_wheel_filenames.extend(available_wheel_filenames) + + for pack, envt in package_envts_not_built: + print( + f'====> FAILED to build any wheel for {pack.name}=={pack.version} ' + f'on os: {envt.operating_system} for Python: {envt.python_version}' + ) + + print(f'==> FETCHING SOURCE DISTRIBUTIONS') + # fetch all sources, keep track of missing + # This is a list of (name, version) + utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir) + + print(f'==> FETCHING ABOUT AND LICENSE FILES') + utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir) + + ############################################################################ + if sync_dejacode: + print(f'==> SYNC WITH DEJACODE') + # try to fetch from DejaCode any missing ABOUT + # create all missing DejaCode packages + pass + + utils_thirdparty.find_problems(dest_dir=thirdparty_dir) + + +if __name__ == '__main__': + bootstrap() diff --git a/etc/scripts/build_wheels.py b/etc/scripts/build_wheels.py new file mode 100644 index 00000000..416adc7c --- /dev/null +++ b/etc/scripts/build_wheels.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import click + +import utils_thirdparty + + +@click.command() + +@click.option('-n', '--name', + type=str, + metavar='PACKAGE_NAME', + required=True, + help='Python package name to add or build.', +) +@click.option('-v', '--version', + type=str, + default=None, + metavar='VERSION', + help='Python package version to add or build.', +) +@click.option('-d', '--thirdparty-dir', + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), + metavar='DIR', + default=utils_thirdparty.THIRDPARTY_DIR, + show_default=True, + help='Path to the thirdparty directory where wheels are built.', +) +@click.option('-p', '--python-version', + type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), + metavar='PYVER', + default=utils_thirdparty.PYTHON_VERSIONS, + show_default=True, + multiple=True, + help='Python version to use for this build.', +) +@click.option('-o', '--operating-system', + type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), + metavar='OS', + default=tuple(utils_thirdparty.PLATFORMS_BY_OS), + multiple=True, + show_default=True, + help='OS to use for this build: one of linux, mac or windows.', +) +@click.option('--build-remotely', + is_flag=True, + help='Build missing wheels remotely.', +) +@click.option('--with-deps', + is_flag=True, + help='Also include all dependent wheels.', +) +@click.option('--verbose', + is_flag=True, + help='Provide verbose output.', +) +@click.help_option('-h', '--help') +def build_wheels( + name, + version, + thirdparty_dir, + python_version, + operating_system, + with_deps, + build_remotely, + verbose, +): + """ + Build to THIRDPARTY_DIR all the wheels for the Python PACKAGE_NAME and + optional VERSION. Build wheels compatible with all the `--python-version` + PYVER(s) and `--operating_system` OS(s). + + Build native wheels remotely if needed when `--build-remotely` and include + all dependencies with `--with-deps`. + """ + utils_thirdparty.add_or_upgrade_built_wheels( + name=name, + version=version, + python_versions=python_version, + operating_systems=operating_system, + dest_dir=thirdparty_dir, + build_remotely=build_remotely, + with_deps=with_deps, + verbose=verbose, + ) + + +if __name__ == '__main__': + build_wheels() diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py new file mode 100644 index 00000000..b29ce2be --- /dev/null +++ b/etc/scripts/check_thirdparty.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import click + +import utils_thirdparty + + +@click.command() + +@click.option('-d', '--thirdparty-dir', + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), + required=True, + help='Path to the thirdparty directory to check.', +) +@click.help_option('-h', '--help') +def check_thirdparty_dir(thirdparty_dir): + """ + Check a thirdparty directory for problems. + """ + utils_thirdparty.find_problems(dest_dir=thirdparty_dir) + + +if __name__ == '__main__': + check_thirdparty_dir() diff --git a/etc/scripts/fetch_requirements.py b/etc/scripts/fetch_requirements.py new file mode 100644 index 00000000..dfd202a7 --- /dev/null +++ b/etc/scripts/fetch_requirements.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import itertools + +import click + +import utils_thirdparty + + +@click.command() + +@click.option('-r', '--requirements-file', + type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), + metavar='FILE', + multiple=True, + default=['requirements.txt'], + show_default=True, + help='Path to the requirements file to use for thirdparty packages.', +) +@click.option('-d', '--thirdparty-dir', + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), + metavar='DIR', + default=utils_thirdparty.THIRDPARTY_DIR, + show_default=True, + help='Path to the thirdparty directory.', +) +@click.option('-p', '--python-version', + type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), + metavar='INT', + multiple=True, + default=['36'], + show_default=True, + help='Python version to use for this build.', +) +@click.option('-o', '--operating-system', + type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), + metavar='OS', + multiple=True, + default=['linux'], + show_default=True, + help='OS to use for this build: one of linux, mac or windows.', +) +@click.option('-s', '--with-sources', + is_flag=True, + help='Fetch the corresponding source distributions.', +) +@click.option('-a', '--with-about', + is_flag=True, + help='Fetch the corresponding ABOUT and LICENSE files.', +) +@click.option('--allow-unpinned', + is_flag=True, + help='Allow requirements without pinned versions.', +) +@click.option('-s', '--only-sources', + is_flag=True, + help='Fetch only the corresponding source distributions.', +) +@click.option('-u', '--remote-links-url', + type=str, + metavar='URL', + default=utils_thirdparty.REMOTE_LINKS_URL, + show_default=True, + help='URL to a PyPI-like links web site. ' + 'Or local path to a directory with wheels.', +) + +@click.help_option('-h', '--help') +def fetch_requirements( + requirements_file, + thirdparty_dir, + python_version, + operating_system, + with_sources, + with_about, + allow_unpinned, + only_sources, + remote_links_url=utils_thirdparty.REMOTE_LINKS_URL, +): + """ + Fetch and save to THIRDPARTY_DIR all the required wheels for pinned + dependencies found in the `--requirement` FILE requirements file(s). Only + fetch wheels compatible with the provided `--python-version` and + `--operating-system`. + Also fetch the corresponding .ABOUT, .LICENSE and .NOTICE files together + with a virtualenv.pyz app. + + Use exclusively wheel not from PyPI but rather found in the PyPI-like link + repo ``remote_links_url`` if this is a URL. Treat this ``remote_links_url`` + as a local directory path to a wheels directory if this is not a a URL. + """ + + # fetch wheels + python_versions = python_version + operating_systems = operating_system + requirements_files = requirements_file + + if not only_sources: + envs = itertools.product(python_versions, operating_systems) + envs = (utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in envs) + + for env, reqf in itertools.product(envs, requirements_files): + + for package, error in utils_thirdparty.fetch_wheels( + environment=env, + requirements_file=reqf, + allow_unpinned=allow_unpinned, + dest_dir=thirdparty_dir, + remote_links_url=remote_links_url, + ): + if error: + print('Failed to fetch wheel:', package, ':', error) + + # optionally fetch sources + if with_sources or only_sources: + + for reqf in requirements_files: + for package, error in utils_thirdparty.fetch_sources( + requirements_file=reqf, + allow_unpinned=allow_unpinned, + dest_dir=thirdparty_dir, + remote_links_url=remote_links_url, + ): + if error: + print('Failed to fetch source:', package, ':', error) + + if with_about: + utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir) + utils_thirdparty.find_problems( + dest_dir=thirdparty_dir, + report_missing_sources=with_sources or only_sources, + report_missing_wheels=not only_sources, + ) + + +if __name__ == '__main__': + fetch_requirements() diff --git a/etc/scripts/fix_thirdparty.py b/etc/scripts/fix_thirdparty.py new file mode 100644 index 00000000..b74b4979 --- /dev/null +++ b/etc/scripts/fix_thirdparty.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import click + +import utils_thirdparty + + +@click.command() + +@click.option('-d', '--thirdparty-dir', + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), + required=True, + help='Path to the thirdparty directory to fix.', +) +@click.option('--build-wheels', + is_flag=True, + help='Build all missing wheels .', +) +@click.option('--build-remotely', + is_flag=True, + help='Build missing wheels remotely.', +) +@click.help_option('-h', '--help') +def fix_thirdparty_dir( + thirdparty_dir, + build_wheels, + build_remotely, +): + """ + Fix a thirdparty directory of dependent package wheels and sdist. + + Multiple fixes are applied: + - fetch or build missing binary wheels + - fetch missing source distributions + - derive, fetch or add missing ABOUT files + - fetch missing .LICENSE and .NOTICE files + - remove outdated package versions and the ABOUT, .LICENSE and .NOTICE files + + Optionally build missing binary wheels for all supported OS and Python + version combos locally or remotely. + """ + print('***FETCH*** MISSING WHEELS') + package_envts_not_fetched = utils_thirdparty.fetch_missing_wheels(dest_dir=thirdparty_dir) + print('***FETCH*** MISSING SOURCES') + src_name_ver_not_fetched = utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir) + + package_envts_not_built = [] + if build_wheels: + print('***BUILD*** MISSING WHEELS') + package_envts_not_built, _wheel_filenames_built = utils_thirdparty.build_missing_wheels( + packages_and_envts=package_envts_not_fetched, + build_remotely=build_remotely, + dest_dir=thirdparty_dir, + ) + + print('***ADD*** ABOUT AND LICENSES') + utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir) + + # report issues + for name, version in src_name_ver_not_fetched: + print(f'{name}=={version}: Failed to fetch source distribution.') + + for package, envt in package_envts_not_built: + print( + f'{package.name}=={package.version}: Failed to build wheel ' + f'on {envt.operating_system} for Python {envt.python_version}') + + print('***FIND PROBLEMS***') + utils_thirdparty.find_problems(dest_dir=thirdparty_dir) + + +if __name__ == '__main__': + fix_thirdparty_dir() diff --git a/etc/scripts/gen_requirements.py b/etc/scripts/gen_requirements.py new file mode 100644 index 00000000..c917c873 --- /dev/null +++ b/etc/scripts/gen_requirements.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import click +import utils_requirements + + +@click.command() + +@click.option('-s', '--site-packages-dir', + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True), + required=True, + metavar='DIR', + help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages', +) +@click.option('-r', '--requirements-file', + type=click.Path(path_type=str, dir_okay=False), + metavar='FILE', + default='requirements.txt', + show_default=True, + help='Path to the requirements file to update or create.', +) +@click.help_option('-h', '--help') +def gen_requirements(site_packages_dir, requirements_file): + """ + Create or replace the `--requirements-file` file FILE requirements file with all + locally installed Python packages.all Python packages found installed in `--site-packages-dir` + """ + utils_requirements.lock_requirements( + requirements_file=requirements_file, + site_packages_dir=site_packages_dir, + ) + + +if __name__ == '__main__': + gen_requirements() diff --git a/etc/scripts/gen_requirements_dev.py b/etc/scripts/gen_requirements_dev.py new file mode 100644 index 00000000..91e0ce61 --- /dev/null +++ b/etc/scripts/gen_requirements_dev.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import click +import utils_requirements + + +@click.command() + +@click.option('-s', '--site-packages-dir', + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True), + required=True, + metavar='DIR', + help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages', +) +@click.option('-d', '--dev-requirements-file', + type=click.Path(path_type=str, dir_okay=False), + metavar='FILE', + default='requirements-dev.txt', + show_default=True, + help='Path to the dev requirements file to update or create.', +) +@click.option('-r', '--main-requirements-file', + type=click.Path(path_type=str, dir_okay=False), + default='requirements.txt', + metavar='FILE', + show_default=True, + help='Path to the main requirements file. Its requirements will be excluded ' + 'from the generated dev requirements.', +) +@click.help_option('-h', '--help') +def gen_dev_requirements(site_packages_dir, dev_requirements_file, main_requirements_file): + """ + Create or overwrite the `--dev-requirements-file` pip requirements FILE with + all Python packages found installed in `--site-packages-dir`. Exclude + package names also listed in the --main-requirements-file pip requirements + FILE (that are assume to the production requirements and therefore to always + be present in addition to the development requirements). + """ + utils_requirements.lock_dev_requirements( + dev_requirements_file=dev_requirements_file, + main_requirements_file=main_requirements_file, + site_packages_dir=site_packages_dir + ) + + +if __name__ == '__main__': + gen_dev_requirements() diff --git a/etc/scripts/test_utils_pip_compatibility_tags.py b/etc/scripts/test_utils_pip_compatibility_tags.py new file mode 100644 index 00000000..30c4ddab --- /dev/null +++ b/etc/scripts/test_utils_pip_compatibility_tags.py @@ -0,0 +1,128 @@ +"""Generate and work with PEP 425 Compatibility Tags. + +copied from pip-20.3.1 pip/tests/unit/test_utils_compatibility_tags.py +download_url: https://raw.githubusercontent.com/pypa/pip/20.3.1/tests/unit/test_utils_compatibility_tags.py + +Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + +from unittest.mock import patch +import sysconfig + +import pytest + +import utils_pip_compatibility_tags + + +@pytest.mark.parametrize('version_info, expected', [ + ((2,), '2'), + ((2, 8), '28'), + ((3,), '3'), + ((3, 6), '36'), + # Test a tuple of length 3. + ((3, 6, 5), '36'), + # Test a 2-digit minor version. + ((3, 10), '310'), +]) +def test_version_info_to_nodot(version_info, expected): + actual = pip_compatibility_tags.version_info_to_nodot(version_info) + assert actual == expected + + +class Testcompatibility_tags(object): + + def mock_get_config_var(self, **kwd): + """ + Patch sysconfig.get_config_var for arbitrary keys. + """ + get_config_var = sysconfig.get_config_var + + def _mock_get_config_var(var): + if var in kwd: + return kwd[var] + return get_config_var(var) + + return _mock_get_config_var + + def test_no_hyphen_tag(self): + """ + Test that no tag contains a hyphen. + """ + import pip._internal.utils.compatibility_tags + + mock_gcf = self.mock_get_config_var(SOABI='cpython-35m-darwin') + + with patch('sysconfig.get_config_var', mock_gcf): + supported = pip._internal.utils.compatibility_tags.get_supported() + + for tag in supported: + assert '-' not in tag.interpreter + assert '-' not in tag.abi + assert '-' not in tag.platform + + +class TestManylinux2010Tags(object): + + @pytest.mark.parametrize("manylinux2010,manylinux1", [ + ("manylinux2010_x86_64", "manylinux1_x86_64"), + ("manylinux2010_i686", "manylinux1_i686"), + ]) + def test_manylinux2010_implies_manylinux1(self, manylinux2010, manylinux1): + """ + Specifying manylinux2010 implies manylinux1. + """ + groups = {} + supported = pip_compatibility_tags.get_supported(platforms=[manylinux2010]) + for tag in supported: + groups.setdefault( + (tag.interpreter, tag.abi), [] + ).append(tag.platform) + + for arches in groups.values(): + if arches == ['any']: + continue + assert arches[:2] == [manylinux2010, manylinux1] + + +class TestManylinux2014Tags(object): + + @pytest.mark.parametrize("manylinuxA,manylinuxB", [ + ("manylinux2014_x86_64", ["manylinux2010_x86_64", "manylinux1_x86_64"]), + ("manylinux2014_i686", ["manylinux2010_i686", "manylinux1_i686"]), + ]) + def test_manylinuxA_implies_manylinuxB(self, manylinuxA, manylinuxB): + """ + Specifying manylinux2014 implies manylinux2010/manylinux1. + """ + groups = {} + supported = pip_compatibility_tags.get_supported(platforms=[manylinuxA]) + for tag in supported: + groups.setdefault( + (tag.interpreter, tag.abi), [] + ).append(tag.platform) + + expected_arches = [manylinuxA] + expected_arches.extend(manylinuxB) + for arches in groups.values(): + if arches == ['any']: + continue + assert arches[:3] == expected_arches diff --git a/etc/scripts/test_utils_pip_compatibility_tags.py.ABOUT b/etc/scripts/test_utils_pip_compatibility_tags.py.ABOUT new file mode 100644 index 00000000..07eee35a --- /dev/null +++ b/etc/scripts/test_utils_pip_compatibility_tags.py.ABOUT @@ -0,0 +1,14 @@ +about_resource: test_utils_pip_compatibility_tags.py + +type: github +namespace: pypa +name: pip +version: 20.3.1 +subpath: tests/unit/test_utils_compatibility_tags.py + +package_url: pkg:github/pypa/pip@20.3.1#tests/unit/test_utils_compatibility_tags.py + +download_url: https://raw.githubusercontent.com/pypa/pip/20.3.1/tests/unit/test_utils_compatibility_tags.py +copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) +license_expression: mit +notes: subset copied from pip for tag handling diff --git a/etc/scripts/test_utils_pypi_supported_tags.py b/etc/scripts/test_utils_pypi_supported_tags.py new file mode 100644 index 00000000..9ad68b21 --- /dev/null +++ b/etc/scripts/test_utils_pypi_supported_tags.py @@ -0,0 +1,91 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from utils_pypi_supported_tags import validate_platforms_for_pypi + +""" +Wheel platform checking tests + +Copied and modified on 2020-12-24 from +https://github.com/pypa/warehouse/blob/37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d/tests/unit/forklift/test_legacy.py +""" + + +def validate_wheel_filename_for_pypi(filename): + """ + Validate if the filename is a PyPI/warehouse-uploadable wheel file name + with supported platform tags. Return a list of unsupported platform tags or + an empty list if all tags are supported. + """ + from utils_thirdparty import Wheel + wheel = Wheel.from_filename(filename) + return validate_platforms_for_pypi(wheel.platforms) + + +@pytest.mark.parametrize( + "plat", + [ + "any", + "win32", + "win_amd64", + "win_ia64", + "manylinux1_i686", + "manylinux1_x86_64", + "manylinux2010_i686", + "manylinux2010_x86_64", + "manylinux2014_i686", + "manylinux2014_x86_64", + "manylinux2014_aarch64", + "manylinux2014_armv7l", + "manylinux2014_ppc64", + "manylinux2014_ppc64le", + "manylinux2014_s390x", + "manylinux_2_5_i686", + "manylinux_2_12_x86_64", + "manylinux_2_17_aarch64", + "manylinux_2_17_armv7l", + "manylinux_2_17_ppc64", + "manylinux_2_17_ppc64le", + "manylinux_3_0_s390x", + "macosx_10_6_intel", + "macosx_10_13_x86_64", + "macosx_11_0_x86_64", + "macosx_10_15_arm64", + "macosx_11_10_universal2", + # A real tag used by e.g. some numpy wheels + ( + "macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64." + "macosx_10_10_intel.macosx_10_10_x86_64" + ), + ], +) +def test_is_valid_pypi_wheel_return_true_for_supported_wheel(plat): + filename = f"foo-1.2.3-cp34-none-{plat}.whl" + assert not validate_wheel_filename_for_pypi(filename) + + +@pytest.mark.parametrize( + "plat", + [ + "linux_x86_64", + "linux_x86_64.win32", + "macosx_9_2_x86_64", + "macosx_12_2_arm64", + "macosx_10_15_amd64", + ], +) +def test_is_valid_pypi_wheel_raise_exception_for_aunsupported_wheel(plat): + filename = f"foo-1.2.3-cp34-none-{plat}.whl" + invalid = validate_wheel_filename_for_pypi(filename) + assert invalid diff --git a/etc/scripts/test_utils_pypi_supported_tags.py.ABOUT b/etc/scripts/test_utils_pypi_supported_tags.py.ABOUT new file mode 100644 index 00000000..176efacc --- /dev/null +++ b/etc/scripts/test_utils_pypi_supported_tags.py.ABOUT @@ -0,0 +1,17 @@ +about_resource: test_utils_pypi_supported_tags.py + +type: github +namespace: pypa +name: warehouse +version: 37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d +subpath: tests/unit/forklift/test_legacy.py + +package_url: pkg:github/pypa/warehouse@37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d#tests/unit/forklift/test_legacy.py + +download_url: https://github.com/pypa/warehouse/blob/37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d/tests/unit/forklift/test_legacy.py +copyright: Copyright (c) The warehouse developers +homepage_url: https://warehouse.readthedocs.io +license_expression: apache-2.0 +notes: Test for wheel platform checking copied and heavily modified on + 2020-12-24 from warehouse. This contains the basic functions to check if a + wheel file name is would be supported for uploading to PyPI. diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py new file mode 100644 index 00000000..bb37de1c --- /dev/null +++ b/etc/scripts/utils_dejacode.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import io +import os +import zipfile + +import requests +import saneyaml + +from packaging import version as packaging_version + +""" +Utility to create and retrieve package and ABOUT file data from DejaCode. +""" + +DEJACODE_API_KEY = os.environ.get('DEJACODE_API_KEY', '') +DEJACODE_API_URL = os.environ.get('DEJACODE_API_URL', '') + +DEJACODE_API_URL_PACKAGES = f'{DEJACODE_API_URL}packages/' +DEJACODE_API_HEADERS = { + 'Authorization': 'Token {}'.format(DEJACODE_API_KEY), + 'Accept': 'application/json; indent=4', +} + + +def can_do_api_calls(): + if not DEJACODE_API_KEY and DEJACODE_API_URL: + print('DejaCode DEJACODE_API_KEY and DEJACODE_API_URL not configured. Doing nothing') + return False + else: + return True + + +def fetch_dejacode_packages(params): + """ + Return a list of package data mappings calling the package API with using + `params` or an empty list. + """ + if not can_do_api_calls(): + return [] + + response = requests.get( + DEJACODE_API_URL_PACKAGES, + params=params, + headers=DEJACODE_API_HEADERS, + ) + + return response.json()['results'] + + +def get_package_data(distribution): + """ + Return a mapping of package data or None for a Distribution `distribution`. + """ + results = fetch_dejacode_packages(distribution.identifiers()) + + len_results = len(results) + + if len_results == 1: + return results[0] + + elif len_results > 1: + print(f'More than 1 entry exists, review at: {DEJACODE_API_URL_PACKAGES}') + else: + print('Could not find package:', distribution.download_url) + + +def update_with_dejacode_data(distribution): + """ + Update the Distribution `distribution` with DejaCode package data. Return + True if data was updated. + """ + package_data = get_package_data(distribution) + if package_data: + return distribution.update(package_data, keep_extra=False) + + print(f'No package found for: {distribution}') + + +def update_with_dejacode_about_data(distribution): + """ + Update the Distribution `distribution` wiht ABOUT code data fetched from + DejaCode. Return True if data was updated. + """ + package_data = get_package_data(distribution) + if package_data: + package_api_url = package_data['api_url'] + about_url = f'{package_api_url}about' + response = requests.get(about_url, headers=DEJACODE_API_HEADERS) + # note that this is YAML-formatted + about_text = response.json()['about_data'] + about_data = saneyaml.load(about_text) + + return distribution.update(about_data, keep_extra=True) + + print(f'No package found for: {distribution}') + + +def fetch_and_save_about_files(distribution, dest_dir='thirdparty'): + """ + Fetch and save in `dest_dir` the .ABOUT, .LICENSE and .NOTICE files fetched + from DejaCode for a Distribution `distribution`. Return True if files were + fetched. + """ + package_data = get_package_data(distribution) + if package_data: + package_api_url = package_data['api_url'] + about_url = f'{package_api_url}about_files' + response = requests.get(about_url, headers=DEJACODE_API_HEADERS) + about_zip = response.content + with io.BytesIO(about_zip) as zf: + with zipfile.ZipFile(zf) as zi: + zi.extractall(path=dest_dir) + return True + + print(f'No package found for: {distribution}') + + +def find_latest_dejacode_package(distribution): + """ + Return a mapping of package data for the closest version to + a Distribution `distribution` or None. + Return the newest of the packages if prefer_newest is True. + Filter out version-specific attributes. + """ + ids = distribution.purl_identifiers(skinny=True) + packages = fetch_dejacode_packages(params=ids) + if not packages: + return + + for package_data in packages: + matched = ( + package_data['download_url'] == distribution.download_url + and package_data['version'] == distribution.version + and package_data['filename'] == distribution.filename + ) + + if matched: + return package_data + + # there was no exact match, find the latest version + # TODO: consider the closest version rather than the latest + # or the version that has the best data + with_versions = [(packaging_version.parse(p['version']), p) for p in packages] + with_versions = sorted(with_versions) + latest_version, latest_package_version = sorted(with_versions)[-1] + print( + f'Found DejaCode latest version: {latest_version} ' + f'for dist: {distribution.package_url}', + ) + + return latest_package_version + + +def create_dejacode_package(distribution): + """ + Create a new DejaCode Package a Distribution `distribution`. + Return the new or existing package data. + """ + if not can_do_api_calls(): + return + + existing_package_data = get_package_data(distribution) + if existing_package_data: + return existing_package_data + + print(f'Creating new DejaCode package for: {distribution}') + + new_package_payload = { + # Trigger data collection, scan, and purl + 'collect_data': 1, + } + + fields_to_carry_over = [ + 'download_url' + 'type', + 'namespace', + 'name', + 'version', + 'qualifiers', + 'subpath', + 'license_expression', + 'copyright', + 'description', + 'homepage_url', + 'primary_language', + 'notice_text', + ] + + for field in fields_to_carry_over: + value = getattr(distribution, field, None) + if value: + new_package_payload[field] = value + + response = requests.post( + DEJACODE_API_URL_PACKAGES, + data=new_package_payload, + headers=DEJACODE_API_HEADERS, + ) + new_package_data = response.json() + if response.status_code != 201: + raise Exception(f'Error, cannot create package for: {distribution}') + + print(f'New Package created at: {new_package_data["absolute_url"]}') + return new_package_data diff --git a/etc/scripts/utils_pip_compatibility_tags.py b/etc/scripts/utils_pip_compatibility_tags.py new file mode 100644 index 00000000..4c6529b1 --- /dev/null +++ b/etc/scripts/utils_pip_compatibility_tags.py @@ -0,0 +1,192 @@ +"""Generate and work with PEP 425 Compatibility Tags. + +copied from pip-20.3.1 pip/_internal/utils/compatibility_tags.py +download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/utils/compatibility_tags.py + +Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + +import re + +from packaging.tags import ( + compatible_tags, + cpython_tags, + generic_tags, + interpreter_name, + interpreter_version, + mac_platforms, +) + +_osx_arch_pat = re.compile(r'(.+)_(\d+)_(\d+)_(.+)') + + +def version_info_to_nodot(version_info): + # type: (Tuple[int, ...]) -> str + # Only use up to the first two numbers. + return ''.join(map(str, version_info[:2])) + + +def _mac_platforms(arch): + # type: (str) -> List[str] + match = _osx_arch_pat.match(arch) + if match: + name, major, minor, actual_arch = match.groups() + mac_version = (int(major), int(minor)) + arches = [ + # Since we have always only checked that the platform starts + # with "macosx", for backwards-compatibility we extract the + # actual prefix provided by the user in case they provided + # something like "macosxcustom_". It may be good to remove + # this as undocumented or deprecate it in the future. + '{}_{}'.format(name, arch[len('macosx_'):]) + for arch in mac_platforms(mac_version, actual_arch) + ] + else: + # arch pattern didn't match (?!) + arches = [arch] + return arches + + +def _custom_manylinux_platforms(arch): + # type: (str) -> List[str] + arches = [arch] + arch_prefix, arch_sep, arch_suffix = arch.partition('_') + if arch_prefix == 'manylinux2014': + # manylinux1/manylinux2010 wheels run on most manylinux2014 systems + # with the exception of wheels depending on ncurses. PEP 599 states + # manylinux1/manylinux2010 wheels should be considered + # manylinux2014 wheels: + # https://www.python.org/dev/peps/pep-0599/#backwards-compatibility-with-manylinux2010-wheels + if arch_suffix in {'i686', 'x86_64'}: + arches.append('manylinux2010' + arch_sep + arch_suffix) + arches.append('manylinux1' + arch_sep + arch_suffix) + elif arch_prefix == 'manylinux2010': + # manylinux1 wheels run on most manylinux2010 systems with the + # exception of wheels depending on ncurses. PEP 571 states + # manylinux1 wheels should be considered manylinux2010 wheels: + # https://www.python.org/dev/peps/pep-0571/#backwards-compatibility-with-manylinux1-wheels + arches.append('manylinux1' + arch_sep + arch_suffix) + return arches + + +def _get_custom_platforms(arch): + # type: (str) -> List[str] + arch_prefix, _arch_sep, _arch_suffix = arch.partition('_') + if arch.startswith('macosx'): + arches = _mac_platforms(arch) + elif arch_prefix in ['manylinux2014', 'manylinux2010']: + arches = _custom_manylinux_platforms(arch) + else: + arches = [arch] + return arches + + +def _expand_allowed_platforms(platforms): + # type: (Optional[List[str]]) -> Optional[List[str]] + if not platforms: + return None + + seen = set() + result = [] + + for p in platforms: + if p in seen: + continue + additions = [c for c in _get_custom_platforms(p) if c not in seen] + seen.update(additions) + result.extend(additions) + + return result + + +def _get_python_version(version): + # type: (str) -> PythonVersion + if len(version) > 1: + return int(version[0]), int(version[1:]) + else: + return (int(version[0]),) + + +def _get_custom_interpreter(implementation=None, version=None): + # type: (Optional[str], Optional[str]) -> str + if implementation is None: + implementation = interpreter_name() + if version is None: + version = interpreter_version() + return "{}{}".format(implementation, version) + + +def get_supported( + version=None, # type: Optional[str] + platforms=None, # type: Optional[List[str]] + impl=None, # type: Optional[str] + abis=None # type: Optional[List[str]] +): + # type: (...) -> List[Tag] + """Return a list of supported tags for each version specified in + `versions`. + + :param version: a string version, of the form "33" or "32", + or None. The version will be assumed to support our ABI. + :param platforms: specify a list of platforms you want valid + tags for, or None. If None, use the local system platform. + :param impl: specify the exact implementation you want valid + tags for, or None. If None, use the local interpreter impl. + :param abis: specify a list of abis you want valid + tags for, or None. If None, use the local interpreter abi. + """ + supported = [] # type: List[Tag] + + python_version = None # type: Optional[PythonVersion] + if version is not None: + python_version = _get_python_version(version) + + interpreter = _get_custom_interpreter(impl, version) + + platforms = _expand_allowed_platforms(platforms) + + is_cpython = (impl or interpreter_name()) == "cp" + if is_cpython: + supported.extend( + cpython_tags( + python_version=python_version, + abis=abis, + platforms=platforms, + ) + ) + else: + supported.extend( + generic_tags( + interpreter=interpreter, + abis=abis, + platforms=platforms, + ) + ) + supported.extend( + compatible_tags( + python_version=python_version, + interpreter=interpreter, + platforms=platforms, + ) + ) + + return supported diff --git a/etc/scripts/utils_pip_compatibility_tags.py.ABOUT b/etc/scripts/utils_pip_compatibility_tags.py.ABOUT new file mode 100644 index 00000000..7bbb026b --- /dev/null +++ b/etc/scripts/utils_pip_compatibility_tags.py.ABOUT @@ -0,0 +1,14 @@ +about_resource: utils_pip_compatibility_tags.py + +type: github +namespace: pypa +name: pip +version: 20.3.1 +subpath: src/pip/_internal/utils/compatibility_tags.py + +package_url: pkg:github/pypa/pip@20.3.1#src/pip/_internal/utils/compatibility_tags.py + +download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/utils/compatibility_tags.py +copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) +license_expression: mit +notes: subset copied from pip for tag handling \ No newline at end of file diff --git a/etc/scripts/utils_pypi_supported_tags.py b/etc/scripts/utils_pypi_supported_tags.py new file mode 100644 index 00000000..8dcb70fb --- /dev/null +++ b/etc/scripts/utils_pypi_supported_tags.py @@ -0,0 +1,109 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +""" +Wheel platform checking + +Copied and modified on 2020-12-24 from +https://github.com/pypa/warehouse/blob/37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d/warehouse/forklift/legacy.py + +This contains the basic functions to check if a wheel file name is would be +supported for uploading to PyPI. +""" + +# These platforms can be handled by a simple static list: +_allowed_platforms = { + "any", + "win32", + "win_amd64", + "win_ia64", + "manylinux1_x86_64", + "manylinux1_i686", + "manylinux2010_x86_64", + "manylinux2010_i686", + "manylinux2014_x86_64", + "manylinux2014_i686", + "manylinux2014_aarch64", + "manylinux2014_armv7l", + "manylinux2014_ppc64", + "manylinux2014_ppc64le", + "manylinux2014_s390x", + "linux_armv6l", + "linux_armv7l", +} +# macosx is a little more complicated: +_macosx_platform_re = re.compile(r"macosx_(?P\d+)_(\d+)_(?P.*)") +_macosx_arches = { + "ppc", + "ppc64", + "i386", + "x86_64", + "arm64", + "intel", + "fat", + "fat32", + "fat64", + "universal", + "universal2", +} +_macosx_major_versions = { + "10", + "11", +} + +# manylinux pep600 is a little more complicated: +_manylinux_platform_re = re.compile(r"manylinux_(\d+)_(\d+)_(?P.*)") +_manylinux_arches = { + "x86_64", + "i686", + "aarch64", + "armv7l", + "ppc64", + "ppc64le", + "s390x", +} + + +def is_supported_platform_tag(platform_tag): + """ + Return True if the ``platform_tag`` is supported on PyPI. + """ + if platform_tag in _allowed_platforms: + return True + m = _macosx_platform_re.match(platform_tag) + if ( + m + and m.group("major") in _macosx_major_versions + and m.group("arch") in _macosx_arches + ): + return True + m = _manylinux_platform_re.match(platform_tag) + if m and m.group("arch") in _manylinux_arches: + return True + return False + + +def validate_platforms_for_pypi(platforms): + """ + Validate if the wheel platforms are supported platform tags on Pypi. Return + a list of unsupported platform tags or an empty list if all tags are + supported. + """ + + # Check that if it's a binary wheel, it's on a supported platform + invalid_tags = [] + for plat in platforms: + if not is_supported_platform_tag(plat): + invalid_tags.append(plat) + return invalid_tags diff --git a/etc/scripts/utils_pypi_supported_tags.py.ABOUT b/etc/scripts/utils_pypi_supported_tags.py.ABOUT new file mode 100644 index 00000000..228a538a --- /dev/null +++ b/etc/scripts/utils_pypi_supported_tags.py.ABOUT @@ -0,0 +1,17 @@ +about_resource: utils_pypi_supported_tags.py + +type: github +namespace: pypa +name: warehouse +version: 37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d +subpath: warehouse/forklift/legacy.py + +package_url: pkg:github/pypa/warehouse@37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d#warehouse/forklift/legacy.py + +download_url: https://github.com/pypa/warehouse/blob/37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d/warehouse/forklift/legacy.py +copyright: Copyright (c) The warehouse developers +homepage_url: https://warehouse.readthedocs.io +license_expression: apache-2.0 +notes: Wheel platform checking copied and heavily modified on 2020-12-24 from + warehouse. This contains the basic functions to check if a wheel file name is + would be supported for uploading to PyPI. diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py new file mode 100644 index 00000000..8b088adf --- /dev/null +++ b/etc/scripts/utils_requirements.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import subprocess + +""" +Utilities to manage requirements files and call pip. +NOTE: this should use ONLY the standard library and not import anything else. +""" + + +def load_requirements(requirements_file='requirements.txt', force_pinned=True): + """ + Yield package (name, version) tuples for each requirement in a `requirement` + file. Every requirement versions must be pinned if `force_pinned` is True. + Otherwise un-pinned requirements are returned with a None version + """ + with open(requirements_file) as reqs: + req_lines = reqs.read().splitlines(False) + return get_required_name_versions(req_lines, force_pinned) + + +def get_required_name_versions(requirement_lines, force_pinned=True): + """ + Yield required (name, version) tuples given a`requirement_lines` iterable of + requirement text lines. Every requirement versions must be pinned if + `force_pinned` is True. Otherwise un-pinned requirements are returned with a + None version + """ + for req_line in requirement_lines: + req_line = req_line.strip() + if not req_line or req_line.startswith('#'): + continue + if '==' not in req_line and force_pinned: + raise Exception(f'Requirement version is not pinned: {req_line}') + name = req_line + version = None + else: + name, _, version = req_line.partition('==') + name = name.lower().strip() + version = version.lower().strip() + yield name, version + + +def parse_requires(requires): + """ + Return a list of requirement lines extracted from the `requires` text from + a setup.cfg *_requires section such as the "install_requires" section. + """ + requires = [c for c in requires.splitlines(False) if c] + if not requires: + return [] + + requires = [''.join(r.split()) for r in requires if r and r.strip()] + return sorted(requires) + + +def lock_requirements(requirements_file='requirements.txt', site_packages_dir=None): + """ + Freeze and lock current installed requirements and save this to the + `requirements_file` requirements file. + """ + with open(requirements_file, 'w') as fo: + fo.write(get_installed_reqs(site_packages_dir=site_packages_dir)) + + +def lock_dev_requirements( + dev_requirements_file='requirements-dev.txt', + main_requirements_file='requirements.txt', + site_packages_dir=None, +): + """ + Freeze and lock current installed development-only requirements and save + this to the `dev_requirements_file` requirements file. Development-only is + achieved by subtracting requirements from the `main_requirements_file` + requirements file from the current requirements using package names (and + ignoring versions). + """ + main_names = {n for n, _v in load_requirements(main_requirements_file)} + all_reqs = get_installed_reqs(site_packages_dir=site_packages_dir) + all_req_lines = all_reqs.splitlines(False) + all_req_nvs = get_required_name_versions(all_req_lines) + dev_only_req_nvs = {n: v for n, v in all_req_nvs if n not in main_names} + + new_reqs = '\n'.join(f'{n}=={v}' for n, v in sorted(dev_only_req_nvs.items())) + with open(dev_requirements_file, 'w') as fo: + fo.write(new_reqs) + + +def get_installed_reqs(site_packages_dir): + """ + Return the installed pip requirements as text found in `site_packages_dir` as a text. + """ + # Also include these packages in the output with --all: wheel, distribute, setuptools, pip + args = ['pip', 'freeze', '--exclude-editable', '--all', '--path', site_packages_dir] + return subprocess.check_output(args, encoding='utf-8') diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py new file mode 100644 index 00000000..360f07a6 --- /dev/null +++ b/etc/scripts/utils_thirdparty.py @@ -0,0 +1,2940 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from collections import defaultdict +import email +import itertools +import operator +import os +import re +import shutil +import subprocess +import tarfile +import tempfile +import time +import urllib + +import attr +import license_expression +import packageurl +import utils_pip_compatibility_tags +import utils_pypi_supported_tags +import requests +import saneyaml + +from commoncode import fileutils +from commoncode.hash import multi_checksums +from packaging import tags as packaging_tags +from packaging import version as packaging_version +from utils_requirements import load_requirements + +""" +Utilities to manage Python thirparty libraries source, binaries and metadata in +local directories and remote repositories. + +- update pip requirement files from installed packages for prod. and dev. +- build and save wheels for all required packages +- also build variants for wheels with native code for all each supported + operating systems (Linux, macOS, Windows) and Python versions (3.x) + combinations using remote Ci jobs +- collect source distributions for all required packages +- keep in sync wheels, distributions, ABOUT and LICENSE files to a PyPI-like + repository (using GitHub) +- create, update and fetch ABOUT, NOTICE and LICENSE metadata for all distributions + + +Approach +-------- + +The processing is organized around these key objects: + +- A PyPiPackage represents a PyPI package with its name and version. It tracks + the downloadable Distribution objects for that version: + + - one Sdist source Distribution object + - a list of Wheel binary Distribution objects + +- A Distribution (either a Wheel or Sdist) is identified by and created from its + filename. It also has the metadata used to populate an .ABOUT file and + document origin and license. A Distribution can be fetched from Repository. + Metadata can be loaded from and dumped to ABOUT files and optionally from + DejaCode package data. + +- An Environment is a combination of a Python version and operating system. + A Wheel Distribution also has Python/OS tags is supports and these can be + supported in a given Environment. + +- Paths or URLs to "filenames" live in a Repository, either a plain + LinksRepository (an HTML page listing URLs or a local directory) or a + PypiRepository (a PyPI simple index where each package name has an HTML page + listing URLs to all distribution types and versions). + Repositories and Distributions are related through filenames. + + + The Wheel models code is partially derived from the mit-licensed pip and the + Distribution/Wheel/Sdist design has been heavily inspired by the packaging- + dists library https://github.com/uranusjr/packaging-dists by Tzu-ping Chung +""" + +TRACE = False + +# Supported environments +PYTHON_VERSIONS = '36', '37', '38', '39', + +ABIS_BY_PYTHON_VERSION = { + '36':['cp36', 'cp36m'], + '37':['cp37', 'cp37m'], + '38':['cp38', 'cp38m'], + '39':['cp39', 'cp39m'], +} + +PLATFORMS_BY_OS = { + 'linux': [ + 'linux_x86_64', + 'manylinux1_x86_64', + 'manylinux2014_x86_64', + 'manylinux2010_x86_64', + ], + 'macos': [ + 'macosx_10_6_intel', 'macosx_10_6_x86_64', + 'macosx_10_9_intel', 'macosx_10_9_x86_64', + 'macosx_10_10_intel', 'macosx_10_10_x86_64', + 'macosx_10_11_intel', 'macosx_10_11_x86_64', + 'macosx_10_12_intel', 'macosx_10_12_x86_64', + 'macosx_10_13_intel', 'macosx_10_13_x86_64', + 'macosx_10_14_intel', 'macosx_10_14_x86_64', + 'macosx_10_15_intel', 'macosx_10_15_x86_64', + ], + 'windows': [ + 'win_amd64', + ], +} + +THIRDPARTY_DIR = 'thirdparty' +CACHE_THIRDPARTY_DIR = '.cache/thirdparty' + +REMOTE_LINKS_URL = 'https://thirdparty.aboutcode.org/pypi' + +EXTENSIONS_APP = '.pyz', +EXTENSIONS_SDIST = '.tar.gz', '.tar.bz2', '.zip', '.tar.xz', +EXTENSIONS_INSTALLABLE = EXTENSIONS_SDIST + ('.whl',) +EXTENSIONS_ABOUT = '.ABOUT', '.LICENSE', '.NOTICE', +EXTENSIONS = EXTENSIONS_INSTALLABLE + EXTENSIONS_ABOUT + EXTENSIONS_APP + +PYPI_SIMPLE_URL = 'https://pypi.org/simple' + +LICENSEDB_API_URL = 'https://scancode-licensedb.aboutcode.org' + +LICENSING = license_expression.Licensing() + +################################################################################ +# +# Fetch remote wheels and sources locally +# +################################################################################ + + +def fetch_wheels( + environment=None, + requirements_file='requirements.txt', + allow_unpinned=False, + dest_dir=THIRDPARTY_DIR, + remote_links_url=REMOTE_LINKS_URL, +): + """ + Download all of the wheel of packages listed in the ``requirements_file`` + requirements file into ``dest_dir`` directory. + + Only get wheels for the ``environment`` Enviromnent constraints. If the + provided ``environment`` is None then the current Python interpreter + environment is used implicitly. + + Only accept pinned requirements (e.g. with a version) unless + ``allow_unpinned`` is True. + + Use exclusively direct downloads from a remote repo at URL + ``remote_links_url``. If ``remote_links_url`` is a path, use this as a + directory of links instead of a URL. + + Yield tuples of (PypiPackage, error) where is None on success. + """ + missed = [] + + if not allow_unpinned: + force_pinned = True + else: + force_pinned = False + + rrp = list(get_required_remote_packages( + requirements_file=requirements_file, + force_pinned=force_pinned, + remote_links_url=remote_links_url, + )) + + fetched_filenames = set() + for name, version, package in rrp: + if not package: + missed.append((name, version,)) + nv = f'{name}=={version}' if version else name + yield None, f'fetch_wheels: Missing package in remote repo: {nv}' + + else: + fetched_filename = package.fetch_wheel( + environment=environment, + fetched_filenames=fetched_filenames, + dest_dir=dest_dir, + ) + + if fetched_filename: + fetched_filenames.add(fetched_filename) + error = None + else: + if fetched_filename in fetched_filenames: + error = None + else: + error = f'Failed to fetch' + yield package, error + + if missed: + rr = get_remote_repo() + print() + print(f'===> fetch_wheels: Missed some packages') + for n, v in missed: + nv = f'{n}=={v}' if v else n + print(f'Missed package {nv} in remote repo, has only:') + for pv in rr.get_versions(n): + print(' ', pv) + + +def fetch_sources( + requirements_file='requirements.txt', + allow_unpinned=False, + dest_dir=THIRDPARTY_DIR, + remote_links_url=REMOTE_LINKS_URL, +): + """ + Download all of the dependent package sources listed in the + ``requirements_file`` requirements file into ``dest_dir`` destination + directory. + + Use direct downloads to achieve this (not pip download). Use exclusively the + packages found from a remote repo at URL ``remote_links_url``. If + ``remote_links_url`` is a path, use this as a directory of links instead of + a URL. + + Only accept pinned requirements (e.g. with a version) unless + ``allow_unpinned`` is True. + + Yield tuples of (PypiPackage, error message) for each package where error + message will empty on success. + """ + missed = [] + + if not allow_unpinned: + force_pinned = True + else: + force_pinned = False + + rrp = list(get_required_remote_packages( + requirements_file=requirements_file, + force_pinned=force_pinned, + remote_links_url=remote_links_url, + )) + + for name, version, package in rrp: + if not package: + missed.append((name, name,)) + nv = f'{name}=={version}' if version else name + yield None, f'fetch_sources: Missing package in remote repo: {nv}' + + elif not package.sdist: + yield package, f'Missing sdist in links' + + else: + fetched = package.fetch_sdist(dest_dir=dest_dir) + error = f'Failed to fetch' if not fetched else None + yield package, error + +################################################################################ +# +# Core models +# +################################################################################ + + +@attr.attributes +class NameVer: + name = attr.ib( + type=str, + metadata=dict(help='Python package name, lowercase and normalized.'), + ) + + version = attr.ib( + type=str, + metadata=dict(help='Python package version string.'), + ) + + @property + def normalized_name(self): + return NameVer.normalize_name(self.name) + + @staticmethod + def normalize_name(name): + """ + Return a normalized package name per PEP503, and copied from + https://www.python.org/dev/peps/pep-0503/#id4 + """ + return name and re.sub(r"[-_.]+", "-", name).lower() or name + + @staticmethod + def standardize_name(name): + """ + Return a standardized package name, e.g. lowercased and using - not _ + """ + return name and re.sub(r"[-_]+", "-", name).lower() or name + + @property + def name_ver(self): + return f'{self.name}-{self.version}' + + def sortable_name_version(self): + """ + Return a tuple of values to sort by name, then version. + This method is a suitable to use as key for sorting NameVer instances. + """ + return self.normalized_name, packaging_version.parse(self.version) + + @classmethod + def sorted(cls, namevers): + return sorted(namevers, key=cls.sortable_name_version) + + +@attr.attributes +class Distribution(NameVer): + + # field names that can be updated from another dist of mapping + updatable_fields = [ + 'license_expression', + 'copyright', + 'description', + 'homepage_url', + 'primary_language', + 'notice_text', + 'extra_data', + ] + + filename = attr.ib( + repr=False, + type=str, + default='', + metadata=dict(help='File name.'), + ) + + path_or_url = attr.ib( + repr=False, + type=str, + default='', + metadata=dict(help='Path or download URL.'), + ) + + sha256 = attr.ib( + repr=False, + type=str, + default='', + metadata=dict(help='SHA256 checksum.'), + ) + + sha1 = attr.ib( + repr=False, + type=str, + default='', + metadata=dict(help='SHA1 checksum.'), + ) + + md5 = attr.ib( + repr=False, + type=int, + default=0, + metadata=dict(help='MD5 checksum.'), + ) + + type = attr.ib( + repr=False, + type=str, + default='pypi', + metadata=dict(help='Package type'), + ) + + namespace = attr.ib( + repr=False, + type=str, + default='', + metadata=dict(help='Package URL namespace'), + ) + + qualifiers = attr.ib( + repr=False, + type=dict, + default=attr.Factory(dict), + metadata=dict(help='Package URL qualifiers'), + ) + + subpath = attr.ib( + repr=False, + type=str, + default='', + metadata=dict(help='Package URL subpath'), + ) + + size = attr.ib( + repr=False, + type=str, + default='', + metadata=dict(help='Size in bytes.'), + ) + + primary_language = attr.ib( + repr=False, + type=str, + default='Python', + metadata=dict(help='Primary Programming language.'), + ) + + description = attr.ib( + repr=False, + type=str, + default='', + metadata=dict(help='Description.'), + ) + + homepage_url = attr.ib( + repr=False, + type=str, + default='', + metadata=dict(help='Homepage URL'), + ) + + notes = attr.ib( + repr=False, + type=str, + default='', + metadata=dict(help='Notes.'), + ) + + copyright = attr.ib( + repr=False, + type=str, + default='', + metadata=dict(help='Copyright.'), + ) + + license_expression = attr.ib( + repr=False, + type=str, + default='', + metadata=dict(help='License expression'), + ) + + licenses = attr.ib( + repr=False, + type=list, + default=attr.Factory(list), + metadata=dict(help='List of license mappings.'), + ) + + notice_text = attr.ib( + repr=False, + type=str, + default='', + metadata=dict(help='Notice text'), + ) + + extra_data = attr.ib( + repr=False, + type=dict, + default=attr.Factory(dict), + metadata=dict(help='Extra data'), + ) + + @property + def package_url(self): + """ + Return a Package URL string of self. + """ + return str(packageurl.PackageURL(**self.purl_identifiers())) + + @property + def download_url(self): + if self.path_or_url and self.path_or_url.startswith('https://'): + return self.path_or_url + else: + return self.get_best_download_url() + + @property + def about_filename(self): + return f'{self.filename}.ABOUT' + + def has_about_file(self, dest_dir=THIRDPARTY_DIR): + return os.path.exists(os.path.join(dest_dir, self.about_filename)) + + @property + def about_download_url(self): + return self.build_remote_download_url(self.about_filename) + + @property + def notice_filename(self): + return f'{self.filename}.NOTICE' + + @property + def notice_download_url(self): + return self.build_remote_download_url(self.notice_filename) + + @classmethod + def from_path_or_url(cls, path_or_url): + """ + Return a distribution built from the data found in the filename of a + `path_or_url` string. Raise an exception if this is not a valid + filename. + """ + filename = os.path.basename(path_or_url.strip('/')) + dist = cls.from_filename(filename) + dist.path_or_url = path_or_url + return dist + + @classmethod + def get_dist_class(cls, filename): + if filename.endswith('.whl'): + return Wheel + elif filename.endswith(('.zip', '.tar.gz',)): + return Sdist + raise InvalidDistributionFilename(filename) + + @classmethod + def from_filename(cls, filename): + """ + Return a distribution built from the data found in a `filename` string. + Raise an exception if this is not a valid filename + """ + clazz = cls.get_dist_class(filename) + return clazz.from_filename(filename) + + @classmethod + def from_data(cls, data, keep_extra=False): + """ + Return a distribution built from a `data` mapping. + """ + filename = data['filename'] + dist = cls.from_filename(filename) + dist.update(data, keep_extra=keep_extra) + return dist + + @classmethod + def from_dist(cls, data, dist): + """ + Return a distribution built from a `data` mapping and update it with data + from another dist Distribution. Return None if it cannot be created + """ + # We can only create from a dist of the same package + has_same_key_fields = all(data.get(kf) == getattr(dist, kf, None) + for kf in ('type', 'namespace', 'name') + ) + if not has_same_key_fields: + print(f'Missing key fields: Cannot derive a new dist from data: {data} and dist: {dist}') + return + + has_key_field_values = all(data.get(kf) for kf in ('type', 'name', 'version')) + if not has_key_field_values: + print(f'Missing key field values: Cannot derive a new dist from data: {data} and dist: {dist}') + return + + data = dict(data) + # do not overwrite the data with the other dist + # only supplement + data.update({k: v for k, v in dist.get_updatable_data().items() if not data.get(k)}) + return cls.from_data(data) + + @classmethod + def build_remote_download_url(cls, filename, base_url=REMOTE_LINKS_URL): + """ + Return a direct download URL for a file in our remote repo + """ + return f'{base_url}/{filename}' + + def get_best_download_url(self): + """ + Return the best download URL for this distribution where best means that + PyPI is better and our own remote repo URLs are second. + If none is found, return a synthetic remote URL. + """ + name = self.normalized_name + version = self.version + filename = self.filename + + pypi_package = get_pypi_package(name=name, version=version) + if pypi_package: + pypi_url = pypi_package.get_url_for_filename(filename) + if pypi_url: + return pypi_url + + remote_package = get_remote_package(name=name, version=version) + if remote_package: + remote_url = remote_package.get_url_for_filename(filename) + if remote_url: + return remote_url + else: + # the package may not have been published yet, so we craft a URL + # using our remote base URL + return self.build_remote_download_url(self.filename) + + def purl_identifiers(self, skinny=False): + """ + Return a mapping of non-empty identifier name/values for the purl + fields. If skinny is True, only inlucde type, namespace and name. + """ + identifiers = dict( + type=self.type, + namespace=self.namespace, + name=self.name, + ) + + if not skinny: + identifiers.update( + version=self.version, + subpath=self.subpath, + qualifiers=self.qualifiers, + ) + + return {k: v for k, v in sorted(identifiers.items()) if v} + + def identifiers(self, purl_as_fields=True): + """ + Return a mapping of non-empty identifier name/values. + Return each purl fields separately if purl_as_fields is True. + Otherwise return a package_url string for the purl. + """ + if purl_as_fields: + identifiers = self.purl_identifiers() + else: + identifiers = dict(package_url=self.package_url) + + identifiers.update( + download_url=self.download_url, + filename=self.filename, + md5=self.md5, + sha1=self.sha1, + package_url=self.package_url, + ) + + return {k: v for k, v in sorted(identifiers.items()) if v} + + def has_key_metadata(self): + """ + Return True if this distribution has key metadata required for basic attribution. + """ + if self.license_expression == 'public-domain': + # copyright not needed + return True + return self.license_expression and self.copyright and self.path_or_url + + def to_about(self): + """ + Return a mapping of ABOUT data from this distribution fields. + """ + about_data = dict( + about_resource=self.filename, + checksum_md5=self.md5, + checksum_sha1=self.sha1, + copyright=self.copyright, + description=self.description, + download_url=self.download_url, + homepage_url=self.homepage_url, + license_expression=self.license_expression, + name=self.name, + namespace=self.namespace, + notes=self.notes, + notice_file=self.notice_filename if self.notice_text else '', + package_url=self.package_url, + primary_language=self.primary_language, + qualifiers=self.qualifiers, + size=self.size, + subpath=self.subpath, + type=self.type, + version=self.version, + ) + + about_data.update(self.extra_data) + about_data = {k: v for k, v in sorted(about_data.items()) if v} + return about_data + + def to_dict(self): + """ + Return a mapping data from this distribution. + """ + return {k: v for k, v in attr.asdict(self).items() if v} + + def save_about_and_notice_files(self, dest_dir=THIRDPARTY_DIR): + """ + Save a .ABOUT file to `dest_dir`. Include a .NOTICE file if there is a + notice_text. + """ + + def save_if_modified(location, content): + if os.path.exists(location): + with open(location) as fi: + existing_content = fi.read() + if existing_content == content: + return False + + if TRACE: print(f'Saving ABOUT (and NOTICE) files for: {self}') + with open(location, 'w') as fo: + fo.write(content) + return True + + save_if_modified( + location=os.path.join(dest_dir, self.about_filename), + content=saneyaml.dump(self.to_about()), + ) + + notice_text = self.notice_text and self.notice_text.strip() + if notice_text: + save_if_modified( + location=os.path.join(dest_dir, self.notice_filename), + content=notice_text, + ) + + def load_about_data(self, about_filename_or_data=None, dest_dir=THIRDPARTY_DIR): + """ + Update self with ABOUT data loaded from an `about_filename_or_data` + which is either a .ABOUT file in `dest_dir` or an ABOUT data mapping. + `about_filename_or_data` defaults to this distribution default ABOUT + filename if not provided. Load the notice_text if present from dest_dir. + """ + if not about_filename_or_data: + about_filename_or_data = self.about_filename + + if isinstance(about_filename_or_data, str): + # that's an about_filename + about_path = os.path.join(dest_dir, about_filename_or_data) + if os.path.exists(about_path): + with open(about_path) as fi: + about_data = saneyaml.load(fi.read()) + else: + return False + else: + about_data = about_filename_or_data + + md5 = about_data.pop('checksum_md5', None) + if md5: + about_data['md5'] = md5 + sha1 = about_data.pop('checksum_sha1', None) + if sha1: + about_data['sha1'] = sha1 + sha256 = about_data.pop('checksum_sha256', None) + if sha256: + about_data['sha256'] = sha256 + + about_data.pop('about_resource', None) + notice_text = about_data.pop('notice_text', None) + notice_file = about_data.pop('notice_file', None) + if notice_text: + about_data['notice_text'] = notice_text + elif notice_file: + notice_loc = os.path.join(dest_dir, notice_file) + if os.path.exists(notice_loc): + with open(notice_loc) as fi: + about_data['notice_text'] = fi.read() + return self.update(about_data, keep_extra=True) + + def load_remote_about_data(self): + """ + Fetch and update self with "remote" data Distribution ABOUT file and + NOTICE file if any. Return True if the data was updated. + """ + try: + about_text = fetch_content_from_path_or_url_through_cache(self.about_download_url) + except RemoteNotFetchedException: + return False + + if not about_text: + return False + + about_data = saneyaml.load(about_text) + notice_file = about_data.pop('notice_file', None) + if notice_file: + try: + notice_text = fetch_content_from_path_or_url_through_cache(self.notice_download_url) + if notice_text: + about_data['notice_text'] = notice_text + except RemoteNotFetchedException: + print(f'Failed to fetch NOTICE file: {self.notice_download_url}') + return self.load_about_data(about_data) + + def get_checksums(self, dest_dir=THIRDPARTY_DIR): + """ + Return a mapping of computed checksums for this dist filename is + `dest_dir`. + """ + dist_loc = os.path.join(dest_dir, self.filename) + if os.path.exists(dist_loc): + return multi_checksums(dist_loc, checksum_names=('md5', 'sha1', 'sha256')) + else: + return {} + + def set_checksums(self, dest_dir=THIRDPARTY_DIR): + """ + Update self with checksums computed for this dist filename is `dest_dir`. + """ + self.update(self.get_checksums(dest_dir), overwrite=True) + + def validate_checksums(self, dest_dir=THIRDPARTY_DIR): + """ + Return True if all checksums that have a value in this dist match + checksums computed for this dist filename is `dest_dir`. + """ + real_checksums = self.get_checksums(dest_dir) + for csk in ('md5', 'sha1', 'sha256'): + csv = getattr(self, csk) + rcv = real_checksums.get(csk) + if csv and rcv and csv != rcv: + return False + return True + + def get_pip_hash(self): + """ + Return a pip hash option string as used in requirements for this dist. + """ + assert self.sha256, f'Missinh SHA256 for dist {self}' + return f'--hash=sha256:{self.sha256}' + + def get_license_keys(self): + return LICENSING.license_keys(self.license_expression, unique=True, simple=True) + + def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): + """ + Fetch license files is missing in `dest_dir`. + Return True if license files were fetched. + """ + paths_or_urls = get_remote_repo().links + errors = [] + extra_lic_names = [l.get('file') for l in self.extra_data.get('licenses', {})] + extra_lic_names += [self.extra_data.get('license_file')] + extra_lic_names = [ln for ln in extra_lic_names if ln] + lic_names = [ f'{key}.LICENSE' for key in self.get_license_keys()] + for filename in lic_names + extra_lic_names: + floc = os.path.join(dest_dir, filename) + if os.path.exists(floc): + continue + + try: + # try remotely first + lic_url = get_link_for_filename( + filename=filename, paths_or_urls=paths_or_urls) + + fetch_and_save_path_or_url( + filename=filename, + dest_dir=dest_dir, + path_or_url=lic_url, + as_text=True, + ) + if TRACE: print(f'Fetched license from remote: {lic_url}') + + except: + try: + # try licensedb second + lic_url = f'{LICENSEDB_API_URL}/{filename}' + fetch_and_save_path_or_url( + filename=filename, + dest_dir=dest_dir, + path_or_url=lic_url, + as_text=True, + ) + if TRACE: print(f'Fetched license from licensedb: {lic_url}') + + except: + msg = f'No text for license {filename} in expression "{self.license_expression}" from {self}' + print(msg) + errors.append(msg) + + return errors + + def extract_pkginfo(self, dest_dir=THIRDPARTY_DIR): + """ + Return the text of the first PKG-INFO or METADATA file found in the + archive of this Distribution in `dest_dir`. Return None if not found. + """ + fmt = 'zip' if self.filename.endswith('.whl') else None + dist = os.path.join(dest_dir, self.filename) + with tempfile.TemporaryDirectory(prefix='pypi-tmp-extract') as td: + shutil.unpack_archive(filename=dist, extract_dir=td, format=fmt) + # NOTE: we only care about the first one found in the dist + # which may not be 100% right + for pi in fileutils.resource_iter(location=td, with_dirs=False): + if pi.endswith(('PKG-INFO', 'METADATA',)): + with open(pi) as fi: + return fi.read() + + def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): + """ + Update self with data loaded from the PKG-INFO file found in the + archive of this Distribution in `dest_dir`. + """ + pkginfo_text = self.extract_pkginfo(dest_dir=dest_dir) + if not pkginfo_text: + print(f'!!!!PKG-INFO not found in {self.filename}') + return + raw_data = email.message_from_string(pkginfo_text) + + classifiers = raw_data.get_all('Classifier') or [] + + declared_license = [raw_data['License']] + [c for c in classifiers if c.startswith('License')] + other_classifiers = [c for c in classifiers if not c.startswith('License')] + + pkginfo_data = dict( + name=raw_data['Name'], + declared_license=declared_license, + version=raw_data['Version'], + description=raw_data['Summary'], + homepage_url=raw_data['Home-page'], + holder=raw_data['Author'], + holder_contact=raw_data['Author-email'], + keywords=raw_data['Keywords'], + classifiers=other_classifiers, + ) + + return self.update(pkginfo_data, keep_extra=True) + + def update_from_other_dist(self, dist): + """ + Update self using data from another dist + """ + return self.update(dist.get_updatable_data()) + + def get_updatable_data(self, data=None): + data = data or self.to_dict() + return { + k: v for k, v in data.items() + if v and k in self.updatable_fields + } + + def update(self, data, overwrite=False, keep_extra=True): + """ + Update self with a mapping of `data`. Keep unknown data as extra_data if + `keep_extra` is True. If `overwrite` is True, overwrite self with `data` + Return True if any data was updated, False otherwise. Raise an exception + if there are key data conflicts. + """ + package_url = data.get('package_url') + if package_url: + purl_from_data = packageurl.PackageURL.from_string(package_url) + purl_from_self = packageurl.PackageURL.from_string(self.package_url) + if purl_from_data != purl_from_self: + print( + f'Invalid dist update attempt, no same same purl with dist: ' + f'{self} using data {data}.') + return + + data.pop('about_resource', None) + dl = data.pop('download_url', None) + if dl: + data['path_or_url'] = dl + + updated = False + extra = {} + for k, v in data.items(): + if isinstance(v, str): + v = v.strip() + if not v: + continue + + if hasattr(self, k): + value = getattr(self, k, None) + if not value or (overwrite and value != v): + try: + setattr(self, k, v) + except Exception as e: + raise Exception(f'{self}, {k}, {v}') from e + updated = True + + elif keep_extra: + # note that we always overwrite extra + extra[k] = v + updated = True + + self.extra_data.update(extra) + + return updated + + +class InvalidDistributionFilename(Exception): + pass + + +@attr.attributes +class Sdist(Distribution): + + extension = attr.ib( + repr=False, + type=str, + default='', + metadata=dict(help='File extension, including leading dot.'), + ) + + @classmethod + def from_filename(cls, filename): + """ + Return a Sdist object built from a filename. + Raise an exception if this is not a valid sdist filename + """ + name_ver = None + extension = None + + for ext in EXTENSIONS_SDIST: + if filename.endswith(ext): + name_ver, extension, _ = filename.rpartition(ext) + break + + if not extension or not name_ver: + raise InvalidDistributionFilename(filename) + + name, _, version = name_ver.rpartition('-') + + if not name or not version: + raise InvalidDistributionFilename(filename) + + return cls( + type='pypi', + name=name, + version=version, + extension=extension, + filename=filename, + ) + + def to_filename(self): + """ + Return an sdist filename reconstructed from its fields (that may not be + the same as the original filename.) + """ + return f'{self.name}-{self.version}.{self.extension}' + + +@attr.attributes +class Wheel(Distribution): + + """ + Represents a wheel file. + + Copied and heavily modified from pip-20.3.1 copied from pip-20.3.1 + pip/_internal/models/wheel.py + + name: pip compatibility tags + version: 20.3.1 + download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py + copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) + license_expression: mit + notes: copied from pip-20.3.1 pip/_internal/models/wheel.py + + Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + """ + + get_wheel_from_filename = re.compile( + r"""^(?P(?P.+?)-(?P.*?)) + ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) + \.whl)$""", + re.VERBOSE + ).match + + build = attr.ib( + type=str, + default='', + metadata=dict(help='Python wheel build.'), + ) + + python_versions = attr.ib( + type=list, + default=attr.Factory(list), + metadata=dict(help='List of wheel Python version tags.'), + ) + + abis = attr.ib( + type=list, + default=attr.Factory(list), + metadata=dict(help='List of wheel ABI tags.'), + ) + + platforms = attr.ib( + type=list, + default=attr.Factory(list), + metadata=dict(help='List of wheel platform tags.'), + ) + + tags = attr.ib( + repr=False, + type=set, + default=attr.Factory(set), + metadata=dict(help='Set of all tags for this wheel.'), + ) + + @classmethod + def from_filename(cls, filename): + """ + Return a wheel object built from a filename. + Raise an exception if this is not a valid wheel filename + """ + wheel_info = cls.get_wheel_from_filename(filename) + if not wheel_info: + raise InvalidDistributionFilename(filename) + + name = wheel_info.group('name').replace('_', '-') + # we'll assume "_" means "-" due to wheel naming scheme + # (https://github.com/pypa/pip/issues/1150) + version = wheel_info.group('ver').replace('_', '-') + build = wheel_info.group('build') + python_versions = wheel_info.group('pyvers').split('.') + abis = wheel_info.group('abis').split('.') + platforms = wheel_info.group('plats').split('.') + + # All the tag combinations from this file + tags = { + packaging_tags.Tag(x, y, z) for x in python_versions + for y in abis for z in platforms + } + + return cls( + filename=filename, + type='pypi', + name=name, + version=version, + build=build, + python_versions=python_versions, + abis=abis, + platforms=platforms, + tags=tags, + ) + + def is_supported_by_tags(self, tags): + """ + Return True is this wheel is compatible with one of a list of PEP 425 tags. + """ + return not self.tags.isdisjoint(tags) + + def is_supported_by_environment(self, environment): + """ + Return True if this wheel is compatible with the Environment + `environment`. + """ + return not self.is_supported_by_tags(environment.tags) + + def to_filename(self): + """ + Return a wheel filename reconstructed from its fields (that may not be + the same as the original filename.) + """ + build = f'-{self.build}' if self.build else '' + pyvers = '.'.join(self.python_versions) + abis = '.'.join(self.abis) + plats = '.'.join(self.platforms) + return f'{self.name}-{self.version}{build}-{pyvers}-{abis}-{plats}.whl' + + def is_pure(self): + """ + Return True if wheel `filename` is for a "pure" wheel e.g. a wheel that + runs on all Pythons 3 and all OSes. + + For example:: + + >>> Wheel.from_filename('aboutcode_toolkit-5.1.0-py2.py3-none-any.whl').is_pure() + True + >>> Wheel.from_filename('beautifulsoup4-4.7.1-py3-none-any.whl').is_pure() + True + >>> Wheel.from_filename('beautifulsoup4-4.7.1-py2-none-any.whl').is_pure() + False + >>> Wheel.from_filename('bitarray-0.8.1-cp36-cp36m-win_amd64.whl').is_pure() + False + >>> Wheel.from_filename('extractcode_7z-16.5-py2.py3-none-macosx_10_13_intel.whl').is_pure() + False + >>> Wheel.from_filename('future-0.16.0-cp36-none-any.whl').is_pure() + False + >>> Wheel.from_filename('foo-4.7.1-py3-none-macosx_10_13_intel.whl').is_pure() + False + >>> Wheel.from_filename('future-0.16.0-py3-cp36m-any.whl').is_pure() + False + """ + return ( + 'py3' in self.python_versions + and 'none' in self.abis + and 'any' in self.platforms + ) + + +def is_pure_wheel(filename): + try: + return Wheel.from_filename(filename).is_pure() + except: + return False + + +@attr.attributes +class PypiPackage(NameVer): + """ + A Python package with its "distributions", e.g. wheels and source + distribution , ABOUT files and licenses or notices. + """ + sdist = attr.ib( + repr=False, + type=str, + default='', + metadata=dict(help='Sdist source distribution for this package.'), + ) + + wheels = attr.ib( + repr=False, + type=list, + default=attr.Factory(list), + metadata=dict(help='List of Wheel for this package'), + ) + + @property + def specifier(self): + """ + A requirement specifier for this package + """ + if self.version: + return f'{self.name}=={self.version}' + else: + return self.name + + @property + def specifier_with_hashes(self): + """ + Return a requirement specifier for this package with --hash options for + all its distributions + """ + items = [self.specifier] + items += [d.get_pip_hashes() for d in self.get_distributions()] + return ' \\\n '.join(items) + + def get_supported_wheels(self, environment): + """ + Yield all the Wheel of this package supported and compatible with the + Environment `environment`. + """ + envt_tags = environment.tags() + for wheel in self.wheels: + if wheel.is_supported_by_tags(envt_tags): + yield wheel + + @classmethod + def package_from_dists(cls, dists): + """ + Return a new PypiPackage built from an iterable of Wheels and Sdist + objects all for the same package name and version. + + For example: + >>> w1 = Wheel(name='bitarray', version='0.8.1', build='', + ... python_versions=['cp36'], abis=['cp36m'], + ... platforms=['linux_x86_64']) + >>> w2 = Wheel(name='bitarray', version='0.8.1', build='', + ... python_versions=['cp36'], abis=['cp36m'], + ... platforms=['macosx_10_9_x86_64', 'macosx_10_10_x86_64']) + >>> sd = Sdist(name='bitarray', version='0.8.1') + >>> package = PypiPackage.package_from_dists(dists=[w1, w2, sd]) + >>> assert package.name == 'bitarray' + >>> assert package.version == '0.8.1' + >>> assert package.sdist == sd + >>> assert package.wheels == [w1, w2] + """ + dists = list(dists) + if not dists: + return + + reference_dist = dists[0] + normalized_name = reference_dist.normalized_name + version = reference_dist.version + + package = PypiPackage(name=normalized_name, version=version) + + for dist in dists: + if dist.normalized_name != normalized_name or dist.version != version: + if TRACE: + print( + f' Skipping inconsistent dist name and version: {dist} ' + f'Expected instead package name: {normalized_name} and version: "{version}"' + ) + continue + + if isinstance(dist, Sdist): + package.sdist = dist + + elif isinstance(dist, Wheel): + package.wheels.append(dist) + + else: + raise Exception(f'Unknown distribution type: {dist}') + + return package + + @classmethod + def packages_from_one_path_or_url(cls, path_or_url): + """ + Yield PypiPackages built from files found in at directory path or the + URL to an HTML page (that will be fetched). + """ + extracted_paths_or_urls = get_paths_or_urls(path_or_url) + return cls.packages_from_many_paths_or_urls(extracted_paths_or_urls) + + @classmethod + def packages_from_many_paths_or_urls(cls, paths_or_urls): + """ + Yield PypiPackages built from a list of paths or URLs. + """ + dists = cls.get_dists(paths_or_urls) + dists = NameVer.sorted(dists) + + for _projver, dists_of_package in itertools.groupby( + dists, key=NameVer.sortable_name_version, + ): + yield PypiPackage.package_from_dists(dists_of_package) + + @classmethod + def get_versions_from_path_or_url(cls, name, path_or_url): + """ + Return a subset list from a list of PypiPackages version at `path_or_url` + that match PypiPackage `name`. + """ + packages = cls.packages_from_one_path_or_url(path_or_url) + return cls.get_versions(name, packages) + + @classmethod + def get_versions(cls, name, packages): + """ + Return a subset list of package versions from a list of `packages` that + match PypiPackage `name`. + The list is sorted by version from oldest to most recent. + """ + norm_name = NameVer.normalize_name(name) + versions = [p for p in packages if p.normalized_name == norm_name] + return cls.sorted(versions) + + @classmethod + def get_latest_version(cls, name, packages): + """ + Return the latest version of PypiPackage `name` from a list of `packages`. + """ + versions = cls.get_versions(name, packages) + if not versions: + return + return versions[-1] + + @classmethod + def get_outdated_versions(cls, name, packages): + """ + Return all versions except the latest version of PypiPackage `name` from a + list of `packages`. + """ + versions = cls.get_versions(name, packages) + return versions[:-1] + + @classmethod + def get_name_version(cls, name, version, packages): + """ + Return the PypiPackage with `name` and `version` from a list of `packages` + or None if it is not found. + If `version` is None, return the latest version found. + """ + if version is None: + return cls.get_latest_version(name, packages) + + nvs = [p for p in cls.get_versions(name, packages) if p.version == version] + + if not nvs: + return + + if len(nvs) == 1: + return nvs[0] + + raise Exception(f'More than one PypiPackage with {name}=={version}') + + def fetch_wheel( + self, + environment=None, + fetched_filenames=None, + dest_dir=THIRDPARTY_DIR, + ): + """ + Download a binary wheel of this package matching the ``environment`` + Enviromnent constraints into ``dest_dir`` directory. + + Return the wheel filename if it was fetched, None otherwise. + + If the provided ``environment`` is None then the current Python + interpreter environment is used implicitly. Do not refetch wheel if + their name is in a provided ``fetched_filenames`` set. + """ + fetched_wheel_filename = None + if fetched_filenames is not None: + fetched_filenames = fetched_filenames + else: + fetched_filenames = set() + + for wheel in self.get_supported_wheels(environment): + + if wheel.filename not in fetched_filenames: + fetch_and_save_path_or_url( + filename=wheel.filename, + path_or_url=wheel.path_or_url, + dest_dir=dest_dir, + as_text=False, + ) + fetched_filenames.add(wheel.filename) + fetched_wheel_filename = wheel.filename + + # TODO: what if there is more than one? + break + + return fetched_wheel_filename + + def fetch_sdist(self, dest_dir=THIRDPARTY_DIR): + """ + Download the source distribution into `dest_dir` directory. Return the + fetched filename if it was fetched, False otherwise. + """ + if self.sdist: + assert self.sdist.filename + if TRACE: print('Fetching source for package:', self.name, self.version) + fetch_and_save_path_or_url( + filename=self.sdist.filename, + dest_dir=dest_dir, + path_or_url=self.sdist.path_or_url, + as_text=False, + ) + if TRACE: print(' --> file:', self.sdist.filename) + return self.sdist.filename + else: + print(f'Missing sdist for: {self.name}=={self.version}') + return False + + def delete_files(self, dest_dir=THIRDPARTY_DIR): + """ + Delete all PypiPackage files from `dest_dir` including wheels, sdist and + their ABOUT files. Note that we do not delete licenses since they can be + shared by several packages: therefore this would be done elsewhere in a + function that is aware of all used licenses. + """ + for to_delete in self.wheels + [self.sdist]: + if not to_delete: + continue + tdfn = to_delete.filename + for deletable in [tdfn, f'{tdfn}.ABOUT', f'{tdfn}.NOTICE']: + target = os.path.join(dest_dir, deletable) + if os.path.exists(target): + print(f'Deleting outdated {target}') + fileutils.delete(target) + + @classmethod + def get_dists(cls, paths_or_urls): + """ + Return a list of Distribution given a list of + `paths_or_urls` to wheels or source distributions. + + Each Distribution receives two extra attributes: + - the path_or_url it was created from + - its filename + + For example: + >>> paths_or_urls =''' + ... /home/foo/bitarray-0.8.1-cp36-cp36m-linux_x86_64.whl + ... bitarray-0.8.1-cp36-cp36m-macosx_10_9_x86_64.macosx_10_10_x86_64.whl + ... bitarray-0.8.1-cp36-cp36m-win_amd64.whl + ... httsp://example.com/bar/bitarray-0.8.1.tar.gz + ... bitarray-0.8.1.tar.gz.ABOUT bit.LICENSE'''.split() + >>> result = list(PypiPackage.get_dists(paths_or_urls)) + >>> for r in results: + ... r.filename = '' + ... r.path_or_url = '' + >>> expected = [ + ... Wheel(name='bitarray', version='0.8.1', build='', + ... python_versions=['cp36'], abis=['cp36m'], + ... platforms=['linux_x86_64']), + ... Wheel(name='bitarray', version='0.8.1', build='', + ... python_versions=['cp36'], abis=['cp36m'], + ... platforms=['macosx_10_9_x86_64', 'macosx_10_10_x86_64']), + ... Wheel(name='bitarray', version='0.8.1', build='', + ... python_versions=['cp36'], abis=['cp36m'], + ... platforms=['win_amd64']), + ... Sdist(name='bitarray', version='0.8.1') + ... ] + >>> assert expected == result + """ + installable = [f for f in paths_or_urls if f.endswith(EXTENSIONS_INSTALLABLE)] + for path_or_url in installable: + try: + yield Distribution.from_path_or_url(path_or_url) + except InvalidDistributionFilename: + if TRACE: + print(f'Skipping invalid distribution from: {path_or_url}') + continue + + def get_distributions(self): + """ + Yield all distributions available for this PypiPackage + """ + if self.sdist: + yield self.sdist + for wheel in self.wheels: + yield wheel + + def get_url_for_filename(self, filename): + """ + Return the URL for this filename or None. + """ + for dist in self.get_distributions(): + if dist.filename == filename: + return dist.path_or_url + + +@attr.attributes +class Environment: + """ + An Environment describes a target installation environment with its + supported Python version, ABI, platform, implementation and related + attributes. We can use these to pass as `pip download` options and force + fetching only the subset of packages that match these Environment + constraints as opposed to the current running Python interpreter + constraints. + """ + + python_version = attr.ib( + type=str, + default='', + metadata=dict(help='Python version supported by this environment.'), + ) + + operating_system = attr.ib( + type=str, + default='', + metadata=dict(help='operating system supported by this environment.'), + ) + + implementation = attr.ib( + type=str, + default='cp', + metadata=dict(help='Python implementation supported by this environment.'), + ) + + abis = attr.ib( + type=list, + default=attr.Factory(list), + metadata=dict(help='List of ABI tags supported by this environment.'), + ) + + platforms = attr.ib( + type=list, + default=attr.Factory(list), + metadata=dict(help='List of platform tags supported by this environment.'), + ) + + @classmethod + def from_pyver_and_os(cls, python_version, operating_system): + if '.' in python_version: + python_version = ''.join(python_version.split('.')) + + return cls( + python_version=python_version, + implementation='cp', + abis=ABIS_BY_PYTHON_VERSION[python_version], + platforms=PLATFORMS_BY_OS[operating_system], + operating_system=operating_system, + ) + + def get_pip_cli_options(self): + """ + Return a list of pip command line options for this environment. + """ + options = [ + '--python-version', self.python_version, + '--implementation', self.implementation, + '--abi', self.abi, + ] + for platform in self.platforms: + options.extend(['--platform', platform]) + return options + + def tags(self): + """ + Return a set of all the PEP425 tags supported by this environment. + """ + return set(utils_pip_compatibility_tags.get_supported( + version=self.python_version or None, + impl=self.implementation or None, + platforms=self.platforms or None, + abis=self.abis or None, + )) + +################################################################################ +# +# PyPI repo and link index for package wheels and sources +# +################################################################################ + + +@attr.attributes +class Repository: + """ + A PyPI or links Repository of Python packages: wheels, sdist, ABOUT, etc. + """ + + packages_by_normalized_name = attr.ib( + type=dict, + default=attr.Factory(lambda: defaultdict(list)), + metadata=dict(help= + 'Mapping of {package name: [package objects]} available in this repo'), + ) + + packages_by_normalized_name_version = attr.ib( + type=dict, + default=attr.Factory(dict), + metadata=dict(help= + 'Mapping of {(name, version): package object} available in this repo'), + ) + + def get_links(self, *args, **kwargs): + raise NotImplementedError() + + def get_versions(self, name): + """ + Return a list of all available PypiPackage version for this package name. + The list may be empty. + """ + raise NotImplementedError() + + def get_package(self, name, version): + """ + Return the PypiPackage with name and version or None. + """ + raise NotImplementedError() + + def get_latest_version(self, name): + """ + Return the latest PypiPackage version for this package name or None. + """ + raise NotImplementedError() + + +@attr.attributes +class LinksRepository(Repository): + """ + Represents a simple links repository which is either a local directory with + Python wheels and sdist or a remote URL to an HTML with links to these. + (e.g. suitable for use with pip --find-links). + """ + path_or_url = attr.ib( + type=str, + default='', + metadata=dict(help='Package directory path or URL'), + ) + + links = attr.ib( + type=list, + default=attr.Factory(list), + metadata=dict(help='List of links available in this repo'), + ) + + def __attrs_post_init__(self): + if not self.links: + self.links = get_paths_or_urls(links_url=self.path_or_url) + if not self.packages_by_normalized_name: + for p in PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=self.links): + normalized_name = p.normalized_name + self.packages_by_normalized_name[normalized_name].append(p) + self.packages_by_normalized_name_version[(normalized_name, p.version)] = p + + def get_links(self, *args, **kwargs): + return self.links or [] + + def get_versions(self, name): + name = name and NameVer.normalize_name(name) + return self.packages_by_normalized_name.get(name, []) + + def get_latest_version(self, name): + return PypiPackage.get_latest_version(name, self.get_versions(name)) + + def get_package(self, name, version): + return PypiPackage.get_name_version(name, version, self.get_versions(name)) + + +@attr.attributes +class PypiRepository(Repository): + """ + Represents the public PyPI simple index. + It is populated lazily based on requested packages names + """ + simple_url = attr.ib( + type=str, + default=PYPI_SIMPLE_URL, + metadata=dict(help='Base PyPI simple URL for this index.'), + ) + + links_by_normalized_name = attr.ib( + type=dict, + default=attr.Factory(lambda: defaultdict(list)), + metadata=dict(help='Mapping of {package name: [links]} available in this repo'), + ) + + def _fetch_links(self, name): + name = name and NameVer.normalize_name(name) + return find_pypi_links(name=name, simple_url=self.simple_url) + + def _populate_links_and_packages(self, name): + name = name and NameVer.normalize_name(name) + if name in self.links_by_normalized_name: + return + + links = self._fetch_links(name) + self.links_by_normalized_name[name] = links + + packages = list(PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=links)) + self.packages_by_normalized_name[name] = packages + + for p in packages: + name = name and NameVer.normalize_name(p.name) + self.packages_by_normalized_name_version[(name, p.version)] = p + + def get_links(self, name, *args, **kwargs): + name = name and NameVer.normalize_name(name) + self._populate_links_and_packages(name) + return self.links_by_normalized_name.get(name, []) + + def get_versions(self, name): + name = name and NameVer.normalize_name(name) + self._populate_links_and_packages(name) + return self.packages_by_normalized_name.get(name, []) + + def get_latest_version(self, name): + return PypiPackage.get_latest_version(name, self.get_versions(name)) + + def get_package(self, name, version): + return PypiPackage.get_name_version(name, version, self.get_versions(name)) + +################################################################################ +# Globals for remote repos to be lazily created and cached on first use for the +# life of the session together with some convenience functions. +################################################################################ + + +def get_local_packages(directory=THIRDPARTY_DIR): + """ + Return the list of all PypiPackage objects built from a local directory. Return + an empty list if the package cannot be found. + """ + return list(PypiPackage.packages_from_one_path_or_url(path_or_url=directory)) + + +def get_local_repo(directory=THIRDPARTY_DIR): + return LinksRepository(path_or_url=directory) + + +_REMOTE_REPO = None + + +def get_remote_repo(remote_links_url=REMOTE_LINKS_URL): + global _REMOTE_REPO + if not _REMOTE_REPO: + _REMOTE_REPO = LinksRepository(path_or_url=remote_links_url) + return _REMOTE_REPO + + +def get_remote_package(name, version, remote_links_url=REMOTE_LINKS_URL): + """ + Return a PypiPackage or None. + """ + try: + return get_remote_repo(remote_links_url).get_package(name, version) + except RemoteNotFetchedException as e: + print(f'Failed to fetch remote package info: {e}') + + +_PYPI_REPO = None + + +def get_pypi_repo(pypi_simple_url=PYPI_SIMPLE_URL): + global _PYPI_REPO + if not _PYPI_REPO: + _PYPI_REPO = PypiRepository(simple_url=pypi_simple_url) + return _PYPI_REPO + + +def get_pypi_package(name, version, pypi_simple_url=PYPI_SIMPLE_URL): + """ + Return a PypiPackage or None. + """ + try: + return get_pypi_repo(pypi_simple_url).get_package(name, version) + except RemoteNotFetchedException as e: + print(f'Failed to fetch remote package info: {e}') + +################################################################################ +# +# Basic file and URL-based operations using a persistent file-based Cache +# +################################################################################ + + +@attr.attributes +class Cache: + """ + A simple file-based cache based only on a filename presence. + This is used to avoid impolite fetching from remote locations. + """ + + directory = attr.ib(type=str, default=CACHE_THIRDPARTY_DIR) + + def __attrs_post_init__(self): + os.makedirs(self.directory, exist_ok=True) + + def clear(self): + shutil.rmtree(self.directory) + + def get(self, path_or_url, as_text=True): + """ + Get a file from a `path_or_url` through the cache. + `path_or_url` can be a path or a URL to a file. + """ + filename = os.path.basename(path_or_url.strip('/')) + cached = os.path.join(self.directory, filename) + + if not os.path.exists(cached): + content = get_file_content(path_or_url=path_or_url, as_text=as_text) + wmode = 'w' if as_text else 'wb' + with open(cached, wmode) as fo: + fo.write(content) + return content + else: + return get_local_file_content(path=cached, as_text=as_text) + + def put(self, filename, content): + """ + Put in the cache the `content` of `filename`. + """ + cached = os.path.join(self.directory, filename) + wmode = 'wb' if isinstance(content, bytes) else 'w' + with open(cached, wmode) as fo: + fo.write(content) + + +def get_file_content(path_or_url, as_text=True): + """ + Fetch and return the content at `path_or_url` from either a local path or a + remote URL. Return the content as bytes is `as_text` is False. + """ + if (path_or_url.startswith('file://') + or (path_or_url.startswith('/') and os.path.exists(path_or_url)) + ): + return get_local_file_content(path=path_or_url, as_text=as_text) + + elif path_or_url.startswith('https://'): + if TRACE: print(f'Fetching: {path_or_url}') + _headers, content = get_remote_file_content(url=path_or_url, as_text=as_text) + return content + + else: + raise Exception(f'Unsupported URL scheme: {path_or_url}') + + +def get_local_file_content(path, as_text=True): + """ + Return the content at `url` as text. Return the content as bytes is + `as_text` is False. + """ + if path.startswith('file://'): + path = path[7:] + + mode = 'r' if as_text else 'rb' + with open(path, mode) as fo: + return fo.read() + + +class RemoteNotFetchedException(Exception): + pass + + +def get_remote_file_content(url, as_text=True, headers_only=False, headers=None, _delay=0,): + """ + Fetch and return a tuple of (headers, content) at `url`. Return content as a + text string if `as_text` is True. Otherwise return the content as bytes. + + If `header_only` is True, return only (headers, None). Headers is a mapping + of HTTP headers. + Retries multiple times to fetch if there is a HTTP 429 throttling response + and this with an increasing delay. + """ + time.sleep(_delay) + headers = headers or {} + # using a GET with stream=True ensure we get the the final header from + # several redirects and that we can ignore content there. A HEAD request may + # not get us this last header + with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: + status = response.status_code + if status != requests.codes.ok: # NOQA + if status == 429 and _delay < 20: + # too many requests: start some exponential delay + increased_delay = (_delay * 2) or 1 + + return get_remote_file_content( + url, + as_text=as_text, + headers_only=headers_only, + _delay=increased_delay, + ) + + else: + raise RemoteNotFetchedException(f'Failed HTTP request from {url} with {status}') + + if headers_only: + return response.headers, None + + return response.headers, response.text if as_text else response.content + + +def get_url_content_if_modified(url, md5, _delay=0,): + """ + Return fetched content bytes at `url` or None if the md5 has not changed. + Retries multiple times to fetch if there is a HTTP 429 throttling response + and this with an increasing delay. + """ + time.sleep(_delay) + headers = None + if md5: + etag = f'"{md5}"' + headers = {'If-None-Match': f'{etag}'} + + # using a GET with stream=True ensure we get the the final header from + # several redirects and that we can ignore content there. A HEAD request may + # not get us this last header + with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: + status = response.status_code + if status == requests.codes.too_many_requests and _delay < 20: # NOQA + # too many requests: start waiting with some exponential delay + _delay = (_delay * 2) or 1 + return get_url_content_if_modified(url=url, md5=md5, _delay=_delay) + + elif status == requests.codes.not_modified: # NOQA + # all is well, the md5 is the same + return None + + elif status != requests.codes.ok: # NOQA + raise RemoteNotFetchedException(f'Failed HTTP request from {url} with {status}') + + return response.content + + +def get_remote_headers(url): + """ + Fetch and return a mapping of HTTP headers of `url`. + """ + headers, _content = get_remote_file_content(url, headers_only=True) + return headers + + +def fetch_and_save_filename_from_paths_or_urls( + filename, + paths_or_urls, + dest_dir=THIRDPARTY_DIR, + as_text=True, +): + """ + Return the content from fetching the `filename` file name found in the + `paths_or_urls` list of URLs or paths and save to `dest_dir`. Raise an + Exception on errors. Treats the content as text if `as_text` is True + otherwise as binary. + """ + path_or_url = get_link_for_filename( + filename=filename, + paths_or_urls=paths_or_urls, + ) + + return fetch_and_save_path_or_url( + filename=filename, + dest_dir=dest_dir, + path_or_url=path_or_url, + as_text=as_text, + ) + + +def fetch_content_from_path_or_url_through_cache(path_or_url, as_text=True, cache=Cache()): + """ + Return the content from fetching at path or URL. Raise an Exception on + errors. Treats the content as text if as_text is True otherwise as treat as + binary. Use the provided file cache. This is the main entry for using the + cache. + + Note: the `cache` argument is a global, though it does not really matter + since it does not hold any state which is only kept on disk. + """ + if cache: + return cache.get(path_or_url=path_or_url, as_text=as_text) + else: + return get_file_content(path_or_url=path_or_url, as_text=as_text) + + +def fetch_and_save_path_or_url(filename, dest_dir, path_or_url, as_text=True, through_cache=True): + """ + Return the content from fetching the `filename` file name at URL or path + and save to `dest_dir`. Raise an Exception on errors. Treats the content as + text if as_text is True otherwise as treat as binary. + """ + if through_cache: + content = fetch_content_from_path_or_url_through_cache(path_or_url, as_text) + else: + content = fetch_content_from_path_or_url_through_cache(path_or_url, as_text, cache=None) + + output = os.path.join(dest_dir, filename) + wmode = 'w' if as_text else 'wb' + with open(output, wmode) as fo: + fo.write(content) + return content + +################################################################################ +# +# Sync and fix local thirdparty directory for various issues and gaps +# +################################################################################ + + +def fetch_missing_sources(dest_dir=THIRDPARTY_DIR): + """ + Given a thirdparty dir, fetch missing source distributions from our remote + repo or PyPI. Return a list of (name, version) tuples for source + distribution that were not found + """ + not_found = [] + local_packages = get_local_packages(directory=dest_dir) + remote_repo = get_remote_repo() + pypi_repo = get_pypi_repo() + + for package in local_packages: + if not package.sdist: + print(f'Finding sources for: {package.name}=={package.version}: ', end='') + try: + pypi_package = pypi_repo.get_package( + name=package.name, version=package.version) + + if pypi_package and pypi_package.sdist: + print(f'Fetching sources from Pypi') + pypi_package.fetch_sdist(dest_dir=dest_dir) + continue + else: + remote_package = remote_repo.get_package( + name=package.name, version=package.version) + + if remote_package and remote_package.sdist: + print(f'Fetching sources from Remote') + remote_package.fetch_sdist(dest_dir=dest_dir) + continue + + except RemoteNotFetchedException as e: + print(f'Failed to fetch remote package info: {e}') + + print(f'No sources found') + not_found.append((package.name, package.version,)) + + return not_found + + +def fetch_missing_wheels( + python_versions=PYTHON_VERSIONS, + operating_systems=PLATFORMS_BY_OS, + dest_dir=THIRDPARTY_DIR, +): + """ + Given a thirdparty dir fetch missing wheels for all known combos of Python + versions and OS. Return a list of tuple (Package, Environment) for wheels + that were not found locally or remotely. + """ + local_packages = get_local_packages(directory=dest_dir) + evts = itertools.product(python_versions, operating_systems) + environments = [Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] + packages_and_envts = itertools.product(local_packages, environments) + + not_fetched = [] + fetched_filenames = set() + for package, envt in packages_and_envts: + + filename = package.fetch_wheel( + environment=envt, + fetched_filenames=fetched_filenames, + dest_dir=dest_dir, + ) + + if filename: + fetched_filenames.add(filename) + else: + not_fetched.append((package, envt,)) + + return not_fetched + + +def build_missing_wheels( + packages_and_envts, + build_remotely=False, + with_deps=False, + dest_dir=THIRDPARTY_DIR, +): + """ + Build all wheels in a list of tuple (Package, Environment) and save in + `dest_dir`. Return a list of tuple (Package, Environment), and a list of + built wheel filenames. + """ + + not_built = [] + built_filenames = [] + + packages_and_envts = itertools.groupby( + sorted(packages_and_envts), key=operator.itemgetter(0)) + + for package, pkg_envts in packages_and_envts: + + envts = [envt for _pkg, envt in pkg_envts] + python_versions = sorted(set(e.python_version for e in envts)) + operating_systems = sorted(set(e.operating_system for e in envts)) + built = None + try: + built = build_wheels( + requirements_specifier=package.specifier, + with_deps=with_deps, + build_remotely=build_remotely, + python_versions=python_versions, + operating_systems=operating_systems, + verbose=False, + dest_dir=dest_dir, + ) + print('.') + except Exception as e: + import traceback + print('#############################################################') + print('############# WHEEL BUILD FAILED ######################') + traceback.print_exc() + print() + print('#############################################################') + + if not built: + for envt in pkg_envts: + not_built.append((package, envt)) + else: + for bfn in built: + print(f' --> Built wheel: {bfn}') + built_filenames.append(bfn) + + return not_built, built_filenames + +################################################################################ +# +# Functions to handle remote or local repo used to "find-links" +# +################################################################################ + + +def get_paths_or_urls(links_url): + if links_url.startswith('https:'): + paths_or_urls = find_links_from_release_url(links_url) + else: + paths_or_urls = find_links_from_dir(links_url) + return paths_or_urls + + +def find_links_from_dir(directory=THIRDPARTY_DIR): + """ + Return a list of path to files in `directory` for any file that ends with + any of the extension in the list of `extensions` strings. + """ + base = os.path.abspath(directory) + files = [os.path.join(base, f) for f in os.listdir(base) if f.endswith(EXTENSIONS)] + return files + + +get_links = re.compile('href="([^"]+)"').findall + + +def find_links_from_release_url(links_url=REMOTE_LINKS_URL): + """ + Return a list of download link URLs found in the HTML page at `links_url` + URL that starts with the `prefix` string and ends with any of the extension + in the list of `extensions` strings. Use the `base_url` to prefix the links. + """ + if TRACE: print(f'Finding links for {links_url}') + + plinks_url = urllib.parse.urlparse(links_url) + + base_url = urllib.parse.SplitResult( + plinks_url.scheme, plinks_url.netloc, '', '', '').geturl() + + if TRACE: print(f'Base URL {base_url}') + + _headers, text = get_remote_file_content(links_url) + links = [] + for link in get_links(text): + if not link.endswith(EXTENSIONS): + continue + + plink = urllib.parse.urlsplit(link) + + if plink.scheme: + # full URL kept as-is + url = link + + if plink.path.startswith('/'): + # absolute link + url = f'{base_url}{link}' + + else: + # relative link + url = f'{links_url}/{link}' + + if TRACE: print(f'Adding URL: {url}') + + links.append(url) + + if TRACE: print(f'Found {len(links)} links at {links_url}') + return links + + +def find_pypi_links(name, simple_url=PYPI_SIMPLE_URL): + """ + Return a list of download link URLs found in a PyPI simple index for package name. + with the list of `extensions` strings. Use the `simple_url` PyPI url. + """ + if TRACE: print(f'Finding links for {simple_url}') + + name = name and NameVer.normalize_name(name) + simple_url = simple_url.strip('/') + simple_url = f'{simple_url}/{name}' + + _headers, text = get_remote_file_content(simple_url) + links = get_links(text) + # TODO: keep sha256 + links = [l.partition('#sha256=') for l in links] + links = [url for url, _, _sha256 in links] + links = [l for l in links if l.endswith(EXTENSIONS)] + return links + + +def get_link_for_filename(filename, paths_or_urls): + """ + Return a link for `filename` found in the `links` list of URLs or paths. Raise an + exception if no link is found or if there are more than one link for that + file name. + """ + path_or_url = [l for l in paths_or_urls if l.endswith(f'/{filename}')] + if not path_or_url: + raise Exception(f'Missing link to file: {filename}') + if not len(path_or_url) == 1: + raise Exception(f'Multiple links to file: {filename}: \n' + '\n'.join(path_or_url)) + return path_or_url[0] + +################################################################################ +# +# Requirements processing +# +################################################################################ + + +class MissingRequirementException(Exception): + pass + + +def get_required_packages(required_name_versions): + """ + Return a tuple of (remote packages, PyPI packages) where each is a mapping + of {(name, version): PypiPackage} for packages listed in the + `required_name_versions` list of (name, version) tuples. Raise a + MissingRequirementException with a list of missing (name, version) if a + requirement cannot be satisfied remotely or in PyPI. + """ + remote_repo = get_remote_repo() + + remote_packages = {(name, version): remote_repo.get_package(name, version) + for name, version in required_name_versions} + + pypi_repo = get_pypi_repo() + pypi_packages = {(name, version): pypi_repo.get_package(name, version) + for name, version in required_name_versions} + + # remove any empty package (e.g. that do not exist in some place) + remote_packages = {nv: p for nv, p in remote_packages.items() if p} + pypi_packages = {nv: p for nv, p in pypi_packages.items() if p} + + # check that we are not missing any + repos_name_versions = set(remote_packages.keys()) | set(pypi_packages.keys()) + missing_name_versions = required_name_versions.difference(repos_name_versions) + if missing_name_versions: + raise MissingRequirementException(sorted(missing_name_versions)) + + return remote_packages, pypi_packages + + +def get_required_remote_packages( + requirements_file='requirements.txt', + force_pinned=True, + remote_links_url=REMOTE_LINKS_URL, +): + """ + Yield tuple of (name, version, PypiPackage) for packages listed in the + `requirements_file` requirements file and found in the PyPI-like link repo + ``remote_links_url`` if this is a URL. Treat this ``remote_links_url`` as a + local directory path to a wheels directory if this is not a a URL. + """ + required_name_versions = load_requirements( + requirements_file=requirements_file, + force_pinned=force_pinned, + ) + + if remote_links_url.startswith('https://'): + repo = get_remote_repo(remote_links_url=remote_links_url) + else: + # a local path + assert os.path.exists(remote_links_url) + repo = get_local_repo(directory=remote_links_url) + + for name, version in required_name_versions: + if version: + yield name, version, repo.get_package(name, version) + else: + yield name, version, repo.get_latest_version(name) + + +def update_requirements(name, version=None, requirements_file='requirements.txt'): + """ + Upgrade or add `package_name` with `new_version` to the `requirements_file` + requirements file. Write back requirements sorted with name and version + canonicalized. Note: this cannot deal with hashed or unpinned requirements. + Do nothing if the version already exists as pinned. + """ + normalized_name = NameVer.normalize_name(name) + + is_updated = False + updated_name_versions = [] + for existing_name, existing_version in load_requirements(requirements_file, force_pinned=False): + + existing_normalized_name = NameVer.normalize_name(existing_name) + + if normalized_name == existing_normalized_name: + if version != existing_version: + is_updated = True + updated_name_versions.append((existing_normalized_name, existing_version,)) + + if is_updated: + updated_name_versions = sorted(updated_name_versions) + nvs = '\n'.join(f'{name}=={version}' for name, version in updated_name_versions) + + with open(requirements_file, 'w') as fo: + fo.write(nvs) + + +def hash_requirements(dest_dir=THIRDPARTY_DIR, requirements_file='requirements.txt'): + """ + Hash all the requirements found in the `requirements_file` + requirements file based on distributions available in `dest_dir` + """ + local_repo = get_local_repo(directory=dest_dir) + packages_by_normalized_name_version = local_repo.packages_by_normalized_name_version + hashed = [] + for name, version in load_requirements(requirements_file, force_pinned=True): + package = packages_by_normalized_name_version.get((name, version)) + if not package: + raise Exception(f'Missing required package {name}=={version}') + hashed.append(package.specifier_with_hashes) + + with open(requirements_file, 'w') as fo: + fo.write('\n'.join(hashed)) + +################################################################################ +# +# Functions to update or fetch ABOUT and license files +# +################################################################################ + + +def add_fetch_or_update_about_and_license_files(dest_dir=THIRDPARTY_DIR, include_remote=True): + """ + Given a thirdparty dir, add missing ABOUT. LICENSE and NOTICE files using + best efforts: + + - use existing ABOUT files + - try to load existing remote ABOUT files + - derive from existing distribution with same name and latest version that + would have such ABOUT file + - extract ABOUT file data from distributions PKGINFO or METADATA files + - TODO: make API calls to fetch package data from DejaCode + + The process consists in load and iterate on every package distributions, + collect data and then acsk to save. + """ + + local_packages = get_local_packages(directory=dest_dir) + local_repo = get_local_repo(directory=dest_dir) + + remote_repo = get_remote_repo() + + def get_other_dists(_package, _dist): + """ + Return a list of all the dists from package that are not the `dist` object + """ + return [d for d in _package.get_distributions() if d != _dist] + + for local_package in local_packages: + for local_dist in local_package.get_distributions(): + local_dist.load_about_data(dest_dir=dest_dir) + local_dist.set_checksums(dest_dir=dest_dir) + + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + local_dist.save_about_and_notice_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir) + continue + + # lets try to get from another dist of the same local package + for otherd in get_other_dists(local_package, local_dist): + updated = local_dist.update_from_other_dist(otherd) + if updated and local_dist.has_key_metadata(): + break + + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + local_dist.save_about_and_notice_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir) + continue + + # try to get a latest version of the same package that is not our version + other_local_packages = [ + p for p in local_repo.get_versions(local_package.name) + if p.version != local_package.version + ] + + latest_local_version = other_local_packages and other_local_packages[-1] + if latest_local_version: + latest_local_dists = list(latest_local_version.get_distributions()) + for latest_local_dist in latest_local_dists: + latest_local_dist.load_about_data(dest_dir=dest_dir) + if not latest_local_dist.has_key_metadata(): + # there is not much value to get other data if we are missing the key ones + continue + else: + local_dist.update_from_other_dist(latest_local_dist) + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + break + + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + local_dist.save_about_and_notice_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir) + continue + + if include_remote: + # lets try to fetch remotely + local_dist.load_remote_about_data() + + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + local_dist.save_about_and_notice_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir) + continue + + # try to get a latest version of the same package that is not our version + other_remote_packages = [ + p for p in remote_repo.get_versions(local_package.name) + if p.version != local_package.version + ] + + latest_version = other_remote_packages and other_remote_packages[-1] + if latest_version: + latest_dists = list(latest_version.get_distributions()) + for remote_dist in latest_dists: + remote_dist.load_remote_about_data() + if not remote_dist.has_key_metadata(): + # there is not much value to get other data if we are missing the key ones + continue + else: + local_dist.update_from_other_dist(remote_dist) + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + break + + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + local_dist.save_about_and_notice_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir) + continue + + # try to get data from pkginfo (no license though) + local_dist.load_pkginfo_data(dest_dir=dest_dir) + + # FIXME: save as this is the last resort for now in all cases + # if local_dist.has_key_metadata() or not local_dist.has_key_metadata(): + local_dist.save_about_and_notice_files(dest_dir) + + lic_errs = local_dist.fetch_license_files(dest_dir) + + # TODO: try to get data from dejacode + + if not local_dist.has_key_metadata(): + print(f'Unable to add essential ABOUT data for: {local_dist}') + if lic_errs: + lic_errs = '\n'.join(lic_errs) + print(f'Failed to fetch some licenses:: {lic_errs}') + +################################################################################ +# +# Functions to build new Python wheels including native on multiple OSes +# +################################################################################ + + +def call(args): + """ + Call args in a subprocess and display output on the fly. + Return or raise stdout, stderr, returncode + """ + if TRACE: print('Calling:', ' '.join(args)) + with subprocess.Popen( + args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding='utf-8' + ) as process: + + while True: + line = process.stdout.readline() + if not line and process.poll() is not None: + break + if TRACE: print(line.rstrip(), flush=True) + + stdout, stderr = process.communicate() + returncode = process.returncode + if returncode == 0: + return returncode, stdout, stderr + else: + raise Exception(returncode, stdout, stderr) + + +def add_or_upgrade_built_wheels( + name, + version=None, + python_versions=PYTHON_VERSIONS, + operating_systems=PLATFORMS_BY_OS, + dest_dir=THIRDPARTY_DIR, + build_remotely=False, + with_deps=False, + verbose=False, +): + """ + Add or update package `name` and `version` as a binary wheel saved in + `dest_dir`. Use the latest version if `version` is None. Return the a list + of the collected, fetched or built wheel file names or an empty list. + + Use the provided lists of `python_versions` (e.g. "36", "39") and + `operating_systems` (e.g. linux, windows or macos) to decide which specific + wheel to fetch or build. + + Include wheels for all dependencies if `with_deps` is True. + Build remotely is `build_remotely` is True. + """ + assert name, 'Name is required' + ver = version and f'=={version}' or '' + print(f'\nAdding wheels for package: {name}{ver}') + + wheel_filenames = [] + # a mapping of {req specifier: {mapping build_wheels kwargs}} + wheels_to_build = {} + for python_version, operating_system in itertools.product(python_versions, operating_systems): + print(f' Adding wheels for package: {name}{ver} on {python_version,} and {operating_system}') + environment = Environment.from_pyver_and_os(python_version, operating_system) + + # Check if requested wheel already exists locally for this version + local_repo = get_local_repo(directory=dest_dir) + local_package = local_repo.get_package(name=name, version=version) + + has_local_wheel = False + if version and local_package: + for wheel in local_package.get_supported_wheels(environment): + has_local_wheel = True + wheel_filenames.append(wheel.filename) + break + if has_local_wheel: + print(f' local wheel exists: {wheel.filename}') + continue + + if not version: + pypi_package = get_pypi_repo().get_latest_version(name) + version = pypi_package.version + + # Check if requested wheel already exists remotely or in Pypi for this version + wheel_filename = fetch_package_wheel( + name=name, version=version, environment=environment, dest_dir=dest_dir) + if wheel_filename: + wheel_filenames.append(wheel_filename) + + # the wheel is not available locally, remotely or in Pypi + # we need to build binary from sources + requirements_specifier = f'{name}=={version}' + to_build = wheels_to_build.get(requirements_specifier) + if to_build: + to_build['python_versions'].append(python_version) + to_build['operating_systems'].append(operating_system) + else: + wheels_to_build[requirements_specifier] = dict( + requirements_specifier=requirements_specifier, + python_versions=[python_version], + operating_systems=[operating_system], + dest_dir=dest_dir, + build_remotely=build_remotely, + with_deps=with_deps, + verbose=verbose, + ) + + for build_wheels_kwargs in wheels_to_build.values(): + bwheel_filenames = build_wheels(**build_wheels_kwargs) + wheel_filenames.extend(bwheel_filenames) + + return sorted(set(wheel_filenames)) + + +def build_wheels( + requirements_specifier, + python_versions=PYTHON_VERSIONS, + operating_systems=PLATFORMS_BY_OS, + dest_dir=THIRDPARTY_DIR, + build_remotely=False, + with_deps=False, + verbose=False, +): + """ + Given a pip `requirements_specifier` string (such as package names or as + name==version), build the corresponding binary wheel(s) for all + `python_versions` and `operating_systems` combinations and save them + back in `dest_dir` and return a list of built wheel file names. + + Include wheels for all dependencies if `with_deps` is True. + + First try to build locally to process pure Python wheels, and fall back to + build remotey on all requested Pythons and operating systems. + """ + all_pure, builds = build_wheels_locally_if_pure_python( + requirements_specifier=requirements_specifier, + with_deps=with_deps, + verbose=verbose, + dest_dir=dest_dir, + ) + for local_build in builds: + print(f'Built wheel: {local_build}') + + if all_pure: + return builds + + if build_remotely: + remote_builds = build_wheels_remotely_on_multiple_platforms( + requirements_specifier=requirements_specifier, + with_deps=with_deps, + python_versions=python_versions, + operating_systems=operating_systems, + verbose=verbose, + dest_dir=dest_dir, + ) + builds.extend(remote_builds) + + return builds + + +def build_wheels_remotely_on_multiple_platforms( + requirements_specifier, + with_deps=False, + python_versions=PYTHON_VERSIONS, + operating_systems=PLATFORMS_BY_OS, + verbose=False, + dest_dir=THIRDPARTY_DIR, +): + """ + Given pip `requirements_specifier` string (such as package names or as + name==version), build the corresponding binary wheel(s) including wheels for + all dependencies for all `python_versions` and `operating_systems` + combinations and save them back in `dest_dir` and return a list of built + wheel file names. + """ + check_romp_is_configured() + pyos_options = get_romp_pyos_options(python_versions, operating_systems) + deps = '' if with_deps else '--no-deps' + verbose = '--verbose' if verbose else '' + + romp_args = ([ + 'romp', + '--interpreter', 'cpython', + '--architecture', 'x86_64', + '--check-period', '5', # in seconds + + ] + pyos_options + [ + + '--artifact-paths', '*.whl', + '--artifact', 'artifacts.tar.gz', + '--command', + # create a virtualenv, upgrade pip +# f'python -m ensurepip --user --upgrade; ' + f'python -m pip {verbose} install --user --upgrade pip setuptools wheel; ' + f'python -m pip {verbose} wheel {deps} {requirements_specifier}', + ]) + + if verbose: + romp_args.append('--verbose') + + print(f'Building wheels for: {requirements_specifier}') + print(f'Using command:', ' '.join(romp_args)) + call(romp_args) + + wheel_filenames = extract_tar('artifacts.tar.gz', dest_dir) + for wfn in wheel_filenames: + print(f' built wheel: {wfn}') + return wheel_filenames + + +def get_romp_pyos_options( + python_versions=PYTHON_VERSIONS, + operating_systems=PLATFORMS_BY_OS, +): + """ + Return a list of CLI options for romp + For example: + >>> expected = ['--version', '3.6', '--version', '3.7', '--version', '3.8', + ... '--version', '3.9', '--platform', 'linux', '--platform', 'macos', + ... '--platform', 'windows'] + >>> assert get_romp_pyos_options() == expected + """ + python_dot_versions = ['.'.join(pv) for pv in sorted(set(python_versions))] + pyos_options = list(itertools.chain.from_iterable( + ('--version', ver) for ver in python_dot_versions)) + + pyos_options += list(itertools.chain.from_iterable( + ('--platform' , plat) for plat in sorted(set(operating_systems)))) + + return pyos_options + + +def check_romp_is_configured(): + # these environment variable must be set before + has_envt = ( + os.environ.get('ROMP_BUILD_REQUEST_URL') and + os.environ.get('ROMP_DEFINITION_ID') and + os.environ.get('ROMP_PERSONAL_ACCESS_TOKEN') and + os.environ.get('ROMP_USERNAME') + ) + + if not has_envt: + raise Exception( + 'ROMP_BUILD_REQUEST_URL, ROMP_DEFINITION_ID, ' + 'ROMP_PERSONAL_ACCESS_TOKEN and ROMP_USERNAME ' + 'are required enironment variables.') + + +def build_wheels_locally_if_pure_python( + requirements_specifier, + with_deps=False, + verbose=False, + dest_dir=THIRDPARTY_DIR, +): + """ + Given pip `requirements_specifier` string (such as package names or as + name==version), build the corresponding binary wheel(s) locally. + + If all these are "pure" Python wheels that run on all Python 3 versions and + operating systems, copy them back in `dest_dir` if they do not exists there + + Return a tuple of (True if all wheels are "pure", list of built wheel file names) + """ + deps = [] if with_deps else ['--no-deps'] + verbose = ['--verbose'] if verbose else [] + + wheel_dir = tempfile.mkdtemp(prefix='scancode-release-wheels-local-') + cli_args = [ + 'pip', 'wheel', + '--wheel-dir', wheel_dir, + ] + deps + verbose + [ + requirements_specifier + ] + + print(f'Building local wheels for: {requirements_specifier}') + print(f'Using command:', ' '.join(cli_args)) + call(cli_args) + + built = os.listdir(wheel_dir) + if not built: + return [] + + all_pure = all(is_pure_wheel(bwfn) for bwfn in built) + + if not all_pure: + print(f' Some wheels are not pure') + + print(f' Copying local wheels') + pure_built = [] + for bwfn in built: + owfn = os.path.join(dest_dir, bwfn) + if not os.path.exists(owfn): + nwfn = os.path.join(wheel_dir, bwfn) + fileutils.copyfile(nwfn, owfn) + pure_built.append(bwfn) + print(f' Built local wheel: {bwfn}') + return all_pure, pure_built + + +# TODO: Use me +def optimize_wheel(wheel_filename, dest_dir=THIRDPARTY_DIR): + """ + Optimize a wheel named `wheel_filename` in `dest_dir` such as renaming its + tags for PyPI compatibility and making it smaller if possible. Return the + name of the new wheel if renamed or the existing new name otherwise. + """ + if is_pure_wheel(wheel_filename): + print(f'Pure wheel: {wheel_filename}, nothing to do.') + return wheel_filename + + original_wheel_loc = os.path.join(dest_dir, wheel_filename) + wheel_dir = tempfile.mkdtemp(prefix='scancode-release-wheels-') + awargs = [ + 'auditwheel', + 'addtag', + '--wheel-dir', wheel_dir, + original_wheel_loc + ] + call(awargs) + + audited = os.listdir(wheel_dir) + if not audited: + # cannot optimize wheel + return wheel_filename + + assert len(audited) == 1 + new_wheel_name = audited[0] + + new_wheel_loc = os.path.join(wheel_dir, new_wheel_name) + + # this needs to go now + os.remove(original_wheel_loc) + + if new_wheel_name == wheel_filename: + os.rename(new_wheel_loc, original_wheel_loc) + return wheel_filename + + new_wheel = Wheel.from_filename(new_wheel_name) + non_pypi_plats = utils_pypi_supported_tags.validate_platforms_for_pypi(new_wheel.platforms) + new_wheel.platforms = [p for p in new_wheel.platforms if p not in non_pypi_plats] + if not new_wheel.platforms: + print(f'Cannot make wheel PyPI compatible: {original_wheel_loc}') + os.rename(new_wheel_loc, original_wheel_loc) + return wheel_filename + + new_wheel_cleaned_filename = new_wheel.to_filename() + new_wheel_cleaned_loc = os.path.join(dest_dir, new_wheel_cleaned_filename) + os.rename(new_wheel_loc, new_wheel_cleaned_loc) + return new_wheel_cleaned_filename + + +def extract_tar(location, dest_dir=THIRDPARTY_DIR,): + """ + Extract a tar archive at `location` in the `dest_dir` directory. Return a + list of extracted locations (either directories or files). + """ + with open(location, 'rb') as fi: + with tarfile.open(fileobj=fi) as tar: + members = list(tar.getmembers()) + tar.extractall(dest_dir, members=members) + + return [os.path.basename(ti.name) for ti in members + if ti.type == tarfile.REGTYPE] + + +def fetch_package_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR): + """ + Fetch the binary wheel for package `name` and `version` and save in + `dest_dir`. Use the provided `environment` Environment to determine which + specific wheel to fetch. + + Return the fetched wheel file name on success or None if it was not fetched. + Trying fetching from our own remote repo, then from PyPI. + """ + wheel_filename = None + remote_package = get_remote_package(name=name, version=version) + if remote_package: + wheel_filename = remote_package.fetch_wheel( + environment=environment, dest_dir=dest_dir) + if wheel_filename: + return wheel_filename + + pypi_package = get_pypi_package(name=name, version=version) + if pypi_package: + wheel_filename = pypi_package.fetch_wheel( + environment=environment, dest_dir=dest_dir) + return wheel_filename + + +def check_about(dest_dir=THIRDPARTY_DIR): + try: + subprocess.check_output(f'bin/about check {dest_dir}'.split()) + except subprocess.CalledProcessError as cpe: + print() + print('Invalid ABOUT files:') + print(cpe.output.decode('utf-8', errors='replace')) + + +def find_problems( + dest_dir=THIRDPARTY_DIR, + report_missing_sources=False, + report_missing_wheels=False, +): + """ + Print the problems found in `dest_dir`. + """ + + local_packages = get_local_packages(directory=dest_dir) + + for package in local_packages: + if report_missing_sources and not package.sdist: + print(f'{package.name}=={package.version}: Missing source distribution.') + if report_missing_wheels and not package.wheels: + print(f'{package.name}=={package.version}: Missing wheels.') + + for dist in package.get_distributions(): + dist.load_about_data(dest_dir=dest_dir) + abpth = os.path.abspath(os.path.join(dest_dir, dist.about_filename)) + if not dist.has_key_metadata(): + print(f' Missing key ABOUT data in file://{abpth}') + if 'classifiers' in dist.extra_data: + print(f' Dangling classifiers data in file://{abpth}') + if not dist.validate_checksums(dest_dir): + print(f' Invalid checksums in file://{abpth}') + if not dist.sha1 and dist.md5: + print(f' Missing checksums in file://{abpth}') + + check_about(dest_dir=dest_dir) diff --git a/etc/scripts/utils_thirdparty.py.ABOUT b/etc/scripts/utils_thirdparty.py.ABOUT new file mode 100644 index 00000000..84803494 --- /dev/null +++ b/etc/scripts/utils_thirdparty.py.ABOUT @@ -0,0 +1,15 @@ +about_resource: utils_thirdparty.py +package_url: pkg:github.com/pypa/pip/@20.3.1#src/pip/_internal/models/wheel.py +type: github +namespace: pypa +name: pip +version: 20.3.1 +subpath: src/pip/_internal/models/wheel.py + +download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py +copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) +license_expression: mit +notes: copied from pip-20.3.1 pip/_internal/models/wheel.py + The models code has been heavily inspired from the ISC-licensed packaging-dists + https://github.com/uranusjr/packaging-dists by Tzu-ping Chung + \ No newline at end of file From 0e1f56b7cdb0a6a09b01111f6e69faafb7080af4 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 1 Sep 2021 19:00:28 -0700 Subject: [PATCH 106/160] Normalize license in load_pkginfo_data #33 * Create copyright statement from holder information Signed-off-by: Jono Yang --- etc/scripts/utils_thirdparty.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 360f07a6..d5a6d999 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -894,14 +894,20 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): classifiers = raw_data.get_all('Classifier') or [] declared_license = [raw_data['License']] + [c for c in classifiers if c.startswith('License')] + license_expression = compute_normalized_license_expression(declared_license) other_classifiers = [c for c in classifiers if not c.startswith('License')] + holder = raw_data['Author'] + holder_contact=raw_data['Author-email'] + copyright = f'Copyright {holder} <{holder_contact}>' pkginfo_data = dict( name=raw_data['Name'], declared_license=declared_license, version=raw_data['Version'], description=raw_data['Summary'], homepage_url=raw_data['Home-page'], + copyright=copyright, + license_expression=license_expression, holder=raw_data['Author'], holder_contact=raw_data['Author-email'], keywords=raw_data['Keywords'], @@ -2938,3 +2944,25 @@ def find_problems( print(f' Missing checksums in file://{abpth}') check_about(dest_dir=dest_dir) + + +def compute_normalized_license_expression(declared_licenses): + if not declared_licenses: + return + + from packagedcode import licensing + from packagedcode.utils import combine_expressions + + detected_licenses = [] + for declared in declared_licenses: + try: + license_expression = licensing.get_normalized_expression( + query_string=declared + ) + except Exception: + return 'unknown' + if not license_expression: + continue + detected_licenses.append(license_expression) + if detected_licenses: + return combine_expressions(detected_licenses) From 288532d448e5740519f69c8b210a95524e5ec538 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 2 Sep 2021 14:57:09 -0700 Subject: [PATCH 107/160] Add --init option to configure #33 * This is used for the case where we are starting off a project and have not yet generated requirements files Signed-off-by: Jono Yang --- configure | 7 +- etc/scripts/gen_pypi_simple.py | 191 ++++++++++++++++++++++++++ etc/scripts/gen_pypi_simple.py.ABOUT | 8 ++ etc/scripts/gen_pypi_simple.py.NOTICE | 56 ++++++++ etc/scripts/requirements.txt | 12 ++ etc/scripts/utils_thirdparty.py | 30 ++-- 6 files changed, 281 insertions(+), 23 deletions(-) create mode 100644 etc/scripts/gen_pypi_simple.py create mode 100644 etc/scripts/gen_pypi_simple.py.ABOUT create mode 100644 etc/scripts/gen_pypi_simple.py.NOTICE create mode 100644 etc/scripts/requirements.txt diff --git a/configure b/configure index 66d939aa..bbe87b0a 100755 --- a/configure +++ b/configure @@ -53,9 +53,6 @@ CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty --find-links https://thirdparty.aboutcode.org/pypi" -if [[ -f "$CFG_ROOT_DIR/requirements.txt" ]] && [[ -f "$CFG_ROOT_DIR/requirements-dev.txt" ]]; then - PIP_EXTRA_ARGS+=" --no-index" -fi ################################ # Set the quiet flag to empty if not defined @@ -161,13 +158,17 @@ install_packages() { # Main command line entry point CFG_DEV_MODE=0 CFG_REQUIREMENTS=$REQUIREMENTS +NO_INDEX="--no-index" case "$CLI_ARGS" in --help) cli_help;; --clean) clean;; --dev) CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;; + --init) NO_INDEX="";; esac +PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS $NO_INDEX" + create_virtualenv "$VIRTUALENV_DIR" install_packages "$CFG_REQUIREMENTS" . "$CFG_BIN_DIR/activate" diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py new file mode 100644 index 00000000..887e407a --- /dev/null +++ b/etc/scripts/gen_pypi_simple.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# SPDX-License-Identifier: BSD-2-Clause-Views AND MIT +# Copyright (c) 2010 David Wolever . All rights reserved. +# originally from https://github.com/wolever/pip2pi + +import os +import re +import shutil + +from html import escape +from pathlib import Path + +""" +name: pip compatibility tags +version: 20.3.1 +download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py +copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) +license_expression: mit +notes: the weel name regex is copied from pip-20.3.1 pip/_internal/models/wheel.py + +Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +get_wheel_from_filename = re.compile( + r"""^(?P(?P.+?)-(?P.*?)) + ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) + \.whl)$""", + re.VERBOSE +).match + +sdist_exts = ".tar.gz", ".tar.bz2", ".zip", ".tar.xz", +wheel_ext = ".whl" +app_ext = ".pyz" +dist_exts = sdist_exts + (wheel_ext, app_ext) + + +class InvalidDistributionFilename(Exception): + pass + + +def get_package_name_from_filename(filename, normalize=True): + """ + Return the package name extracted from a package ``filename``. + Optionally ``normalize`` the name according to distribution name rules. + Raise an ``InvalidDistributionFilename`` if the ``filename`` is invalid:: + + >>> get_package_name_from_filename("foo-1.2.3_rc1.tar.gz") + 'foo' + >>> get_package_name_from_filename("foo-bar-1.2-py27-none-any.whl") + 'foo-bar' + >>> get_package_name_from_filename("Cython-0.17.2-cp26-none-linux_x86_64.whl") + 'cython' + >>> get_package_name_from_filename("python_ldap-2.4.19-cp27-none-macosx_10_10_x86_64.whl") + 'python-ldap' + >>> get_package_name_from_filename("foo.whl") + Traceback (most recent call last): + ... + InvalidDistributionFilename: ... + >>> get_package_name_from_filename("foo.png") + Traceback (most recent call last): + ... + InvalidFilePackageName: ... + """ + if not filename or not filename.endswith(dist_exts): + raise InvalidDistributionFilename(filename) + + filename = os.path.basename(filename) + + if filename.endswith(sdist_exts): + name_ver = None + extension = None + + for ext in sdist_exts: + if filename.endswith(ext): + name_ver, extension, _ = filename.rpartition(ext) + break + + if not extension or not name_ver: + raise InvalidDistributionFilename(filename) + + name, _, version = name_ver.rpartition('-') + + if not (name and version): + raise InvalidDistributionFilename(filename) + + elif filename.endswith(wheel_ext): + + wheel_info = get_wheel_from_filename(filename) + + if not wheel_info: + raise InvalidDistributionFilename(filename) + + name = wheel_info.group('name') + version = wheel_info.group('version') + + if not (name and version): + raise InvalidDistributionFilename(filename) + + elif filename.endswith(app_ext): + name_ver, extension, _ = filename.rpartition(".pyz") + + if "-" in filename: + name, _, version = name_ver.rpartition('-') + else: + name = name_ver + + if not name: + raise InvalidDistributionFilename(filename) + + if normalize: + name = name.lower().replace('_', '-') + return name + + +def build_pypi_index(directory, write_index=False): + """ + Using a ``directory`` directory of wheels and sdists, create the a PyPI simple + directory index at ``directory``/simple/ populated with the proper PyPI simple + index directory structure crafted using symlinks. + + WARNING: The ``directory``/simple/ directory is removed if it exists. + """ + + directory = Path(directory) + + index_dir = directory / "simple" + if index_dir.exists(): + shutil.rmtree(str(index_dir), ignore_errors=True) + + index_dir.mkdir(parents=True) + + if write_index: + simple_html_index = [ + "PyPI Simple Index", + "", + ] + + package_names = set() + for pkg_file in directory.iterdir(): + + pkg_filename = pkg_file.name + + if ( + not pkg_file.is_file() + or not pkg_filename.endswith(dist_exts) + or pkg_filename.startswith(".") + ): + continue + + pkg_name = get_package_name_from_filename(pkg_filename) + pkg_index_dir = index_dir / pkg_name + pkg_index_dir.mkdir(parents=True, exist_ok=True) + pkg_indexed_file = pkg_index_dir / pkg_filename + link_target = Path("../..") / pkg_filename + pkg_indexed_file.symlink_to(link_target) + + if write_index and pkg_name not in package_names: + esc_name = escape(pkg_name) + simple_html_index.append(f'{esc_name}
') + package_names.add(pkg_name) + + if write_index: + simple_html_index.append("") + index_html = index_dir / "index.html" + index_html.write_text("\n".join(simple_html_index)) + + +if __name__ == "__main__": + import sys + pkg_dir = sys.argv[1] + build_pypi_index(pkg_dir) diff --git a/etc/scripts/gen_pypi_simple.py.ABOUT b/etc/scripts/gen_pypi_simple.py.ABOUT new file mode 100644 index 00000000..4de5ded0 --- /dev/null +++ b/etc/scripts/gen_pypi_simple.py.ABOUT @@ -0,0 +1,8 @@ +about_resource: gen_pypi_simple.py +name: gen_pypi_simple.py +license_expression: bsd-2-clause-views and mit +copyright: Copyright (c) nexB Inc. + Copyright (c) 2010 David Wolever + Copyright (c) The pip developers +notes: Originally from https://github.com/wolever/pip2pi and modified extensivley + Also partially derived from pip code diff --git a/etc/scripts/gen_pypi_simple.py.NOTICE b/etc/scripts/gen_pypi_simple.py.NOTICE new file mode 100644 index 00000000..6e0fbbcd --- /dev/null +++ b/etc/scripts/gen_pypi_simple.py.NOTICE @@ -0,0 +1,56 @@ +SPDX-License-Identifier: BSD-2-Clause-Views AND mit + +Copyright (c) nexB Inc. +Copyright (c) 2010 David Wolever +Copyright (c) The pip developers + + +Original code: copyright 2010 David Wolever . All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation are those +of the authors and should not be interpreted as representing official policies, +either expressed or implied, of David Wolever. + + +Original code: Copyright (c) 2008-2020 The pip developers + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/etc/scripts/requirements.txt b/etc/scripts/requirements.txt new file mode 100644 index 00000000..6591e49c --- /dev/null +++ b/etc/scripts/requirements.txt @@ -0,0 +1,12 @@ +aboutcode_toolkit +github-release-retry2 +attrs +commoncode +click +requests +saneyaml +romp +pip +setuptools +twine +wheel \ No newline at end of file diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index d5a6d999..c0613c3a 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -899,7 +899,8 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): holder = raw_data['Author'] holder_contact=raw_data['Author-email'] - copyright = f'Copyright {holder} <{holder_contact}>' + copyright = f'Copyright (c) {holder} <{holder_contact}>' + pkginfo_data = dict( name=raw_data['Name'], declared_license=declared_license, @@ -908,8 +909,8 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): homepage_url=raw_data['Home-page'], copyright=copyright, license_expression=license_expression, - holder=raw_data['Author'], - holder_contact=raw_data['Author-email'], + holder=holder, + holder_contact=holder_contact, keywords=raw_data['Keywords'], classifiers=other_classifiers, ) @@ -2949,20 +2950,9 @@ def find_problems( def compute_normalized_license_expression(declared_licenses): if not declared_licenses: return - - from packagedcode import licensing - from packagedcode.utils import combine_expressions - - detected_licenses = [] - for declared in declared_licenses: - try: - license_expression = licensing.get_normalized_expression( - query_string=declared - ) - except Exception: - return 'unknown' - if not license_expression: - continue - detected_licenses.append(license_expression) - if detected_licenses: - return combine_expressions(detected_licenses) + try: + from packagedcode import pypi + return pypi.compute_normalized_license(declared_licenses) + except ImportError: + # Scancode is not installed, we join all license strings and return it + return ' '.join(declared_licenses) From a5ae4f35473a38427f3fba9b225b6de0ec16522c Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 2 Sep 2021 16:48:04 -0700 Subject: [PATCH 108/160] Update README.rst #33 Signed-off-by: Jono Yang --- README.rst | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index b84a0491..a52d8050 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ A Simple Python Project Skeleton ================================ This repo attempts to standardize our python repositories using modern python packaging and configuration techniques. Using this `blog post`_ as inspiration, this -repository will serve as the base for all new python projects and will be adopted to all +repository will serve as the base for all new python projects and will be adopted to all our existing ones as well. .. _blog post: https://blog.jaraco.com/a-project-skeleton-for-python-projects/ @@ -33,7 +33,6 @@ Update an existing project This is also the workflow to use when updating the skeleton files in any given repository. - Customizing ----------- @@ -42,6 +41,72 @@ You typically want to perform these customizations: - remove or update the src/README.rst and tests/README.rst files - check the configure and configure.bat defaults +Initializing a project +---------------------- + +All projects using the skeleton will be expected to pull all of it dependencies +from thirdparty.aboutcode.org/pypi or the local thirdparty directory, using +requirements.txt and/or requirements-dev.txt to determine what version of a +package to collect. By default, PyPI will not be used to find and collect +packages from. + +In the case where we are starting a new project where we do not have +requirements.txt and requirements-dev.txt and whose dependencies are not yet on +thirdparty.aboutcode.org/pypi, we run the following command after adding and +customizing the skeleton files to your project: + +.. code-block:: bash + + ./configure --init + +This will initialize the virtual environment for the project, pull in the +dependencies from PyPI and add them to the virtual environment. + +Generating requirements.txt and requirements-dev.txt +---------------------------------------------------- + +After the project has been initialized, we can generate the requirements.txt and +requirements-dev.txt files. + +Ensure the virtual environment is enabled. + +To generate requirements.txt: + +.. code-block:: bash + + python etc/scripts/gen_requirements.py -s tmp/lib/python/site-packages/ + +Replace \ with the version number of the Python being used. + +To generate requirements-dev.txt after requirements.txt has been generated: + +.. code-block:: bash + ./configure --dev + source tmp/bin/activate + python etc/scripts/gen_requirements_dev.py -s tmp/lib/python/site-packages/ + +Collecting and generating ABOUT files for dependencies +------------------------------------------------------ + +Once we have requirements.txt and requirements-dev.txt, we can fetch the project +dependencies as wheels and generate ABOUT files for them: + +.. code-block:: bash + + python etc/scripts/bootstrap.py -r requirements.txt -r requirements-dev.txt --with-deps + +There may be issues with the generated ABOUT files, which will have to be +corrected. You can check to see if your corrections are valid by running: + +.. code-block:: bash + + python etc/scripts/check_thirdparty.py -d thirdparty + +Once the wheels are collected and the ABOUT files are generated and correct, +upload them to thirdparty.aboutcode.org/pypi by placing the wheels and ABOUT +files from the thirdparty directory to the pypi directory at +https://github.com/nexB/thirdparty-packages + Release Notes ------------- From 593e2379c688e92985a3c6eceabf69cb721207a5 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 2 Sep 2021 17:09:06 -0700 Subject: [PATCH 109/160] Use venv as virtual environment directory name #37 * Replace all references to `tmp` with `venv` Signed-off-by: Jono Yang --- .gitignore | 1 + .travis.yml | 2 +- README.rst | 6 +++--- azure-pipelines.yml | 14 +++++++------- configure | 4 ++-- configure.bat | 4 ++-- pyproject.toml | 3 ++- 7 files changed, 18 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 68de2d22..339dca50 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ /Lib /pip-selfcheck.json /tmp +/venv .Python /include /Include diff --git a/.travis.yml b/.travis.yml index 1a90a385..ea48ceb5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,4 +19,4 @@ python: install: ./configure --dev # Scripts to run at script stage -script: tmp/bin/pytest +script: venv/bin/pytest diff --git a/README.rst b/README.rst index a52d8050..08ef0835 100644 --- a/README.rst +++ b/README.rst @@ -74,7 +74,7 @@ To generate requirements.txt: .. code-block:: bash - python etc/scripts/gen_requirements.py -s tmp/lib/python/site-packages/ + python etc/scripts/gen_requirements.py -s venv/lib/python/site-packages/ Replace \ with the version number of the Python being used. @@ -82,8 +82,8 @@ To generate requirements-dev.txt after requirements.txt has been generated: .. code-block:: bash ./configure --dev - source tmp/bin/activate - python etc/scripts/gen_requirements_dev.py -s tmp/lib/python/site-packages/ + source venv/bin/activate + python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ Collecting and generating ABOUT files for dependencies ------------------------------------------------------ diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 31ef36f0..22c12c43 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,7 +13,7 @@ jobs: image_name: ubuntu-16.04 python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: - all: tmp/bin/pytest -vvs + all: venv/bin/pytest -vvs - template: etc/ci/azure-posix.yml parameters: @@ -21,7 +21,7 @@ jobs: image_name: ubuntu-18.04 python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: - all: tmp/bin/pytest -n 2 -vvs + all: venv/bin/pytest -n 2 -vvs - template: etc/ci/azure-posix.yml parameters: @@ -29,7 +29,7 @@ jobs: image_name: ubuntu-20.04 python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: - all: tmp/bin/pytest -n 2 -vvs + all: venv/bin/pytest -n 2 -vvs - template: etc/ci/azure-posix.yml parameters: @@ -37,7 +37,7 @@ jobs: image_name: macos-10.14 python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: - all: tmp/bin/pytest -n 2 -vvs + all: venv/bin/pytest -n 2 -vvs - template: etc/ci/azure-posix.yml parameters: @@ -45,7 +45,7 @@ jobs: image_name: macos-10.15 python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: - all: tmp/bin/pytest -n 2 -vvs + all: venv/bin/pytest -n 2 -vvs - template: etc/ci/azure-win.yml parameters: @@ -53,7 +53,7 @@ jobs: image_name: vs2017-win2016 python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: - all: tmp\Scripts\pytest -n 2 -vvs + all: venv\Scripts\pytest -n 2 -vvs - template: etc/ci/azure-win.yml parameters: @@ -61,4 +61,4 @@ jobs: image_name: windows-2019 python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: - all: tmp\Scripts\pytest -n 2 -vvs + all: venv\Scripts\pytest -n 2 -vvs diff --git a/configure b/configure index bbe87b0a..7c162c7e 100755 --- a/configure +++ b/configure @@ -30,12 +30,12 @@ REQUIREMENTS="--editable . --constraint requirements.txt" DEV_REQUIREMENTS="--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" # where we create a virtualenv -VIRTUALENV_DIR=tmp +VIRTUALENV_DIR=venv # Cleanable files and directories with the --clean option CLEANABLE=" build - tmp" + venv" # extra arguments passed to pip PIP_EXTRA_ARGS=" " diff --git a/configure.bat b/configure.bat index 75cab5fc..529c3718 100644 --- a/configure.bat +++ b/configure.bat @@ -28,10 +28,10 @@ set "REQUIREMENTS=--editable . --constraint requirements.txt" set "DEV_REQUIREMENTS=--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" @rem # where we create a virtualenv -set "VIRTUALENV_DIR=tmp" +set "VIRTUALENV_DIR=venv" @rem # Cleanable files and directories to delete with the --clean option -set "CLEANABLE=build tmp" +set "CLEANABLE=build venv" @rem # extra arguments passed to pip set "PIP_EXTRA_ARGS= " diff --git a/pyproject.toml b/pyproject.toml index 852f0fce..1e10f326 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,10 +33,11 @@ norecursedirs = [ "Scripts", "thirdparty", "tmp", + "venv", "tests/data", ".eggs" ] - + python_files = "*.py" python_classes = "Test" From 9342bc1057da73bf39f1b0cb86b85bc581a76793 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 2 Sep 2021 18:26:29 -0700 Subject: [PATCH 110/160] Update configure.bat #33 * Add --init option to configure.bat * Update help text in configure and configure.bat Signed-off-by: Jono Yang --- configure | 4 ++++ configure.bat | 16 +++++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/configure b/configure index 7c162c7e..3c607882 100755 --- a/configure +++ b/configure @@ -81,10 +81,14 @@ cli_help() { echo " usage: ./configure [options]" echo echo The default is to configure for regular use. Use --dev for development. + echo Use the --init option if starting a new project and the project + echo dependencies are not available on thirdparty.aboutcode.org/pypi/ + echo and requirements.txt and/or requirements-dev.txt has not been generated. echo echo The options are: echo " --clean: clean built and installed files and exit." echo " --dev: configure the environment for development." + echo " --init: pull dependencies from PyPI. Used when first setting up a project." echo " --help: display this help message and exit." echo echo By default, the python interpreter version found in the path is used. diff --git a/configure.bat b/configure.bat index 529c3718..dc6db8b3 100644 --- a/configure.bat +++ b/configure.bat @@ -49,11 +49,6 @@ set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" @rem ################################ @rem # Thirdparty package locations and index handling -if exist ""%CFG_ROOT_DIR%\requirements.txt"" if exist ""%CFG_ROOT_DIR%\requirements-dev.txt"" ( - set "INDEX_ARG= --no-index" -) else ( - set "INDEX_ARG= " -) set "PIP_EXTRA_ARGS=--find-links %CFG_ROOT_DIR%\thirdparty --find-links https://thirdparty.aboutcode.org/pypi" & %INDEX_ARG% @rem ################################ @@ -69,6 +64,7 @@ if not defined CFG_QUIET ( @rem # Main command line entry point set CFG_DEV_MODE=0 set "CFG_REQUIREMENTS=%REQUIREMENTS%" +set "NO_INDEX=--no-index" if "%1" EQU "--help" (goto cli_help) if "%1" EQU "--clean" (goto clean) @@ -76,12 +72,18 @@ if "%1" EQU "--dev" ( set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" set CFG_DEV_MODE=1 ) +if "%1" EQU "--init" ( + set "NO_INDEX= " +) if "%1" EQU "--python" ( echo "The --python option is now DEPRECATED. Use the PYTHON_EXECUTABLE environment" echo "variable instead. Run configure --help for details." exit /b 0 ) +set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% %NO_INDEX%" + + @rem ################################ @rem # find a proper Python to run @rem # Use environment variables or a file if available. @@ -170,10 +172,14 @@ exit /b 0 echo " usage: configure [options]" echo " " echo The default is to configure for regular use. Use --dev for development. + echo Use the --init option if starting a new project and the project + echo dependencies are not available on thirdparty.aboutcode.org/pypi/ + echo and requirements.txt and/or requirements-dev.txt has not been generated. echo " " echo The options are: echo " --clean: clean built and installed files and exit." echo " --dev: configure the environment for development." + echo " --init: pull dependencies from PyPI. Used when first setting up a project." echo " --help: display this help message and exit." echo " " echo By default, the python interpreter version found in the path is used. From 45e4a2aaf2e887f1ccade825c323be68bad7d127 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 2 Sep 2021 18:42:03 -0700 Subject: [PATCH 111/160] Add placeholder requirements.txt files #33 Signed-off-by: Jono Yang --- requirements-dev.txt | 0 requirements.txt | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 requirements-dev.txt create mode 100644 requirements.txt diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..e69de29b diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..e69de29b From 944fbaee4ee4c317252ecdafe36af946dc58a9b8 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Fri, 3 Sep 2021 14:33:56 -0700 Subject: [PATCH 112/160] Handle multiple options in configure #33 Signed-off-by: Jono Yang --- README.rst | 4 ++-- configure | 18 ++++++++++++------ configure.bat | 31 ++++++++++++++++++------------- 3 files changed, 32 insertions(+), 21 deletions(-) diff --git a/README.rst b/README.rst index 08ef0835..78ab9f49 100644 --- a/README.rst +++ b/README.rst @@ -76,12 +76,12 @@ To generate requirements.txt: python etc/scripts/gen_requirements.py -s venv/lib/python/site-packages/ -Replace \ with the version number of the Python being used. +Replace \ with the version number of the Python being used, for example: ``venv/lib/python3.6/site-packages/`` To generate requirements-dev.txt after requirements.txt has been generated: .. code-block:: bash - ./configure --dev + ./configure --init --dev source venv/bin/activate python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ diff --git a/configure b/configure index 3c607882..b965692d 100755 --- a/configure +++ b/configure @@ -164,12 +164,18 @@ CFG_DEV_MODE=0 CFG_REQUIREMENTS=$REQUIREMENTS NO_INDEX="--no-index" -case "$CLI_ARGS" in - --help) cli_help;; - --clean) clean;; - --dev) CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;; - --init) NO_INDEX="";; -esac +# We are using getopts to parse option arguments that start with "-" +while getopts :-: optchar; do + case "${optchar}" in + -) + case "${OPTARG}" in + help ) cli_help;; + clean ) clean;; + dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;; + init ) NO_INDEX="";; + esac;; + esac +done PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS $NO_INDEX" diff --git a/configure.bat b/configure.bat index dc6db8b3..31f91c46 100644 --- a/configure.bat +++ b/configure.bat @@ -66,19 +66,24 @@ set CFG_DEV_MODE=0 set "CFG_REQUIREMENTS=%REQUIREMENTS%" set "NO_INDEX=--no-index" -if "%1" EQU "--help" (goto cli_help) -if "%1" EQU "--clean" (goto clean) -if "%1" EQU "--dev" ( - set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" - set CFG_DEV_MODE=1 -) -if "%1" EQU "--init" ( - set "NO_INDEX= " -) -if "%1" EQU "--python" ( - echo "The --python option is now DEPRECATED. Use the PYTHON_EXECUTABLE environment" - echo "variable instead. Run configure --help for details." - exit /b 0 +:again +if not "%1" == "" ( + if "%1" EQU "--help" (goto cli_help) + if "%1" EQU "--clean" (goto clean) + if "%1" EQU "--dev" ( + set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" + set CFG_DEV_MODE=1 + ) + if "%1" EQU "--init" ( + set "NO_INDEX= " + ) + if "%1" EQU "--python" ( + echo "The --python option is now DEPRECATED. Use the PYTHON_EXECUTABLE environment" + echo "variable instead. Run configure --help for details." + exit /b 0 + ) + shift + goto again ) set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% %NO_INDEX%" From 3532b22ed0bb15e77dc315c527908dc87629e0e2 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Fri, 3 Sep 2021 17:01:20 -0700 Subject: [PATCH 113/160] Fix path to aboutcode in utils_thirdparty.py #33 * Update README.rst Signed-off-by: Jono Yang --- README.rst | 11 ++++++++++- etc/scripts/utils_thirdparty.py | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 78ab9f49..5853bf57 100644 --- a/README.rst +++ b/README.rst @@ -70,6 +70,10 @@ requirements-dev.txt files. Ensure the virtual environment is enabled. +.. code-block:: bash + + source venv/bin/activate + To generate requirements.txt: .. code-block:: bash @@ -82,12 +86,17 @@ To generate requirements-dev.txt after requirements.txt has been generated: .. code-block:: bash ./configure --init --dev - source venv/bin/activate python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ Collecting and generating ABOUT files for dependencies ------------------------------------------------------ +Ensure that the dependencies used by ``etc/scripts/bootstrap.py`` are installed: + +.. code-block:: bash + + pip install -r etc/scripts/requirements.txt + Once we have requirements.txt and requirements-dev.txt, we can fetch the project dependencies as wheels and generate ABOUT files for them: diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index c0613c3a..23e837ff 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -2908,7 +2908,7 @@ def fetch_package_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR): def check_about(dest_dir=THIRDPARTY_DIR): try: - subprocess.check_output(f'bin/about check {dest_dir}'.split()) + subprocess.check_output(f'venv/bin/about check {dest_dir}'.split()) except subprocess.CalledProcessError as cpe: print() print('Invalid ABOUT files:') From 9c78ddb5100dec3e6c57079bb3c06fbdc7b79b1c Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Fri, 3 Sep 2021 17:34:38 -0700 Subject: [PATCH 114/160] Update release notes in README.rst Signed-off-by: Jono Yang --- README.rst | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 5853bf57..40226e04 100644 --- a/README.rst +++ b/README.rst @@ -39,6 +39,7 @@ Customizing You typically want to perform these customizations: - remove or update the src/README.rst and tests/README.rst files +- set project info and dependencies in setup.cfg - check the configure and configure.bat defaults Initializing a project @@ -118,7 +119,15 @@ https://github.com/nexB/thirdparty-packages Release Notes -------------- - -- 2021-05-11: adopt new configure scripts from ScanCode TK that allows correct - configuration of which Python version is used. +============= + +- 2021-09-03: + - ``configure`` now requires pinned dependencies via the use of ``requirements.txt`` and ``requirements-dev.txt`` + - ``configure`` can now accept multiple options at once + - Add utility scripts from scancode-toolkit/etc/release/ for use in generating project files + - Rename virtual environment directory from ``tmp`` to ``venv`` + - Update README.rst with instructions for generating ``requirements.txt`` and ``requirements-dev.txt``, + as well as collecting dependencies as wheels and generating ABOUT files for them. + +- 2021-05-11: + - Adopt new configure scripts from ScanCode TK that allows correct configuration of which Python version is used. From ebcfb933a7483d0a7cd1fc02d724cec5ef9b2d28 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Fri, 3 Sep 2021 18:59:49 -0700 Subject: [PATCH 115/160] Handle ExpressionParseError #33 * Update README.rst with instructions for post-initialization usage Signed-off-by: Jono Yang --- README.rst | 47 +++++++++++++++++++++++++++++++++ etc/scripts/utils_thirdparty.py | 6 ++++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 40226e04..b15be20d 100644 --- a/README.rst +++ b/README.rst @@ -89,6 +89,14 @@ To generate requirements-dev.txt after requirements.txt has been generated: ./configure --init --dev python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ +Note: on Windows, the ``site-packages`` directory is located at ``venv\Lib\site-packages\`` + +.. code-block:: bash + + python .\\etc\\scripts\\gen_requirements.py -s .\\venv\\Lib\\site-packages\\ + .\configure --init --dev + python .\\etc\\scripts\\gen_requirements_dev.py -s .\\venv\\Lib\\site-packages\\ + Collecting and generating ABOUT files for dependencies ------------------------------------------------------ @@ -118,6 +126,45 @@ files from the thirdparty directory to the pypi directory at https://github.com/nexB/thirdparty-packages +Usage after project initialization +---------------------------------- + +Once the ``requirements.txt`` and ``requirements-dev.txt`` has been generated +and the project dependencies and their ABOUT files have been uploaded to +thirdparty.aboutcode.org/pypi, you can configure the project without using the +``--init`` option. + +If the virtual env for the project becomes polluted, or you would like to remove +it, use the ``--clean`` option: + +.. code-block:: bash + + ./configure --clean + +Then you can run ``./configure`` again to set up the project virtual environment. + +To set up the project for development use: + +.. code-block:: bash + + ./configure --dev + +To update the project dependencies (adding, removing, updating packages, etc.), +update the dependencies in ``setup.cfg``, then run: + +.. code-block:: bash + + ./configure --clean # Remove existing virtual environment + ./configure --init # Create project virtual environment, pull in new dependencies + source venv/bin/activate # Ensure virtual environment is activated + python etc/scripts/gen_requirements.py -s venv/lib/python/site-packages/ # Regenerate requirements.txt + python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ # Regenerate requirements-dev.txt + pip install -r etc/scripts/requirements.txt # Install dependencies needed by etc/scripts/bootstrap.py + python etc/scripts/bootstrap.py -r requirements.txt -r requirements-dev.txt --with-deps # Collect dependency wheels and their ABOUT files + +Ensure that the generated ABOUT files are valid, then take the dependency wheels +and ABOUT files and upload them to thirdparty.aboutcode.org/pypi. + Release Notes ============= diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 23e837ff..978f0e13 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -814,7 +814,11 @@ def get_pip_hash(self): return f'--hash=sha256:{self.sha256}' def get_license_keys(self): - return LICENSING.license_keys(self.license_expression, unique=True, simple=True) + try: + keys = LICENSING.license_keys(self.license_expression, unique=True, simple=True) + except license_expression.ExpressionParseError: + return ['unknown'] + return keys def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): """ From 6ab9c10e405b7cff243751082b7a7da4354256a8 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 7 Sep 2021 19:42:08 +0200 Subject: [PATCH 116/160] Update README.rst Signed-off-by: Philippe Ombredanne --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index b15be20d..deaaa341 100644 --- a/README.rst +++ b/README.rst @@ -129,7 +129,7 @@ https://github.com/nexB/thirdparty-packages Usage after project initialization ---------------------------------- -Once the ``requirements.txt`` and ``requirements-dev.txt`` has been generated +Once the ``requirements.txt`` and ``requirements-dev.txt`` have been generated and the project dependencies and their ABOUT files have been uploaded to thirdparty.aboutcode.org/pypi, you can configure the project without using the ``--init`` option. From bfdc6ff042a5866e67aa4adab4cd5ac71d47285e Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 7 Sep 2021 12:27:08 -0700 Subject: [PATCH 117/160] Address review comments #33 * Replace references to scancode-toolkit repo with links to the skeleton repo * Remove --python option from configure.bat Signed-off-by: Jono Yang --- configure.bat | 5 ----- etc/scripts/bootstrap.py | 6 +++--- etc/scripts/build_wheels.py | 2 +- etc/scripts/check_thirdparty.py | 2 +- etc/scripts/fetch_requirements.py | 4 ++-- etc/scripts/fix_thirdparty.py | 2 +- etc/scripts/gen_requirements.py | 2 +- etc/scripts/gen_requirements_dev.py | 2 +- etc/scripts/utils_dejacode.py | 2 +- etc/scripts/utils_requirements.py | 2 +- etc/scripts/utils_thirdparty.py | 2 +- 11 files changed, 13 insertions(+), 18 deletions(-) diff --git a/configure.bat b/configure.bat index 31f91c46..0c824a47 100644 --- a/configure.bat +++ b/configure.bat @@ -77,11 +77,6 @@ if not "%1" == "" ( if "%1" EQU "--init" ( set "NO_INDEX= " ) - if "%1" EQU "--python" ( - echo "The --python option is now DEPRECATED. Use the PYTHON_EXECUTABLE environment" - echo "variable instead. Run configure --help for details." - exit /b 0 - ) shift goto again ) diff --git a/etc/scripts/bootstrap.py b/etc/scripts/bootstrap.py index 54701f63..fde505bc 100644 --- a/etc/scripts/bootstrap.py +++ b/etc/scripts/bootstrap.py @@ -5,7 +5,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -84,7 +84,7 @@ def bootstrap( OS(s) defaulting to all supported combinations. Create or fetch .ABOUT and .LICENSE files. - Optionally ignore version specifiers and use the ``--latest-version`` + Optionally ignore version specifiers and use the ``--latest-version`` of everything. Sources and wheels are fetched with attempts first from PyPI, then our remote repository. @@ -172,7 +172,7 @@ def bootstrap( (PypiPackage(name, version), envt) for name, version, envt in name_version_envt_to_build ] - + print(f'==> BUILDING #{len(packages_and_envts_to_build)} MISSING WHEELS') package_envts_not_built, wheel_filenames_built = utils_thirdparty.build_missing_wheels( diff --git a/etc/scripts/build_wheels.py b/etc/scripts/build_wheels.py index 416adc7c..352b7055 100644 --- a/etc/scripts/build_wheels.py +++ b/etc/scripts/build_wheels.py @@ -5,7 +5,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import click diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index b29ce2be..e48cfce3 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -5,7 +5,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import click diff --git a/etc/scripts/fetch_requirements.py b/etc/scripts/fetch_requirements.py index dfd202a7..21de865b 100644 --- a/etc/scripts/fetch_requirements.py +++ b/etc/scripts/fetch_requirements.py @@ -5,7 +5,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import itertools @@ -108,7 +108,7 @@ def fetch_requirements( envs = (utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in envs) for env, reqf in itertools.product(envs, requirements_files): - + for package, error in utils_thirdparty.fetch_wheels( environment=env, requirements_file=reqf, diff --git a/etc/scripts/fix_thirdparty.py b/etc/scripts/fix_thirdparty.py index b74b4979..061d3fac 100644 --- a/etc/scripts/fix_thirdparty.py +++ b/etc/scripts/fix_thirdparty.py @@ -5,7 +5,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import click diff --git a/etc/scripts/gen_requirements.py b/etc/scripts/gen_requirements.py index c917c873..3be974cc 100644 --- a/etc/scripts/gen_requirements.py +++ b/etc/scripts/gen_requirements.py @@ -5,7 +5,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import click diff --git a/etc/scripts/gen_requirements_dev.py b/etc/scripts/gen_requirements_dev.py index 91e0ce61..ff4ce500 100644 --- a/etc/scripts/gen_requirements_dev.py +++ b/etc/scripts/gen_requirements_dev.py @@ -5,7 +5,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import click diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index bb37de1c..8b6e5d20 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -5,7 +5,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import io diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 8b088adf..ddbed612 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -5,7 +5,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import subprocess diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 978f0e13..0ebf6b24 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -5,7 +5,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # from collections import defaultdict From 71d8dad4444f3de7a66a776e8848b0f1d1b1e201 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 8 Sep 2021 12:12:48 -0700 Subject: [PATCH 118/160] Update READMEs Signed-off-by: Jono Yang --- README.rst | 156 +-------------------------------------- docs/skeleton-usage.rst | 157 ++++++++++++++++++++++++++++++++++++++++ etc/scripts/README.rst | 147 +++++++++++++++++++++++++++++++++++++ 3 files changed, 305 insertions(+), 155 deletions(-) create mode 100644 docs/skeleton-usage.rst create mode 100755 etc/scripts/README.rst diff --git a/README.rst b/README.rst index deaaa341..41736895 100644 --- a/README.rst +++ b/README.rst @@ -9,161 +9,7 @@ our existing ones as well. Usage ===== -A brand new project -------------------- -.. code-block:: bash - - git init my-new-repo - cd my-new-repo - git pull git@github.com:nexB/skeleton - - # Create the new repo on GitHub, then update your remote - git remote set-url origin git@github.com:nexB/your-new-repo.git - -From here, you can make the appropriate changes to the files for your specific project. - -Update an existing project ---------------------------- -.. code-block:: bash - - cd my-existing-project - git remote add skeleton git@github.com:nexB/skeleton - git fetch skeleton - git merge skeleton/main --allow-unrelated-histories - -This is also the workflow to use when updating the skeleton files in any given repository. - -Customizing ------------ - -You typically want to perform these customizations: - -- remove or update the src/README.rst and tests/README.rst files -- set project info and dependencies in setup.cfg -- check the configure and configure.bat defaults - -Initializing a project ----------------------- - -All projects using the skeleton will be expected to pull all of it dependencies -from thirdparty.aboutcode.org/pypi or the local thirdparty directory, using -requirements.txt and/or requirements-dev.txt to determine what version of a -package to collect. By default, PyPI will not be used to find and collect -packages from. - -In the case where we are starting a new project where we do not have -requirements.txt and requirements-dev.txt and whose dependencies are not yet on -thirdparty.aboutcode.org/pypi, we run the following command after adding and -customizing the skeleton files to your project: - -.. code-block:: bash - - ./configure --init - -This will initialize the virtual environment for the project, pull in the -dependencies from PyPI and add them to the virtual environment. - -Generating requirements.txt and requirements-dev.txt ----------------------------------------------------- - -After the project has been initialized, we can generate the requirements.txt and -requirements-dev.txt files. - -Ensure the virtual environment is enabled. - -.. code-block:: bash - - source venv/bin/activate - -To generate requirements.txt: - -.. code-block:: bash - - python etc/scripts/gen_requirements.py -s venv/lib/python/site-packages/ - -Replace \ with the version number of the Python being used, for example: ``venv/lib/python3.6/site-packages/`` - -To generate requirements-dev.txt after requirements.txt has been generated: - -.. code-block:: bash - ./configure --init --dev - python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ - -Note: on Windows, the ``site-packages`` directory is located at ``venv\Lib\site-packages\`` - -.. code-block:: bash - - python .\\etc\\scripts\\gen_requirements.py -s .\\venv\\Lib\\site-packages\\ - .\configure --init --dev - python .\\etc\\scripts\\gen_requirements_dev.py -s .\\venv\\Lib\\site-packages\\ - -Collecting and generating ABOUT files for dependencies ------------------------------------------------------- - -Ensure that the dependencies used by ``etc/scripts/bootstrap.py`` are installed: - -.. code-block:: bash - - pip install -r etc/scripts/requirements.txt - -Once we have requirements.txt and requirements-dev.txt, we can fetch the project -dependencies as wheels and generate ABOUT files for them: - -.. code-block:: bash - - python etc/scripts/bootstrap.py -r requirements.txt -r requirements-dev.txt --with-deps - -There may be issues with the generated ABOUT files, which will have to be -corrected. You can check to see if your corrections are valid by running: - -.. code-block:: bash - - python etc/scripts/check_thirdparty.py -d thirdparty - -Once the wheels are collected and the ABOUT files are generated and correct, -upload them to thirdparty.aboutcode.org/pypi by placing the wheels and ABOUT -files from the thirdparty directory to the pypi directory at -https://github.com/nexB/thirdparty-packages - - -Usage after project initialization ----------------------------------- - -Once the ``requirements.txt`` and ``requirements-dev.txt`` have been generated -and the project dependencies and their ABOUT files have been uploaded to -thirdparty.aboutcode.org/pypi, you can configure the project without using the -``--init`` option. - -If the virtual env for the project becomes polluted, or you would like to remove -it, use the ``--clean`` option: - -.. code-block:: bash - - ./configure --clean - -Then you can run ``./configure`` again to set up the project virtual environment. - -To set up the project for development use: - -.. code-block:: bash - - ./configure --dev - -To update the project dependencies (adding, removing, updating packages, etc.), -update the dependencies in ``setup.cfg``, then run: - -.. code-block:: bash - - ./configure --clean # Remove existing virtual environment - ./configure --init # Create project virtual environment, pull in new dependencies - source venv/bin/activate # Ensure virtual environment is activated - python etc/scripts/gen_requirements.py -s venv/lib/python/site-packages/ # Regenerate requirements.txt - python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ # Regenerate requirements-dev.txt - pip install -r etc/scripts/requirements.txt # Install dependencies needed by etc/scripts/bootstrap.py - python etc/scripts/bootstrap.py -r requirements.txt -r requirements-dev.txt --with-deps # Collect dependency wheels and their ABOUT files - -Ensure that the generated ABOUT files are valid, then take the dependency wheels -and ABOUT files and upload them to thirdparty.aboutcode.org/pypi. +Usage instructions can be found in ``docs/skeleton-usage.rst``. Release Notes ============= diff --git a/docs/skeleton-usage.rst b/docs/skeleton-usage.rst new file mode 100644 index 00000000..7d16259c --- /dev/null +++ b/docs/skeleton-usage.rst @@ -0,0 +1,157 @@ +Usage +===== +A brand new project +------------------- +.. code-block:: bash + + git init my-new-repo + cd my-new-repo + git pull git@github.com:nexB/skeleton + + # Create the new repo on GitHub, then update your remote + git remote set-url origin git@github.com:nexB/your-new-repo.git + +From here, you can make the appropriate changes to the files for your specific project. + +Update an existing project +--------------------------- +.. code-block:: bash + + cd my-existing-project + git remote add skeleton git@github.com:nexB/skeleton + git fetch skeleton + git merge skeleton/main --allow-unrelated-histories + +This is also the workflow to use when updating the skeleton files in any given repository. + +Customizing +----------- + +You typically want to perform these customizations: + +- remove or update the src/README.rst and tests/README.rst files +- set project info and dependencies in setup.cfg +- check the configure and configure.bat defaults + +Initializing a project +---------------------- + +All projects using the skeleton will be expected to pull all of it dependencies +from thirdparty.aboutcode.org/pypi or the local thirdparty directory, using +requirements.txt and/or requirements-dev.txt to determine what version of a +package to collect. By default, PyPI will not be used to find and collect +packages from. + +In the case where we are starting a new project where we do not have +requirements.txt and requirements-dev.txt and whose dependencies are not yet on +thirdparty.aboutcode.org/pypi, we run the following command after adding and +customizing the skeleton files to your project: + +.. code-block:: bash + + ./configure --init + +This will initialize the virtual environment for the project, pull in the +dependencies from PyPI and add them to the virtual environment. + +Generating requirements.txt and requirements-dev.txt +---------------------------------------------------- + +After the project has been initialized, we can generate the requirements.txt and +requirements-dev.txt files. + +Ensure the virtual environment is enabled. + +.. code-block:: bash + + source venv/bin/activate + +To generate requirements.txt: + +.. code-block:: bash + + python etc/scripts/gen_requirements.py -s venv/lib/python/site-packages/ + +Replace \ with the version number of the Python being used, for example: ``venv/lib/python3.6/site-packages/`` + +To generate requirements-dev.txt after requirements.txt has been generated: + +.. code-block:: bash + ./configure --init --dev + python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ + +Note: on Windows, the ``site-packages`` directory is located at ``venv\Lib\site-packages\`` + +.. code-block:: bash + + python .\\etc\\scripts\\gen_requirements.py -s .\\venv\\Lib\\site-packages\\ + .\configure --init --dev + python .\\etc\\scripts\\gen_requirements_dev.py -s .\\venv\\Lib\\site-packages\\ + +Collecting and generating ABOUT files for dependencies +------------------------------------------------------ + +Ensure that the dependencies used by ``etc/scripts/bootstrap.py`` are installed: + +.. code-block:: bash + + pip install -r etc/scripts/requirements.txt + +Once we have requirements.txt and requirements-dev.txt, we can fetch the project +dependencies as wheels and generate ABOUT files for them: + +.. code-block:: bash + + python etc/scripts/bootstrap.py -r requirements.txt -r requirements-dev.txt --with-deps + +There may be issues with the generated ABOUT files, which will have to be +corrected. You can check to see if your corrections are valid by running: + +.. code-block:: bash + + python etc/scripts/check_thirdparty.py -d thirdparty + +Once the wheels are collected and the ABOUT files are generated and correct, +upload them to thirdparty.aboutcode.org/pypi by placing the wheels and ABOUT +files from the thirdparty directory to the pypi directory at +https://github.com/nexB/thirdparty-packages + + +Usage after project initialization +---------------------------------- + +Once the ``requirements.txt`` and ``requirements-dev.txt`` have been generated +and the project dependencies and their ABOUT files have been uploaded to +thirdparty.aboutcode.org/pypi, you can configure the project without using the +``--init`` option. + +If the virtual env for the project becomes polluted, or you would like to remove +it, use the ``--clean`` option: + +.. code-block:: bash + + ./configure --clean + +Then you can run ``./configure`` again to set up the project virtual environment. + +To set up the project for development use: + +.. code-block:: bash + + ./configure --dev + +To update the project dependencies (adding, removing, updating packages, etc.), +update the dependencies in ``setup.cfg``, then run: + +.. code-block:: bash + + ./configure --clean # Remove existing virtual environment + ./configure --init # Create project virtual environment, pull in new dependencies + source venv/bin/activate # Ensure virtual environment is activated + python etc/scripts/gen_requirements.py -s venv/lib/python/site-packages/ # Regenerate requirements.txt + python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ # Regenerate requirements-dev.txt + pip install -r etc/scripts/requirements.txt # Install dependencies needed by etc/scripts/bootstrap.py + python etc/scripts/bootstrap.py -r requirements.txt -r requirements-dev.txt --with-deps # Collect dependency wheels and their ABOUT files + +Ensure that the generated ABOUT files are valid, then take the dependency wheels +and ABOUT files and upload them to thirdparty.aboutcode.org/pypi. diff --git a/etc/scripts/README.rst b/etc/scripts/README.rst new file mode 100755 index 00000000..4cb6ec7d --- /dev/null +++ b/etc/scripts/README.rst @@ -0,0 +1,147 @@ +This directory contains the tools to: + +- manage a directory of thirdparty Python package source, wheels and metadata: + pin, build, update, document and publish to a PyPI-like repo (GitHub release) + +- build and publish scancode releases as wheel, sources and OS-specific bundles. + + +NOTE: These are tested to run ONLY on Linux. + + +Thirdparty packages management scripts +====================================== + +Pre-requisites +-------------- + +* There are two run "modes": + + * To generate or update pip requirement files, you need to start with a clean + virtualenv as instructed below (This is to avoid injecting requirements + specific to the tools here in the main requirements). + + * For other usages, the tools here can run either in their own isolated + virtualenv best or in the the main configured development virtualenv. + These requireements need to be installed:: + + pip install --requirement etc/release/requirements.txt + +TODO: we need to pin the versions of these tools + + + +Generate or update pip requirement files +---------------------------------------- + +Scripts +~~~~~~~ + +**gen_requirements.py**: create/update requirements files from currently + installed requirements. + +**gen_requirements_dev.py** does the same but can subtract the main requirements + to get extra requirements used in only development. + + +Usage +~~~~~ + +The sequence of commands to run are: + + +* Start with these to generate the main pip requirements file:: + + ./configure --clean + ./configure + python etc/release/gen_requirements.py --site-packages-dir + +* You can optionally install or update extra main requirements after the + ./configure step such that these are included in the generated main requirements. + +* Optionally, generate a development pip requirements file by running these:: + + ./configure --clean + ./configure --dev + python etc/release/gen_requirements_dev.py --site-packages-dir + +* You can optionally install or update extra dev requirements after the + ./configure step such that these are included in the generated dev + requirements. + +Notes: we generate development requirements after the main as this step requires +the main requirements.txt to be up-to-date first. See **gen_requirements.py and +gen_requirements_dev.py** --help for details. + +Note: this does NOT hash requirements for now. + +Note: Be aware that if you are using "conditional" requirements (e.g. only for +OS or Python versions) in setup.py/setp.cfg/requirements.txt as these are NOT +yet supported. + + +Populate a thirdparty directory with wheels, sources, .ABOUT and license files +------------------------------------------------------------------------------ + +Scripts +~~~~~~~ + +* **fetch_requirements.py** will fetch package wheels, their ABOUT, LICENSE and + NOTICE files to populate a local a thirdparty directory strictly from our + remote repo and using only pinned packages listed in one or more pip + requirements file(s). Fetch only requirements for specific python versions and + operating systems. Optionally fetch the corresponding source distributions. + +* **publish_files.py** will upload/sync a thirdparty directory of files to our + remote repo. Requires a GitHub personal access token. + +* **build_wheels.py** will build a package binary wheel for multiple OS and + python versions. Optionally wheels that contain native code are built + remotely. Dependent wheels are optionally included. Requires Azure credentials + and tokens if building wheels remotely on multiple operatin systems. + +* **fix_thirdparty.py** will fix a thirdparty directory with a best effort to + add missing wheels, sources archives, create or fetch or fix .ABOUT, .NOTICE + and .LICENSE files. Requires Azure credentials and tokens if requesting the + build of missing wheels remotely on multiple operatin systems. + +* **check_thirdparty.py** will check a thirdparty directory for errors. + +* **bootstrap.py** will bootstrap a thirdparty directory from a requirements + file(s) to add or build missing wheels, sources archives and create .ABOUT, + .NOTICE and .LICENSE files. Requires Azure credentials and tokens if + requesting the build of missing wheels remotely on multiple operatin systems. + + + +Usage +~~~~~ + +See each command line --help option for details. + +* (TODO) **add_package.py** will add or update a Python package including wheels, + sources and ABOUT files and this for multiple Python version and OSes(for use + with upload_packages.py afterwards) You will need an Azure personal access + token for buidling binaries and an optional DejaCode API key to post and fetch + new package versions there. TODO: explain how we use romp + + +Upgrade virtualenv app +---------------------- + +The bundled virtualenv.pyz has to be upgraded by hand and is stored under +etc/thirdparty + +* Fetch https://github.com/pypa/get-virtualenv/raw//public/virtualenv.pyz + for instance https://github.com/pypa/get-virtualenv/raw/20.2.2/public/virtualenv.pyz + and save to thirdparty and update the ABOUT and LICENSE files as needed. + +* This virtualenv app contains also bundled pip, wheel and setuptools that are + essential for the installation to work. + + +Other files +=========== + +The other files and scripts are test, support and utility modules used by the +main scripts documented here. From d2bafb9f48995d5d2ea8bded503e30a6c25b2ef7 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Wed, 15 Sep 2021 15:58:48 +0800 Subject: [PATCH 119/160] Fixed #41 - Handled encoding issue when generating ABOUT files Signed-off-by: Chin Yeung Li --- etc/scripts/utils_thirdparty.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 0ebf6b24..d77afc39 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -693,7 +693,8 @@ def save_if_modified(location, content): return False if TRACE: print(f'Saving ABOUT (and NOTICE) files for: {self}') - with open(location, 'w') as fo: + wmode = 'wb' if isinstance(content, bytes) else 'w' + with open(location, wmode, encoding="utf-8") as fo: fo.write(content) return True @@ -725,6 +726,8 @@ def load_about_data(self, about_filename_or_data=None, dest_dir=THIRDPARTY_DIR): if os.path.exists(about_path): with open(about_path) as fi: about_data = saneyaml.load(fi.read()) + if not about_data: + return False else: return False else: @@ -1842,7 +1845,7 @@ def get(self, path_or_url, as_text=True): if not os.path.exists(cached): content = get_file_content(path_or_url=path_or_url, as_text=as_text) wmode = 'w' if as_text else 'wb' - with open(cached, wmode) as fo: + with open(cached, wmode, encoding="utf-8") as fo: fo.write(content) return content else: @@ -1854,7 +1857,7 @@ def put(self, filename, content): """ cached = os.path.join(self.directory, filename) wmode = 'wb' if isinstance(content, bytes) else 'w' - with open(cached, wmode) as fo: + with open(cached, wmode, encoding="utf-8") as fo: fo.write(content) @@ -2362,7 +2365,7 @@ def update_requirements(name, version=None, requirements_file='requirements.txt' updated_name_versions = sorted(updated_name_versions) nvs = '\n'.join(f'{name}=={version}' for name, version in updated_name_versions) - with open(requirements_file, 'w') as fo: + with open(requirements_file, 'w', encoding="utf-8") as fo: fo.write(nvs) @@ -2380,7 +2383,7 @@ def hash_requirements(dest_dir=THIRDPARTY_DIR, requirements_file='requirements.t raise Exception(f'Missing required package {name}=={version}') hashed.append(package.specifier_with_hashes) - with open(requirements_file, 'w') as fo: + with open(requirements_file, 'w', encoding="utf-8") as fo: fo.write('\n'.join(hashed)) ################################################################################ From 95780b400e6f58213829c8b5dd0414f7541b8e66 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 28 Sep 2021 10:34:01 +0200 Subject: [PATCH 120/160] Do not use __init__.py when this is not a module Signed-off-by: Philippe Ombredanne --- src/__init__.py | 0 tests/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/__init__.py delete mode 100644 tests/__init__.py diff --git a/src/__init__.py b/src/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 From 9cca4ef09024c1b9590b45bc30c9965692b5d6a6 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 28 Sep 2021 10:46:42 +0200 Subject: [PATCH 121/160] Move test data in proper place Signed-off-by: Philippe Ombredanne --- tests/{ => test_data}/rpmvercmp.at | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{ => test_data}/rpmvercmp.at (100%) diff --git a/tests/rpmvercmp.at b/tests/test_data/rpmvercmp.at similarity index 100% rename from tests/rpmvercmp.at rename to tests/test_data/rpmvercmp.at From 4b5f38a8b47511b818c61cbfb0442eff05006dcf Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 28 Sep 2021 11:08:36 +0200 Subject: [PATCH 122/160] Add ABOUT and license file for rpmvercmp.at test #2 Signed-off-by: Philippe Ombredanne --- tests/test_data/gpl-2.0.LICENSE | 339 ++++++++++++++++++++++++++++ tests/test_data/rpmvercmp.at.ABOUT | 9 + tests/test_data/rpmvercmp.at.NOTICE | 27 +++ 3 files changed, 375 insertions(+) create mode 100644 tests/test_data/gpl-2.0.LICENSE create mode 100644 tests/test_data/rpmvercmp.at.ABOUT create mode 100644 tests/test_data/rpmvercmp.at.NOTICE diff --git a/tests/test_data/gpl-2.0.LICENSE b/tests/test_data/gpl-2.0.LICENSE new file mode 100644 index 00000000..d159169d --- /dev/null +++ b/tests/test_data/gpl-2.0.LICENSE @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/tests/test_data/rpmvercmp.at.ABOUT b/tests/test_data/rpmvercmp.at.ABOUT new file mode 100644 index 00000000..9738776d --- /dev/null +++ b/tests/test_data/rpmvercmp.at.ABOUT @@ -0,0 +1,9 @@ +about_resource: rpmvercmp.at +description: rpm version comparison tests +license_expression: gpl-2.0 +download_url: https://github.com/rpm-software-management/rpm/blob/04de5e6c854edf4ef4a79b44eac3b20451c88df5/tests/rpmvercmp.at +copyright: Copyright (c) Red Hat Software, Inc. +package_url: pkg:github/rpm-software-management/rpm@04de5e6c854edf4ef4a79b44eac3b20451c88df5#/tests/rpmvercmp.at +notes: this test file does not have an explicit license but is assumed to be GPL-licensed like the rest of RPM +homepage_url: https://github.com/rpm-software-management/rpm +notice_file: rpmvercmp.at.NOTICE \ No newline at end of file diff --git a/tests/test_data/rpmvercmp.at.NOTICE b/tests/test_data/rpmvercmp.at.NOTICE new file mode 100644 index 00000000..0d8df96d --- /dev/null +++ b/tests/test_data/rpmvercmp.at.NOTICE @@ -0,0 +1,27 @@ +Copyright (c) Red Hat Software, Inc. + +RPM was originally written by: + Erik Troan + Marc Ewing + +This test file has been maintained by: + Panu Matilainen @pmatilai + +with contributions by: + Pavlina Varekova @pavlinamv + Igor Gnatenko + + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA \ No newline at end of file From 4fc8379d38e6dfe2e279d516db3a5b6c2393b88f Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 28 Sep 2021 11:36:15 +0200 Subject: [PATCH 123/160] Refine rpm version tests #2 Signed-off-by: Philippe Ombredanne --- tests/test_rpm_vercmp.py | 85 +++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 45 deletions(-) diff --git a/tests/test_rpm_vercmp.py b/tests/test_rpm_vercmp.py index afdcfc93..57e89e16 100644 --- a/tests/test_rpm_vercmp.py +++ b/tests/test_rpm_vercmp.py @@ -1,4 +1,5 @@ # +# Copyright (c) nexB Inc. and others # Copyright (c) SAS Institute Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,36 +15,36 @@ # limitations under the License. # -from __future__ import print_function -from __future__ import unicode_literals +# This test script has been extensively modified from the original + import io -import re import os - -try: - import unittest2 as unittest -except ImportError: - import unittest +import re +import unittest from univers import rpm as vercmp +get_vercmp_test = re.compile( + r"(.*)" r"RPMVERCMP\(" r"([^, ]*) *, *" r"([^, ]*) *, *" r"([^\)]*)" r"\).*" +).match -class ACParser(object): - R_STMT = re.compile(r"(.*)RPMVERCMP\(([^, ]*) *, *([^, ]*) *, *([^\)]*)\).*") - def __init__(self, fobj, with_buggy_comparisons=True): - self.fobj = fobj - self.with_buggy_comparisons = with_buggy_comparisons - - def __iter__(self): - for row in self.fobj: - m = self.R_STMT.match(row) - if not m: - continue - if not self.with_buggy_comparisons: - if m.group(1).startswith("dnl "): - continue - yield m.group(2), m.group(3), int(m.group(4)) +def parse_rpmvercmp_tests(fobj, with_buggy_comparisons=True): + """ + Yield triples (version1, version2, comparison result) describing the RPM + version comparison tests found in the rpmvercmp.at test file-like object at + ``fobj`` Optionally includ "buggy" upstream tests. + """ + for line in fobj: + line = line.strip() + if not line: + continue + if not with_buggy_comparisons and line.startswith("dnl"): + continue + match = get_vercmp_test(line) + if not match: + continue + yield match.group(2), match.group(3), int(match.group(4)) class ParserTest(unittest.TestCase): @@ -56,9 +57,9 @@ def test_parse_without_buggy(self): dnl RPMVERCMP(1a, 1b, -1) """ ) - parser = ACParser(fobj, with_buggy_comparisons=False) - exp = [("1", "2", -1)] - self.assertEqual(exp, list(parser)) + parser = parse_rpmvercmp_tests(fobj, with_buggy_comparisons=False) + expected = [("1", "2", -1)] + assert list(parser) == expected def test_parse_with_buggy(self): fobj = io.StringIO( @@ -69,29 +70,23 @@ def test_parse_with_buggy(self): dnl RPMVERCMP(1a, 1b, -1) """ ) - parser = ACParser(fobj, with_buggy_comparisons=True) - exp = [("1", "2", -1), ("1a", "1b", -1)] - self.assertEqual(exp, list(parser)) + parser = parse_rpmvercmp_tests(fobj, with_buggy_comparisons=True) + expected = [("1", "2", -1), ("1a", "1b", -1)] + assert list(parser) == expected class VersionCompareTest(unittest.TestCase): - TestFile = os.path.join(os.path.dirname(__file__), "test_data", "rpmvercmp.at") + def test_from_rpmtest(self): + test_file = os.path.join(os.path.dirname(__file__), "test_data", "rpmvercmp.at") - def setUp(self): - super(VersionCompareTest, self).setUp() - self.acfobj = io.open(self.TestFile, encoding="utf-8") + with io.open(test_file, encoding="utf-8") as rpmtests: + tests = list(parse_rpmvercmp_tests(rpmtests, with_buggy_comparisons=True)) + + for test_count, (ver1, ver2, expected_comparison) in enumerate(tests, 1): + result = vercmp.vercmp(ver1, ver2) + if result != expected_comparison: + assert result == (expected_comparison, ver1, ver2) - def test_from_rpmtest(self): - parser = ACParser(self.acfobj, with_buggy_comparisons=True) - test_count = 0 - for first, second, exp in parser: - test_count += 1 - ret = vercmp.vercmp(first, second) - try: - self.assertEqual(exp, ret) - except: - print(first, second, exp, ret) - raise # Make sure we still test something, in case the m4 file drops # content this will fail the test - self.assertGreater(test_count, 20) + assert test_count > 20 From 3ca6c3444be7fda6495d041d94946f0067ebba6c Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 28 Sep 2021 11:55:38 +0200 Subject: [PATCH 124/160] Improve origin and license documentation Signed-off-by: Philippe Ombredanne --- src/univers/arch.py | 8 ++++++- src/univers/debian.py | 2 +- src/univers/gentoo.py | 7 +++++- src/univers/maven.py | 24 ++++--------------- src/univers/rpm.py | 18 +++----------- src/univers/utils.py | 13 +--------- src/univers/version_range.py | 13 +--------- src/univers/version_specifier.py | 14 +---------- src/univers/versions.py | 14 ++--------- tests/test_codestyle.py | 13 +--------- tests/test_debian_version.py | 1 + ...est_debian_version_python_deb_pkg_tools.py | 6 ++--- ...bian_version_python_deb_pkg_tools.py.ABOUT | 2 +- tests/test_gentoo.py | 1 + 14 files changed, 32 insertions(+), 104 deletions(-) diff --git a/src/univers/arch.py b/src/univers/arch.py index cbde1f6c..25611eac 100644 --- a/src/univers/arch.py +++ b/src/univers/arch.py @@ -1,9 +1,15 @@ +# # Copyright 2016-2020 Christoph Reiter # SPDX-License-Identifier: MIT +# Version comparision utility extracted from msys2 and further stripped down. import re from itertools import zip_longest -from typing import List, Tuple, Optional, Dict, Set +from typing import Dict +from typing import List +from typing import Optional +from typing import Set +from typing import Tuple def vercmp(v1: str, v2: str) -> int: diff --git a/src/univers/debian.py b/src/univers/debian.py index d2859d3e..c1ae1b8f 100644 --- a/src/univers/debian.py +++ b/src/univers/debian.py @@ -1,6 +1,6 @@ # # Copyright (c) nexB Inc. and others. -# http://nexb.com and https://github.com/nexB/debian_inspector/ +# Exatrcted from http://nexb.com and https://github.com/nexB/debian_inspector/ # Copyright (c) Peter Odding # Author: Peter Odding # URL: https://github.com/xolox/python-deb-pkg-tools diff --git a/src/univers/gentoo.py b/src/univers/gentoo.py index d1aca5c1..3a0ca964 100644 --- a/src/univers/gentoo.py +++ b/src/univers/gentoo.py @@ -1,4 +1,7 @@ -"""gentoo ebuild specific base package class""" +# +# Copyright (c) 2006-2019, pkgcore contributors +# SPDX-License-Identifier: BSD-3-Clause +# Version comparision utility extracted from pkgcore and further stripped down. import re @@ -8,6 +11,8 @@ suffix_value = {"pre": -2, "p": 1, "alpha": -4, "beta": -3, "rc": -1} +"""gentoo ebuild specific base package class""" + # while the package section looks fugly, there is a reason for it- # to prevent version chunks from showing up in the package diff --git a/src/univers/maven.py b/src/univers/maven.py index 7e6af3ce..51ec3201 100644 --- a/src/univers/maven.py +++ b/src/univers/maven.py @@ -1,29 +1,13 @@ # # Copyright (c) SAS Institute Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# TODO: Add something to track origin and the changes. - -""" -Versioning of artifacts -""" +# SPDX-License-Identifier: Apache-2.0 +# Version comparision utility extracted from pymaven and further stripped down +# and significantly modified from the original at pymaven import functools - from itertools import zip_longest + # TODO: Use common exceptions with other modules class VersionRangeParseError(ValueError): pass diff --git a/src/univers/rpm.py b/src/univers/rpm.py index 65351a97..61f9fea9 100644 --- a/src/univers/rpm.py +++ b/src/univers/rpm.py @@ -1,21 +1,9 @@ # # Copyright (c) SAS Institute Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +# SPDX-License-Identifier: Apache-2.0 +# Version comparision utility extracted from python-rpm-vercmp and further +# stripped down and significantly modified from the original at python-rpm-vercmp -from __future__ import print_function -from __future__ import unicode_literals import re diff --git a/src/univers/utils.py b/src/univers/utils.py index 6d5b226d..5067e5e0 100644 --- a/src/univers/utils.py +++ b/src/univers/utils.py @@ -1,19 +1,8 @@ +# # Copyright (c) nexB Inc. and others. # SPDX-License-Identifier: Apache-2.0 # # Visit https://aboutcode.org and https://github.com/nexB/ for support and download. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. def remove_spaces(string): diff --git a/src/univers/version_range.py b/src/univers/version_range.py index 4268c068..69f829c1 100644 --- a/src/univers/version_range.py +++ b/src/univers/version_range.py @@ -1,19 +1,8 @@ +# # Copyright (c) nexB Inc. and others. # SPDX-License-Identifier: Apache-2.0 # # Visit https://aboutcode.org and https://github.com/nexB/ for support and download. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import operator as operator_module diff --git a/src/univers/version_specifier.py b/src/univers/version_specifier.py index 4857afa8..a4e6b610 100644 --- a/src/univers/version_specifier.py +++ b/src/univers/version_specifier.py @@ -1,19 +1,7 @@ +# # Copyright (c) nexB Inc. and others. # SPDX-License-Identifier: Apache-2.0 -# # Visit https://aboutcode.org and https://github.com/nexB/ for support and download. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. from semantic_version import Version diff --git a/src/univers/versions.py b/src/univers/versions.py index e4260202..3e54f452 100644 --- a/src/univers/versions.py +++ b/src/univers/versions.py @@ -1,19 +1,9 @@ +# # Copyright (c) nexB Inc. and others. # SPDX-License-Identifier: Apache-2.0 # # Visit https://aboutcode.org and https://github.com/nexB/ for support and download. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. + import re import attr diff --git a/tests/test_codestyle.py b/tests/test_codestyle.py index d5edc830..c0e4e703 100644 --- a/tests/test_codestyle.py +++ b/tests/test_codestyle.py @@ -1,19 +1,8 @@ +# # Copyright (c) nexB Inc. and others. # SPDX-License-Identifier: Apache-2.0 # # Visit https://aboutcode.org and https://github.com/nexB/ for support and download. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import subprocess import unittest diff --git a/tests/test_debian_version.py b/tests/test_debian_version.py index ca5309e4..7cab968e 100644 --- a/tests/test_debian_version.py +++ b/tests/test_debian_version.py @@ -7,6 +7,7 @@ # http://nexb.com and https://github.com/nexB/debian_inspector/ # SPDX-License-Identifier: Apache-2.0 +# this has been significantly modified from the original from unittest import TestCase diff --git a/tests/test_debian_version_python_deb_pkg_tools.py b/tests/test_debian_version_python_deb_pkg_tools.py index e7b0089d..2bd01556 100644 --- a/tests/test_debian_version_python_deb_pkg_tools.py +++ b/tests/test_debian_version_python_deb_pkg_tools.py @@ -1,11 +1,9 @@ -# Debian packaging tools: Automated tests. # -# Author: Peter Odding -# Last Change: March 9, 2021 +# Copyright (c) Peter Odding # URL: https://github.com/xolox/python-deb-pkg-tools - # SPDX-License-Identifier: MIT + from unittest import TestCase from univers.debian import Version diff --git a/tests/test_debian_version_python_deb_pkg_tools.py.ABOUT b/tests/test_debian_version_python_deb_pkg_tools.py.ABOUT index 9be39c3e..4b8e19d0 100644 --- a/tests/test_debian_version_python_deb_pkg_tools.py.ABOUT +++ b/tests/test_debian_version_python_deb_pkg_tools.py.ABOUT @@ -2,7 +2,7 @@ about_resource: test_version_python_deb_pkg_tools.py name: python-debian license_expression: mit download_url: https://raw.githubusercontent.com/xolox/python-deb-pkg-tools/a3d6ef1d82c6342b6a57876fc2360875e033f8f0/deb_pkg_tools/tests.py -copyright: Copyright (C) Peter Odding +copyright: Copyright (c) Peter Odding package_url: pkg:pypi/deb-pkg-tools@8.4#deb_pkg_tools/tests.py notes: this subset of tests has been modified to tests version comparison and parsing homepage_url: https://github.com/xolox/python-deb-pkg-tools \ No newline at end of file diff --git a/tests/test_gentoo.py b/tests/test_gentoo.py index 04e95f32..a18fe2b7 100644 --- a/tests/test_gentoo.py +++ b/tests/test_gentoo.py @@ -1,6 +1,7 @@ # test_vercmp.py -- Portage Unit Testing Functionality # Copyright 2006 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +# this has been significant;ly modified from the original from unittest import TestCase from univers.gentoo import vercmp From b2c31f9d6a89d51ea038a465d83271a912d2addf Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 28 Sep 2021 12:18:26 +0200 Subject: [PATCH 125/160] Move files to target location Signed-off-by: Philippe Ombredanne --- src/{pkgcore/ebuild/cpv.py => univers/gentoo.py} | 0 tests/{ebuild/test_cpv.py => test_gentoo_pkgcore.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/{pkgcore/ebuild/cpv.py => univers/gentoo.py} (100%) rename tests/{ebuild/test_cpv.py => test_gentoo_pkgcore.py} (100%) diff --git a/src/pkgcore/ebuild/cpv.py b/src/univers/gentoo.py similarity index 100% rename from src/pkgcore/ebuild/cpv.py rename to src/univers/gentoo.py diff --git a/tests/ebuild/test_cpv.py b/tests/test_gentoo_pkgcore.py similarity index 100% rename from tests/ebuild/test_cpv.py rename to tests/test_gentoo_pkgcore.py From 8b48bf91dbe1f9009354ea276ffec5bc9dcd9f46 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 28 Sep 2021 12:19:19 +0200 Subject: [PATCH 126/160] Streamline notices and add missing license doc Aslo perform minor test refactoring Signed-off-by: Philippe Ombredanne --- src/univers/gentoo.py.ABOUT | 3 +- tests/apache-2.0.LICENSE | 201 +++++++++++++++++++ tests/gpl-2.0.LICENSE | 339 ++++++++++++++++++++++++++++++++ tests/test_gentoo.py | 5 +- tests/test_gentoo.py.ABOUT | 10 + tests/test_gentoo.py.NOTICE | 12 ++ tests/test_maven_version.py | 23 +-- tests/test_pacman_vercmp.py | 35 ++-- tests/test_pypi_version.py | 14 +- tests/test_rpm_vercmp.py | 17 +- tests/test_version_range.py | 16 +- tests/test_version_specifier.py | 13 +- 12 files changed, 591 insertions(+), 97 deletions(-) create mode 100644 tests/apache-2.0.LICENSE create mode 100644 tests/gpl-2.0.LICENSE create mode 100644 tests/test_gentoo.py.ABOUT create mode 100644 tests/test_gentoo.py.NOTICE diff --git a/src/univers/gentoo.py.ABOUT b/src/univers/gentoo.py.ABOUT index 3c078644..7dccf0b2 100644 --- a/src/univers/gentoo.py.ABOUT +++ b/src/univers/gentoo.py.ABOUT @@ -6,7 +6,6 @@ copyright: | license_expression: BSD-3-Clause homepage_url: https://github.com/pkgcore/pkgcore/blob/master/src/pkgcore/ebuild/cpv.py -notes: | - The version comparision utility is extracted from pkgcore and further stripped down. +notes: The version comparision utility is extracted from pkgcore and further stripped down. notice_file: gentoo.py.NOTICE \ No newline at end of file diff --git a/tests/apache-2.0.LICENSE b/tests/apache-2.0.LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/tests/apache-2.0.LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tests/gpl-2.0.LICENSE b/tests/gpl-2.0.LICENSE new file mode 100644 index 00000000..d159169d --- /dev/null +++ b/tests/gpl-2.0.LICENSE @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/tests/test_gentoo.py b/tests/test_gentoo.py index a18fe2b7..d5a84313 100644 --- a/tests/test_gentoo.py +++ b/tests/test_gentoo.py @@ -1,7 +1,8 @@ # test_vercmp.py -- Portage Unit Testing Functionality # Copyright 2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# this has been significant;ly modified from the original +# SPDX-License-Identifier: GPL-2.0-only +# this has been significantly modified from the original + from unittest import TestCase from univers.gentoo import vercmp diff --git a/tests/test_gentoo.py.ABOUT b/tests/test_gentoo.py.ABOUT new file mode 100644 index 00000000..5dda22d8 --- /dev/null +++ b/tests/test_gentoo.py.ABOUT @@ -0,0 +1,10 @@ +about_resource: test_gentoo.py +package_url: pkg:ebuild/sys-apps/portage@3.0.13#tests/versions/test_vercmp.py +copyright: Copyright 1998-2016 Gentoo Foundation + +license_expression: GPL-2.0-only +homepage_url: https://gitweb.gentoo.org/proj/portage.git/ + +notes: version comparison tests extracted from portage and further stripped down. + +notice_file: gentoo.py.NOTICE \ No newline at end of file diff --git a/tests/test_gentoo.py.NOTICE b/tests/test_gentoo.py.NOTICE new file mode 100644 index 00000000..e9ed1a01 --- /dev/null +++ b/tests/test_gentoo.py.NOTICE @@ -0,0 +1,12 @@ ++about_resource: test_gentoo.py ++package_url: pkg:ebuild/sys-apps/portage@3.0.13#tests/versions/test_vercmp.py ++copyright: | ++ Copyright 1998-2016 Gentoo Foundation ++ ++license_expression: GPL-2.0-only ++homepage_url: https://gitweb.gentoo.org/proj/portage.git/ ++ ++notes: | ++ The version comparison utility is extracted from portage and further stripped down. ++ ++notice_file: gentoo.py.NOTICE \ No newline at end of file diff --git a/tests/test_maven_version.py b/tests/test_maven_version.py index eab3d6df..f5d36680 100644 --- a/tests/test_maven_version.py +++ b/tests/test_maven_version.py @@ -1,23 +1,9 @@ # # Copyright (c) SAS Institute Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +# SPDX-License-Identifier: Apache-2.0 +# this has been significantly modified from the original# -""" -Test version handling -""" import unittest from univers.maven import Version @@ -136,7 +122,7 @@ def test_compare(self): assert 1 < r1 def test_string_repr(self): - for input in ( + for test in ( "[1.0]", "[1.0,)", "[1.0,2.0]", @@ -146,8 +132,7 @@ def test_string_repr(self): "[,2.0)", "(,2.0)", ): - actual = str(Restriction(input)) - assert input == actual, "Restriction(%s) == %s, wanted %s" % (input, actual, input) + assert str(Restriction(test)) == test class TestVersion(unittest.TestCase): diff --git a/tests/test_pacman_vercmp.py b/tests/test_pacman_vercmp.py index d38c7b9f..e0fd6aa8 100644 --- a/tests/test_pacman_vercmp.py +++ b/tests/test_pacman_vercmp.py @@ -1,17 +1,20 @@ +# Copyright (c) 2009-2020 by Pacman Development Team +# Copyright (c) 2008 by Dan McGee +# SPDX-License-Identifier: Apache-2.0 +# this has been significantly modified from the original + from univers.versions import ArchVersion -# all similar length, no pkgrel + def test_same_length(): assert ArchVersion("1.5.0") == ArchVersion("1.5.0") assert ArchVersion("1.5.1") > ArchVersion("1.5.0") -# mixed length def test_mixed_length(): assert ArchVersion("1.5.1") > ArchVersion("1.5") -# with pkgrel, simple def test_with_pkgrel_same_length(): assert ArchVersion("1.5.0-1") == ArchVersion("1.5.0-1") assert ArchVersion("1.5.0-1") < ArchVersion("1.5.0-2") @@ -19,14 +22,12 @@ def test_with_pkgrel_same_length(): assert ArchVersion("1.5.0-2") < ArchVersion("1.5.1-1") -# going crazy? alpha-dotted versions -def test_alpha_dotted(): +def test_alpha_dotted_versions(): assert ArchVersion("1.5.a") > ArchVersion("1.5") assert ArchVersion("1.5.b") > ArchVersion("1.5.a") assert ArchVersion("1.5.1") > ArchVersion("1.5.b") -# epoch included version comparisons def test_with_epoch(): assert ArchVersion("0:1.0") == ArchVersion("0:1.0") assert ArchVersion("0:1.0") < ArchVersion("0:1.1") @@ -35,13 +36,11 @@ def test_with_epoch(): assert ArchVersion("1:1.0") < ArchVersion("2:1.1") -# epoch + sometimes present pkgrel def test_with_epoch_mixed_pkgrel(): assert ArchVersion("1:1.0") > ArchVersion("0:1.0-1") assert ArchVersion("1:1.0-1") > ArchVersion("0:1.1-1") -# epoch included on one version def test_with_only_one_version_with_epoch(): assert ArchVersion("0:1.0") == ArchVersion("1.0") assert ArchVersion("0:1.0") < ArchVersion("1.1") @@ -51,13 +50,11 @@ def test_with_only_one_version_with_epoch(): assert ArchVersion("1:1.1") > ArchVersion("1.1") -# alpha dots and dashes -def test_alpha_dot_dash(): +def test_alpha_dot_and_dashes(): assert ArchVersion("1.5.b-1") == ArchVersion("1.5.b") assert ArchVersion("1.5-1") < ArchVersion("1.5.b") -# # same/similar content, differing separators # def test_same_content_different_separators(): # assert ArchVersion("2.0") == ArchVersion("2_0") # assert ArchVersion("2.0_a") == ArchVersion("2_0.a") @@ -65,15 +62,13 @@ def test_alpha_dot_dash(): # assert ArchVersion("2___a") == ArchVersion("2_a") -# with pkgrel, mixed lengths def test_with_pkgrel_mixed_length(): assert ArchVersion("1.5-1") < ArchVersion("1.5.1-1") assert ArchVersion("1.5-2") < ArchVersion("1.5.1-1") assert ArchVersion("1.5-2") < ArchVersion("1.5.1-2") -# mixed pkgrel inclusion -def test_with_pkgrel_mixed(): +def test_with_mixed_pkgrel_inclusion(): assert ArchVersion("1.5") == ArchVersion("1.5-1") assert ArchVersion("1.5-1") == ArchVersion("1.5") assert ArchVersion("1.1-1") == ArchVersion("1.1") @@ -81,13 +76,13 @@ def test_with_pkgrel_mixed(): assert ArchVersion("1.1-1") > ArchVersion("1.0") -# alphanumeric versions -assert ArchVersion("1.5b-1") < ArchVersion("1.5-1") -assert ArchVersion("1.5b ") < ArchVersion("1.5 ") -assert ArchVersion("1.5b-1") < ArchVersion("1.5 ") -assert ArchVersion("1.5b ") < ArchVersion("1.5.1") +def test_alphanumeric_versions(): + assert ArchVersion("1.5b-1") < ArchVersion("1.5-1") + assert ArchVersion("1.5b ") < ArchVersion("1.5 ") + assert ArchVersion("1.5b-1") < ArchVersion("1.5 ") + assert ArchVersion("1.5b ") < ArchVersion("1.5.1") + -# from the manpage def test_manpage_cases(): assert ArchVersion("1.0a") < ArchVersion("1.0alpha") assert ArchVersion("1.0alpha") < ArchVersion("1.0b") diff --git a/tests/test_pypi_version.py b/tests/test_pypi_version.py index 72947cd5..c828dbcd 100644 --- a/tests/test_pypi_version.py +++ b/tests/test_pypi_version.py @@ -3,18 +3,6 @@ # SPDX-License-Identifier: Apache-2.0 # # Visit https://aboutcode.org and https://github.com/nexB/ for support and download. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. from unittest import TestCase from packaging import version @@ -30,4 +18,4 @@ def test_constructor(self): self.assertRaises(versions.InvalidVersion, versions.PYPIVersion, "2.//////") - # Comparision is already tested at https://github.com/pypa/packaging/blob/main/tests/test_version.py + # comparison is already tested at https://github.com/pypa/packaging/blob/main/tests/test_version.py diff --git a/tests/test_rpm_vercmp.py b/tests/test_rpm_vercmp.py index 57e89e16..96a84f8f 100644 --- a/tests/test_rpm_vercmp.py +++ b/tests/test_rpm_vercmp.py @@ -1,21 +1,8 @@ # # Copyright (c) nexB Inc. and others # Copyright (c) SAS Institute Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# This test script has been extensively modified from the original +# SPDX-License-Identifier: Apache-2.0 +# this has been significantly modified from the original import io import os diff --git a/tests/test_version_range.py b/tests/test_version_range.py index c80608c9..338b0522 100644 --- a/tests/test_version_range.py +++ b/tests/test_version_range.py @@ -3,18 +3,6 @@ # SPDX-License-Identifier: Apache-2.0 # # Visit https://aboutcode.org and https://github.com/nexB/ for support and download. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import pytest @@ -43,7 +31,7 @@ ("1.0.0b1", ">=1.0.0b1", True), ], ) -def test_pypi_comparision(version, spec, result): +def test_pypi_comparison(version, spec, result): version_class = version_class_by_scheme["pypi"] version_object = version_class(version) assert (version_object in VersionRange(spec, "pypi")) == result @@ -62,7 +50,7 @@ def test_pypi_comparision(version, spec, result): ("1.0.0+20130313144700", "=1.0.0+9999999999", False), ], ) -def test_semver_comparision(version, spec, result): +def test_semver_comparison(version, spec, result): version_class = version_class_by_scheme["semver"] version_object = version_class(version) assert (version_object in VersionRange(spec, "semver")) == result diff --git a/tests/test_version_specifier.py b/tests/test_version_specifier.py index 914b03fd..f3c8e6d1 100644 --- a/tests/test_version_specifier.py +++ b/tests/test_version_specifier.py @@ -3,18 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # # Visit https://aboutcode.org and https://github.com/nexB/ for support and download. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. + from unittest import TestCase From b178596032c7ceea11ab32725a73dd72023f25ea Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 28 Sep 2021 14:50:43 +0200 Subject: [PATCH 127/160] Add missing license info Signed-off-by: Philippe Ombredanne --- src/univers/bsd-new.LICENSE | 24 +++++ tests/bsd-new.LICENSE | 24 +++++ tests/test_gentoo_pkgcore.py | 149 +++++++++++++++++----------- tests/test_gentoo_pkgcore.py.ABOUT | 7 ++ tests/test_gentoo_pkgcore.py.NOTICE | 23 +++++ 5 files changed, 171 insertions(+), 56 deletions(-) create mode 100644 src/univers/bsd-new.LICENSE create mode 100644 tests/bsd-new.LICENSE create mode 100644 tests/test_gentoo_pkgcore.py.ABOUT create mode 100644 tests/test_gentoo_pkgcore.py.NOTICE diff --git a/src/univers/bsd-new.LICENSE b/src/univers/bsd-new.LICENSE new file mode 100644 index 00000000..cb6a877f --- /dev/null +++ b/src/univers/bsd-new.LICENSE @@ -0,0 +1,24 @@ +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +Neither the name of the ORGANIZATION nor the names of its contributors may be +used to endorse or promote products derived from this software without specific +prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/tests/bsd-new.LICENSE b/tests/bsd-new.LICENSE new file mode 100644 index 00000000..cb6a877f --- /dev/null +++ b/tests/bsd-new.LICENSE @@ -0,0 +1,24 @@ +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +Neither the name of the ORGANIZATION nor the names of its contributors may be +used to endorse or promote products derived from this software without specific +prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/tests/test_gentoo_pkgcore.py b/tests/test_gentoo_pkgcore.py index 7bd1f67d..0c215661 100644 --- a/tests/test_gentoo_pkgcore.py +++ b/tests/test_gentoo_pkgcore.py @@ -1,8 +1,13 @@ +# +# Copyright (c) 2006-2019, pkgcore contributors +# SPDX-License-Identifier: BSD-3-Clause +# Version comparision utility extracted from pkgcore and further stripped down. + from random import shuffle import pytest -from pkgcore.ebuild import cpv -from snakeoil.compatibility import cmp + +from univers.gentoo import vercmp def generate_misc_sufs(): @@ -10,39 +15,68 @@ def generate_misc_sufs(): suf_nums = list(range(100)) shuffle(suf_nums) - good_sufs = (simple_good_sufs + [f"{x}{suf_nums.pop()}" for x in simple_good_sufs]) + good_sufs = simple_good_sufs + [f"{x}{suf_nums.pop()}" for x in simple_good_sufs] l = len(good_sufs) - good_sufs = good_sufs + [ - good_sufs[x] + good_sufs[l - x - 1] for x in range(l)] + good_sufs = good_sufs + [good_sufs[x] + good_sufs[l - x - 1] for x in range(l)] - bad_sufs = ["_a", "_9", "_"] + [x+" " for x in simple_good_sufs] + bad_sufs = ["_a", "_9", "_"] + [x + " " for x in simple_good_sufs] return good_sufs, bad_sufs class TestCPV: good_cats = ( - "dev-util", "dev+", "dev-util+", "DEV-UTIL", "aaa0", - "aaa-0", "multi/depth", "cross-dev_idiot.hacks-suck", "a", - "foo---", "multi--hyphen") - bad_cats = (".util", "_dev", "", "dev-util ", "multi//depth") - good_pkgs = ("diffball", "a9", "a9+", "a-100dpi", "diff-mode-", - "multi--hyphen", "timidity--", "frob---", "diffball-9-") - bad_pkgs = ("diffball ", "diffball-9", "a-3D", "-df", "+dfa", - "timidity--9f", "ormaybe---13_beta") - - good_cp = ( - "bbb-9/foon", "dev-util/diffball", "dev-util/diffball-a9", - "dev-ut-asdf/emacs-cvs", "xfce-base/xfce4", "bah/f-100dpi", - "dev-util/diffball-blah-monkeys", "virtual/7z") + "dev-util", + "dev+", + "dev-util+", + "DEV-UTIL", + "aaa0", + "aaa-0", + "multi/depth", + "cross-dev_idiot.hacks-suck", + "a", + "foo---", + "multi--hyphen", + ) + bad_cats = (".util", "_dev", "", "dev-util ", "multi//depth") + good_pkgs = ( + "diffball", + "a9", + "a9+", + "a-100dpi", + "diff-mode-", + "multi--hyphen", + "timidity--", + "frob---", + "diffball-9-", + ) + bad_pkgs = ( + "diffball ", + "diffball-9", + "a-3D", + "-df", + "+dfa", + "timidity--9f", + "ormaybe---13_beta", + ) + + good_cp = ( + "bbb-9/foon", + "dev-util/diffball", + "dev-util/diffball-a9", + "dev-ut-asdf/emacs-cvs", + "xfce-base/xfce4", + "bah/f-100dpi", + "dev-util/diffball-blah-monkeys", + "virtual/7z", + ) good_vers = ("1", "2.3.4", "2.3.4a", "02.3", "2.03", "3d", "3D") - bad_vers = ("2.3a.4", "2.a.3", "2.3_", "2.3 ", "2.3.", "cvs.2") + bad_vers = ("2.3a.4", "2.a.3", "2.3_", "2.3 ", "2.3.", "cvs.2") good_sufs, bad_sufs = generate_misc_sufs() - good_revs = ("-r1", "-r300", "-r0", "", - "-r1000000000000000000") + good_revs = ("-r1", "-r300", "-r0", "", "-r1000000000000000000") bad_revs = ("-r", "-ra", "-r", "-R1") testing_secondary_args = False @@ -57,12 +91,14 @@ def make_inst(self, cat, pkg, fullver=""): def test_simple_key(self): with pytest.raises(cpv.InvalidCPV): self.make_inst("da", "ba-3", "3.3") - for src in [[("dev-util", "diffball", "0.7.1"), "dev-util/diffball"], + for src in [ + [("dev-util", "diffball", "0.7.1"), "dev-util/diffball"], ["dev-util/diffball"], ["dev-perl/mod_perl"], ["dev-perl/mod_p"], [("dev-perl", "mod-p", ""), "dev-perl/mod-p"], - ["dev-perl/mod-p-1", "dev-perl/mod-p"],]: + ["dev-perl/mod-p-1", "dev-perl/mod-p"], + ]: if len(src) == 1: key = src[0] else: @@ -72,7 +108,7 @@ def test_simple_key(self): vals = pkgver.rsplit("-", 1) if len(vals) == 1: pkg = pkgver - ver = '' + ver = "" else: pkg, ver = vals else: @@ -89,32 +125,27 @@ def test_init(self): def test_parsing(self): # check for gentoo bug 263787 - self.process_pkg(False, 'app-text', 'foo-123-bar') - self.process_ver(False, 'app-text', 'foo-123-bar', '2.0017a_p', '-r5') + self.process_pkg(False, "app-text", "foo-123-bar") + self.process_ver(False, "app-text", "foo-123-bar", "2.0017a_p", "-r5") with pytest.raises(cpv.InvalidCPV): - cpv.UnversionedCPV('app-text/foo-123') + cpv.UnversionedCPV("app-text/foo-123") for cat_ret, cats in [[False, self.good_cats], [True, self.bad_cats]]: for cat in cats: - for pkg_ret, pkgs in [[False, self.good_pkgs], - [True, self.bad_pkgs]]: + for pkg_ret, pkgs in [[False, self.good_pkgs], [True, self.bad_pkgs]]: for pkg in pkgs: self.process_pkg(cat_ret or pkg_ret, cat, pkg) for cp in self.good_cp: cat, pkg = cp.rsplit("/", 1) - for rev_ret, revs in [[False, self.good_revs], - [True, self.bad_revs]]: + for rev_ret, revs in [[False, self.good_revs], [True, self.bad_revs]]: for rev in revs: - for ver_ret, vers in [[False, self.good_vers], - [True, self.bad_vers]]: + for ver_ret, vers in [[False, self.good_vers], [True, self.bad_vers]]: for ver in vers: - self.process_ver(ver_ret or rev_ret, cat, pkg, - ver, rev) + self.process_ver(ver_ret or rev_ret, cat, pkg, ver, rev) for x in (10, 18, 19, 36, 100): assert cpv.CPV("da", "ba", f"1-r0{'0' * x}").revision == 0 - assert \ - int(cpv.CPV("da", "ba", f"1-r1{'0' * x}1").revision) == int(f"1{'0' * x}1") + assert int(cpv.CPV("da", "ba", f"1-r1{'0' * x}1").revision) == int(f"1{'0' * x}1") def process_pkg(self, ret, cat, pkg): if ret: @@ -162,16 +193,16 @@ def process_ver(self, ret, cat, pkg, ver, rev): for suf in self.bad_sufs: # check standalone. - self.process_suf(True, cat, pkg, ver+suf, rev) + self.process_suf(True, cat, pkg, ver + suf, rev) def process_suf(self, ret, cat, pkg, ver, rev): if ret: with pytest.raises(cpv.InvalidCPV): - self.make_inst(cat, pkg, ver+rev) + self.make_inst(cat, pkg, ver + rev) else: # redundant in light of process_ver... combine these somehow. c = self.make_inst(cat, pkg, ver + rev) - if rev == '' or rev == '-r0': + if rev == "" or rev == "-r0": assert c.cpvstr == f"{cat}/{pkg}-{ver}" assert c.revision == 0 if rev: @@ -199,10 +230,10 @@ def test_cmp(self): if suf == "": sufs = [suf] else: - sufs = [suf, f'{suf}4'] + sufs = [suf, f"{suf}4"] for x in sufs: - cur = vkls(f'{base}{x}{rev}') - assert cur == vkls(f'{base}{x}{rev}') + cur = vkls(f"{base}{x}{rev}") + assert cur == vkls(f"{base}{x}{rev}") if last is not None: assert cur > last @@ -235,9 +266,11 @@ def test_cmp(self): assert vkls("da/ba-6.01.0-r0") == vkls("da/ba-6.01.0-r00") assert vkls("da/ba-6.01.0-r1") == vkls("da/ba-6.01.0-r001") - for v1, v2 in (("1.001000000000000000001", "1.001000000000000000002"), + for v1, v2 in ( + ("1.001000000000000000001", "1.001000000000000000002"), ("1.00100000000", "1.0010000000000000001"), - ("1.01", "1.1")): + ("1.01", "1.1"), + ): assert vkls(f"da/ba-{v2}") > vkls(f"da/ba-{v1}") for x in (18, 36, 100): @@ -248,14 +281,17 @@ def test_cmp(self): s = "0" * x assert vkls(f"da/ba-1-r10{s}1") > vkls(f"da/ba-1-r1{s}1") - assert vkls('sys-apps/net-tools-1.60_p2010081516093') > \ - vkls('sys-apps/net-tools-1.60_p2009072801401') + assert vkls("sys-apps/net-tools-1.60_p2010081516093") > vkls( + "sys-apps/net-tools-1.60_p2009072801401" + ) - assert vkls('sys-apps/net-tools-1.60_p20100815160931') > \ - vkls('sys-apps/net-tools-1.60_p20090728014017') + assert vkls("sys-apps/net-tools-1.60_p20100815160931") > vkls( + "sys-apps/net-tools-1.60_p20090728014017" + ) - assert vkls('sys-apps/net-tools-1.60_p20100815160931') > \ - vkls('sys-apps/net-tools-1.60_p20090728014017-r1') + assert vkls("sys-apps/net-tools-1.60_p20100815160931") > vkls( + "sys-apps/net-tools-1.60_p20090728014017-r1" + ) # Regression test: python does comparison slightly differently # if the classes do not match exactly (it prefers rich @@ -267,8 +303,9 @@ class DummySubclass(cpv.CPV): assert DummySubclass("da/ba-6.0_alpha0", versioned=True) == vkls("da/ba-6.0_alpha") assert DummySubclass("da/ba-6.0", versioned=True) != "foon" - assert DummySubclass("da/ba-6.0", versioned=True) == \ - DummySubclass("da/ba-6.0-r0", versioned=True) + assert DummySubclass("da/ba-6.0", versioned=True) == DummySubclass( + "da/ba-6.0-r0", versioned=True + ) def test_no_init(self): """Test if the cpv is in a somewhat sane state if __init__ fails. @@ -280,10 +317,10 @@ def test_no_init(self): uninited = cpv.CPV.__new__(cpv.CPV) broken = cpv.CPV.__new__(cpv.CPV) with pytest.raises(cpv.InvalidCPV): - broken.__init__('broken', versioned=True) + broken.__init__("broken", versioned=True) for thing in (uninited, broken): # the c version returns None, the py version does not have the attr - getattr(thing, 'cpvstr', None) + getattr(thing, "cpvstr", None) repr(thing) str(thing) # The c version returns a constant, the py version raises diff --git a/tests/test_gentoo_pkgcore.py.ABOUT b/tests/test_gentoo_pkgcore.py.ABOUT new file mode 100644 index 00000000..d116e181 --- /dev/null +++ b/tests/test_gentoo_pkgcore.py.ABOUT @@ -0,0 +1,7 @@ +about_resource: test_gentoo_pkgcore.py +package_url: pkg:pypi/pkgcore@0.11.8#tests/ebuild/test_cpv.py +copyright: Copyright (c) 2006-2019, pkgcore contributors +license_expression: BSD-3-Clause +homepage_url: https://github.com/pkgcore/pkgcore/blob/master/tests/ebuild/test_cpv.py +notes: The version comparision utility code is extracted from pkgcore and further stripped down. +notice_file: gentoo.py.NOTICE \ No newline at end of file diff --git a/tests/test_gentoo_pkgcore.py.NOTICE b/tests/test_gentoo_pkgcore.py.NOTICE new file mode 100644 index 00000000..fbcbbdb9 --- /dev/null +++ b/tests/test_gentoo_pkgcore.py.NOTICE @@ -0,0 +1,23 @@ +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of pkgcore nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file From d48b1ba4f1836c841becd86af720e51306a56ed8 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 28 Sep 2021 14:51:19 +0200 Subject: [PATCH 128/160] Use same shared cmp() utility function everywhere Signed-off-by: Philippe Ombredanne --- src/univers/arch.py | 5 ++--- src/univers/gentoo.py | 21 +++++++++++++-------- src/univers/maven.py | 18 ++---------------- src/univers/utils.py | 24 ++++++++++++++++++++++++ src/univers/versions.py | 2 +- 5 files changed, 42 insertions(+), 28 deletions(-) diff --git a/src/univers/arch.py b/src/univers/arch.py index 25611eac..dee2aa69 100644 --- a/src/univers/arch.py +++ b/src/univers/arch.py @@ -11,11 +11,10 @@ from typing import Set from typing import Tuple +from univers.utils import cmp -def vercmp(v1: str, v2: str) -> int: - def cmp(a: int, b: int) -> int: - return (a > b) - (a < b) +def vercmp(v1: str, v2: str) -> int: def split(v: str) -> Tuple[str, str, Optional[str]]: if ":" in v: e, v = v.split(":", 1) diff --git a/src/univers/gentoo.py b/src/univers/gentoo.py index 3a0ca964..f4edf3d8 100644 --- a/src/univers/gentoo.py +++ b/src/univers/gentoo.py @@ -5,23 +5,24 @@ import re +from univers.utils import cmp + + suffix_regexp = re.compile("^(alpha|beta|rc|pre|p)(\\d*)$") revision_regexp = re.compile(".*(-r\d+)") suffix_value = {"pre": -2, "p": 1, "alpha": -4, "beta": -3, "rc": -1} -"""gentoo ebuild specific base package class""" - -# while the package section looks fugly, there is a reason for it- -# to prevent version chunks from showing up in the package - - -def cmp(a, b): - return bool(a > b) - bool(a < b) +""" +gentoo ebuild version comparison +""" def parse_version_and_revision(version_string): + """ + Return a tuple of (version string, revision int) given a ``version_string``. + """ revision = 0 version = version_string match = revision_regexp.search(version_string) @@ -33,6 +34,10 @@ def parse_version_and_revision(version_string): def vercmp(ver1, ver2): + """ + Compare two versions ``ver1`` and ``ver2`` and return 0, 1, or -1 according to the + cmp semantics + """ ver1, rev1 = parse_version_and_revision(ver1) ver2, rev2 = parse_version_and_revision(ver2) diff --git a/src/univers/maven.py b/src/univers/maven.py index 51ec3201..6abb8d39 100644 --- a/src/univers/maven.py +++ b/src/univers/maven.py @@ -7,6 +7,8 @@ import functools from itertools import zip_longest +from univers.utils import cmp + # TODO: Use common exceptions with other modules class VersionRangeParseError(ValueError): @@ -37,22 +39,6 @@ def list2tuple(l): return tuple(list2tuple(x) if isinstance(x, list) else x for x in l) -def cmp(x, y): - """ - Replacement for built-in funciton cmp that was removed in Python 3 - Compare the two objects x and y and return an integer according to - the outcome. The return value is negative if x < y, zero if x == y - and strictly positive if x > y. - """ - if x is None and y is None: - return 0 - elif x is None: - return -1 - elif y is None: - return 1 - return (x > y) - (x < y) - - @functools.total_ordering class Restriction(object): """Describes a restriction in versioning""" diff --git a/src/univers/utils.py b/src/univers/utils.py index 5067e5e0..74fbbc08 100644 --- a/src/univers/utils.py +++ b/src/univers/utils.py @@ -7,3 +7,27 @@ def remove_spaces(string): return string.replace(" ", "") + + +def cmp(x, y): + """ + Replacement for built-in Python 2 function cmp that was removed in Python 3 + From https://docs.python.org/2/library/functions.html?highlight=cmp#cmp : + + Compare the two objects x and y and return an integer according to the + outcome. The return value is negative if x < y, zero if x == y and + strictly positive if x > y. + """ + if x == y: + return 0 + elif x is None: + if y is None: + return 0 + else: + return -1 + elif y is None: + return 1 + else: + # TODO: consider casting the values to string or int or floats? + # note that this is the minimal replacement function + return (x > y) - (x < y) diff --git a/src/univers/versions.py b/src/univers/versions.py index 3e54f452..1177cc6d 100644 --- a/src/univers/versions.py +++ b/src/univers/versions.py @@ -61,7 +61,7 @@ def __init__(self, version_string): @staticmethod def validate(version_string): - match = pypi_version.Version._regex.search(version_string) + match = pypi_version.Version._regex.search(version_string) # NOQA if not match: raise InvalidVersion(f"Invalid version: '{version_string}'") From 02e39ea4a99ee6bd055d8562db350b349fbc8be0 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 28 Sep 2021 14:56:36 +0200 Subject: [PATCH 129/160] Prepare release Signed-off-by: Philippe Ombredanne --- AUTHORS.rst | 4 +++- CHANGELOG.rst | 15 ++++++++++++--- bsd-new.LICENSE | 24 ++++++++++++++++++++++++ mit.LICENSE | 18 ++++++++++++++++++ setup.cfg | 29 +++++++++++++++++++++++++---- 5 files changed, 82 insertions(+), 8 deletions(-) create mode 100644 bsd-new.LICENSE create mode 100644 mit.LICENSE diff --git a/AUTHORS.rst b/AUTHORS.rst index 51a19cc8..8b8cc92a 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -1,3 +1,5 @@ The following organizations or individuals have contributed to this repo: -- +- Shivam Sandbhor +- nexB Inc. +- Philippe Ombredanne diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 02b6ac95..dba1b5c6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,13 +1,22 @@ Release notes ------------- + +### Version v30.0.0 + +- Switching back to semver +- Improve origin and license documentation +- Add tests for carets in RPMs +- Format, stremmline and refactor code +- Improve testing + ### Version v21.4.9 -- Added support Gentoo style versions. +- Add support Gentoo style versions. ### Version v21.4.8 -- More package types supported. -- Version classes are hashable and frozen now +- Add support for more package types. +- Version classes are now hashable and frozen ### Version v21.4.6 diff --git a/bsd-new.LICENSE b/bsd-new.LICENSE new file mode 100644 index 00000000..cb6a877f --- /dev/null +++ b/bsd-new.LICENSE @@ -0,0 +1,24 @@ +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +Neither the name of the ORGANIZATION nor the names of its contributors may be +used to endorse or promote products derived from this software without specific +prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/mit.LICENSE b/mit.LICENSE new file mode 100644 index 00000000..2d4a9898 --- /dev/null +++ b/mit.LICENSE @@ -0,0 +1,18 @@ +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 7655140d..12e01d10 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,17 +1,20 @@ [metadata] license_files = apache-2.0.LICENSE + bsd-new.LICENSE + mit.LICENSE NOTICE AUTHORS.rst CHANGELOG.rst + README.rst name = univers -author = nexB. Inc. and others +author = Shivam Sandbhor, nexB. Inc. and others author_email = info@aboutcode.org -license = Apache-2.0 +license = Apache-2.0 AND BSD-3-Clause AND MIT -description = univers +description = A mostly universal library to parse and compare software package versions and version ranges. A companion to Package URLs. long_description = file:README.rst -url = https://github.com/sbs2001/univers +url = https://github.com/nexB/univers classifiers = Development Status :: 5 - Production/Stable Intended Audience :: Developers @@ -20,7 +23,22 @@ classifiers = Topic :: Software Development Topic :: Utilities keywords = + semver utilities + version + release + version range + package URL + purl + arch + pacman + pypi + rpm + gentoo + ebuild + maven + debian + rubygems [options] package_dir= @@ -29,6 +47,9 @@ packages=find: include_package_data = true zip_safe = false install_requires = + packaging + semantic-version + attrs setup_requires = setuptools_scm[toml] >= 4 [options.packages.find] From 703205cc700a0abcd26a0225c707dbaf2152c644 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 28 Sep 2021 14:56:45 +0200 Subject: [PATCH 130/160] Format code Signed-off-by: Philippe Ombredanne --- tests/test_codestyle.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_codestyle.py b/tests/test_codestyle.py index c0e4e703..3d7d5b5e 100644 --- a/tests/test_codestyle.py +++ b/tests/test_codestyle.py @@ -14,4 +14,7 @@ def test_codestyle(self): try: subprocess.check_output(args.split()) except Exception as e: - raise Exception(f"Black style check failed, please format the code using black") from e + raise Exception( + "Black style check failed; please format the code using: " + "python -m black --check -l 100" + ) from e From 58ef9fee29961af5e9b357833ddec4b57af87007 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 28 Sep 2021 14:57:20 +0200 Subject: [PATCH 131/160] Use one generated function per tests #2 Ths way we can wrap some as expected failures for now. Signed-off-by: Philippe Ombredanne --- tests/test_rpm_vercmp.py | 58 ++++++++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/tests/test_rpm_vercmp.py b/tests/test_rpm_vercmp.py index 96a84f8f..80a39375 100644 --- a/tests/test_rpm_vercmp.py +++ b/tests/test_rpm_vercmp.py @@ -63,17 +63,53 @@ def test_parse_with_buggy(self): class VersionCompareTest(unittest.TestCase): - def test_from_rpmtest(self): - test_file = os.path.join(os.path.dirname(__file__), "test_data", "rpmvercmp.at") + pass - with io.open(test_file, encoding="utf-8") as rpmtests: - tests = list(parse_rpmvercmp_tests(rpmtests, with_buggy_comparisons=True)) - for test_count, (ver1, ver2, expected_comparison) in enumerate(tests, 1): - result = vercmp.vercmp(ver1, ver2) - if result != expected_comparison: - assert result == (expected_comparison, ver1, ver2) +def create_test_function(ver1, ver2, expected, name): + """ + Return a test function closed on test arguments. + """ + + def test_rpm_version(self): + print(f"testing (ver1={ver1}, ver2={ver2}, expected=expected{expected}") + result = vercmp.vercmp(ver1, ver2) + if result != expected: + assert result == (expected, ver1, ver2) + + test_rpm_version.__name__ = name + return test_rpm_version + + +def get_tests(): + """ + Yield test function from rpmvercmp.at data. + """ + test_file = os.path.join(os.path.dirname(__file__), "test_data", "rpmvercmp.at") + + with io.open(test_file, encoding="utf-8") as rpmtests: + tests = list(parse_rpmvercmp_tests(rpmtests, with_buggy_comparisons=True)) + for test_count, (ver1, ver2, expected) in enumerate(tests, 1): + name = f"test_rpm_version_{test_count}" + func = create_test_function(ver1, ver2, expected, name) + if "^" in ver1 or "^" in ver2: + result = vercmp.vercmp(ver1, ver2) + if result != expected: + func = unittest.expectedFailure(func) + yield func + + # Make sure we still test something, in case the m4 file drops + # content this will fail the test + assert test_count > 20 + + +def build_tests(): + """ + Create tests from rpmvercmp.at data and attach them to a test class. + """ + for i, test_func in enumerate(get_tests()): + name = f"test_rpm_version_{i}" + setattr(VersionCompareTest, name, test_func) + - # Make sure we still test something, in case the m4 file drops - # content this will fail the test - assert test_count > 20 +build_tests() From d12341d0144095860d4aff311e5356383000fc2e Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 28 Sep 2021 16:42:56 +0200 Subject: [PATCH 132/160] Add pkgcore gentoo tests Signed-off-by: Philippe Ombredanne --- src/univers/gentoo.py | 16 +- tests/test_gentoo_pkgcore.py | 418 +++++++++-------------------------- 2 files changed, 124 insertions(+), 310 deletions(-) diff --git a/src/univers/gentoo.py b/src/univers/gentoo.py index f4edf3d8..234db61c 100644 --- a/src/univers/gentoo.py +++ b/src/univers/gentoo.py @@ -35,9 +35,21 @@ def parse_version_and_revision(version_string): def vercmp(ver1, ver2): """ - Compare two versions ``ver1`` and ``ver2`` and return 0, 1, or -1 according to the - cmp semantics + Compare two versions ``ver1`` and ``ver2`` and return 0, 1, or -1 according + to the Python 2 cmp() semantics: + + Compare the two objects x and y and return an integer according to the + outcome. The return value is negative if x < y, zero if x == y and + strictly positive if x > y. """ + if not ver1: + if not ver2: + return 0 + else: + return -1 + elif not ver2: + return 1 + ver1, rev1 = parse_version_and_revision(ver1) ver2, rev2 = parse_version_and_revision(ver2) diff --git a/tests/test_gentoo_pkgcore.py b/tests/test_gentoo_pkgcore.py index 0c215661..321ba948 100644 --- a/tests/test_gentoo_pkgcore.py +++ b/tests/test_gentoo_pkgcore.py @@ -5,8 +5,7 @@ from random import shuffle -import pytest - +from univers.gentoo import parse_version_and_revision from univers.gentoo import vercmp @@ -24,206 +23,59 @@ def generate_misc_sufs(): return good_sufs, bad_sufs -class TestCPV: +good_vers = ("1", "2.3.4", "2.3.4a", "02.3", "2.03", "3d", "3D") - good_cats = ( - "dev-util", - "dev+", - "dev-util+", - "DEV-UTIL", - "aaa0", - "aaa-0", - "multi/depth", - "cross-dev_idiot.hacks-suck", - "a", - "foo---", - "multi--hyphen", - ) - bad_cats = (".util", "_dev", "", "dev-util ", "multi//depth") - good_pkgs = ( - "diffball", - "a9", - "a9+", - "a-100dpi", - "diff-mode-", - "multi--hyphen", - "timidity--", - "frob---", - "diffball-9-", - ) - bad_pkgs = ( - "diffball ", - "diffball-9", - "a-3D", - "-df", - "+dfa", - "timidity--9f", - "ormaybe---13_beta", - ) - - good_cp = ( - "bbb-9/foon", - "dev-util/diffball", - "dev-util/diffball-a9", - "dev-ut-asdf/emacs-cvs", - "xfce-base/xfce4", - "bah/f-100dpi", - "dev-util/diffball-blah-monkeys", - "virtual/7z", - ) - - good_vers = ("1", "2.3.4", "2.3.4a", "02.3", "2.03", "3d", "3D") - bad_vers = ("2.3a.4", "2.a.3", "2.3_", "2.3 ", "2.3.", "cvs.2") - - good_sufs, bad_sufs = generate_misc_sufs() - good_revs = ("-r1", "-r300", "-r0", "", "-r1000000000000000000") - bad_revs = ("-r", "-ra", "-r", "-R1") - - testing_secondary_args = False - - def make_inst(self, cat, pkg, fullver=""): - if self.testing_secondary_args: - return cpv.CPV(cat, pkg, fullver, versioned=bool(fullver)) - if fullver: - return cpv.VersionedCPV(f"{cat}/{pkg}-{fullver}") - return cpv.UnversionedCPV(f"{cat}/{pkg}") - - def test_simple_key(self): - with pytest.raises(cpv.InvalidCPV): - self.make_inst("da", "ba-3", "3.3") - for src in [ - [("dev-util", "diffball", "0.7.1"), "dev-util/diffball"], - ["dev-util/diffball"], - ["dev-perl/mod_perl"], - ["dev-perl/mod_p"], - [("dev-perl", "mod-p", ""), "dev-perl/mod-p"], - ["dev-perl/mod-p-1", "dev-perl/mod-p"], - ]: - if len(src) == 1: - key = src[0] - else: - key = src[1] - if isinstance(src[0], str): - cat, pkgver = src[0].rsplit("/", 1) - vals = pkgver.rsplit("-", 1) - if len(vals) == 1: - pkg = pkgver - ver = "" - else: - pkg, ver = vals - else: - cat, pkg, ver = src[0] - - assert self.make_inst(cat, pkg, ver).key == key - - def test_init(self): - cpv.CPV("dev-util", "diffball", "0.7.1") - cpv.CPV("dev-util", "diffball") - cpv.CPV("dev-util/diffball-0.7.1", versioned=True) - with pytest.raises(TypeError): - cpv.VersionedCPV("dev-util", "diffball", None) - - def test_parsing(self): - # check for gentoo bug 263787 - self.process_pkg(False, "app-text", "foo-123-bar") - self.process_ver(False, "app-text", "foo-123-bar", "2.0017a_p", "-r5") - with pytest.raises(cpv.InvalidCPV): - cpv.UnversionedCPV("app-text/foo-123") - for cat_ret, cats in [[False, self.good_cats], [True, self.bad_cats]]: - for cat in cats: - for pkg_ret, pkgs in [[False, self.good_pkgs], [True, self.bad_pkgs]]: - for pkg in pkgs: - self.process_pkg(cat_ret or pkg_ret, cat, pkg) - - for cp in self.good_cp: - cat, pkg = cp.rsplit("/", 1) - for rev_ret, revs in [[False, self.good_revs], [True, self.bad_revs]]: - for rev in revs: - for ver_ret, vers in [[False, self.good_vers], [True, self.bad_vers]]: - for ver in vers: - self.process_ver(ver_ret or rev_ret, cat, pkg, ver, rev) +good_sufs, bad_sufs = generate_misc_sufs() +good_revs = ("-r1", "-r300", "-r0", "", "-r1000000000000000000") + + +def older(v1, v2): + return vercmp(v1, v2) < 0 + + +def newer(v1, v2): + return vercmp(v1, v2) > 0 + + +def equal(v1, v2): + return vercmp(v1, v2) == 0 + + +class TestCPV: + def test_parse_version_and_revision_rev(self): for x in (10, 18, 19, 36, 100): - assert cpv.CPV("da", "ba", f"1-r0{'0' * x}").revision == 0 - assert int(cpv.CPV("da", "ba", f"1-r1{'0' * x}1").revision) == int(f"1{'0' * x}1") - - def process_pkg(self, ret, cat, pkg): - if ret: - with pytest.raises(cpv.InvalidCPV): - self.make_inst(cat, pkg) - else: - c = self.make_inst(cat, pkg) - assert c.cpvstr == f"{cat}/{pkg}" - assert c.category == cat - assert c.package == pkg - assert c.key == f"{cat}/{pkg}" - assert c.revision is None - assert c.version is None - assert c.fullver is None - - def process_ver(self, ret, cat, pkg, ver, rev): - if ret: - with pytest.raises(cpv.InvalidCPV): - self.make_inst(cat, pkg, f"{ver}{rev}") - else: - c = self.make_inst(cat, pkg, ver + rev) - if rev == "" or rev == "-r0": - assert c.cpvstr == f"{cat}/{pkg}-{ver}" - assert c.revision == 0 - if rev: - assert c.fullver == ver + rev - else: - assert c.revision == "" - assert c.fullver == ver - else: - assert c.revision == int(rev.lstrip("-r")) - assert c.cpvstr == f"{cat}/{pkg}-{ver}{rev}" - assert c.fullver == ver + rev - assert c.category == cat - assert c.package == pkg - assert c.key == f"{cat}/{pkg}" - assert c.version == ver - - for suf in self.good_sufs: - self.process_suf(ret, cat, pkg, ver + suf, rev) - for bad_suf in self.bad_sufs: - # double process, front and back. - self.process_suf(True, cat, pkg, suf + bad_suf, rev) - self.process_suf(True, cat, pkg, bad_suf + suf, rev) - - for suf in self.bad_sufs: - # check standalone. - self.process_suf(True, cat, pkg, ver + suf, rev) - - def process_suf(self, ret, cat, pkg, ver, rev): - if ret: - with pytest.raises(cpv.InvalidCPV): - self.make_inst(cat, pkg, ver + rev) - else: - # redundant in light of process_ver... combine these somehow. - c = self.make_inst(cat, pkg, ver + rev) - if rev == "" or rev == "-r0": - assert c.cpvstr == f"{cat}/{pkg}-{ver}" - assert c.revision == 0 - if rev: - assert c.fullver == ver + rev - else: - assert c.revision == "" - assert c.fullver == ver - else: - assert c.cpvstr == f"{cat}/{pkg}-{ver}{rev}" - assert c.revision == int(rev.lstrip("-r")) - assert c.fullver == ver + rev - assert c.category == cat - assert c.package == pkg - assert c.key == f"{cat}/{pkg}" - assert c.version == ver + _ver, rev = parse_version_and_revision(f"1-r0{'0' * x}") + assert rev == 0 + + _ver, rev = parse_version_and_revision(f"1-r1{'0' * x}1") + assert int(rev) == int(f"1{'0' * x}1") + + def test_parse_version_and_revisions(self): + for ver in good_vers: + v_r = parse_version_and_revision(ver) + assert v_r == (ver, 0) + for rev in good_revs: + version = f"{ver}{rev}" + try: + v_r = parse_version_and_revision(version) + expected_rev = rev.lstrip("-r") + expected_rev = expected_rev and int(expected_rev) or 0 + assert v_r == (ver, expected_rev), f"Failed to parse: {version}" + except Exception as e: + raise Exception(f"Failed to parse: {version!r}") from e + + for suf in good_sufs: + version = f"{ver}{suf}{rev}" + v_r = parse_version_and_revision(version) + assert v_r == (f"{ver}{suf}", expected_rev) def test_cmp(self): - ukls, vkls = cpv.UnversionedCPV, cpv.VersionedCPV - assert vkls("dev-util/diffball-0.1") < vkls("dev-util/diffball-0.2") - base = "dev-util/diffball-0.7.1" - assert vkls(base) == vkls(base) + assert older("0.1", "0.2") + + def test_cmp1(self): + base = "0.7.1" + base = equal(base, base) for rev in ("", "-r1"): last = None for suf in ["_alpha", "_beta", "_pre", "", "_p"]: @@ -232,141 +84,91 @@ def test_cmp(self): else: sufs = [suf, f"{suf}4"] for x in sufs: - cur = vkls(f"{base}{x}{rev}") - assert cur == vkls(f"{base}{x}{rev}") + cur = f"{base}{x}{rev}" + assert equal(cur, f"{base}{x}{rev}") if last is not None: - assert cur > last + assert newer(cur, last) - assert vkls("da/ba-6a") > vkls("da/ba-6") - assert vkls("da/ba-6a-r1") > vkls("da/ba-6a") - assert vkls("da/ba-6.0") > vkls("da/ba-6") - assert vkls("da/ba-6.0.0") > vkls("da/ba-6.0b") - assert vkls("da/ba-6.02") > vkls("da/ba-6.0.0") - # float comparison rules. - assert vkls("da/ba-6.2") > vkls("da/ba-6.054") - assert vkls("da/ba-6") == vkls("da/ba-6") - assert ukls("db/ba") > ukls("da/ba") - assert ukls("da/bb") > ukls("da/ba") - assert vkls("da/ba-6.0_alpha0_p1") > vkls("da/ba-6.0_alpha") - assert vkls("da/ba-6.0_alpha") == vkls("da/ba-6.0_alpha0") - assert vkls("da/ba-6.1") > vkls("da/ba-6.09") - assert vkls("da/ba-6.0.1") > vkls("da/ba-6.0") - assert vkls("da/ba-12.2.5") > vkls("da/ba-12.2b") + def test_cmp2(self): + assert newer("6a", "6") + assert newer("6a-r1", "6a") + assert newer("6.0", "6") + assert newer("6.0.0", "6.0b") + assert newer("6.02", "6.0.0") + def test_cmp_float(self): + # float comparison rules. + assert newer("6.2", "6.054") + assert equal("6", "6") + assert newer("6.0_alpha0_p1", "6.0_alpha") + assert equal("6.0_alpha", "6.0_alpha0") + assert newer("6.1", "6.09") + assert newer("6.0.1", "6.0") + assert newer("12.2.5", "12.2b") + + def test_cmp_287848(self): # test for gentoo bug 287848 - assert vkls("dev-lang/erlang-12.2.5") > vkls("dev-lang/erlang-12.2b") - assert vkls("dev-lang/erlang-12.2.5-r1") > vkls("dev-lang/erlang-12.2b") + assert newer("12.2.5", "12.2b") + assert newer("12.2.5-r1", "12.2b") + def test_cmp_equiv(self): # equivalent versions - assert vkls("da/ba-6.01.0") == vkls("da/ba-6.010.0") - assert vkls("da/ba-6.0.1") == vkls("da/ba-6.000.1") + assert equal("6.01.0", "6.010.0") + assert equal("6.0.1", "6.000.1") + def test_cmp_equiv_rev(self): # equivalent revisions - assert vkls("da/ba-6.01.0") == vkls("da/ba-6.01.0-r0") - assert vkls("da/ba-6.01.0-r0") == vkls("da/ba-6.01.0-r00") - assert vkls("da/ba-6.01.0-r1") == vkls("da/ba-6.01.0-r001") + assert equal("6.01.0", "6.01.0-r0") + assert equal("6.01.0-r0", "6.01.0-r00") + assert equal("6.01.0-r1", "6.01.0-r001") for v1, v2 in ( ("1.001000000000000000001", "1.001000000000000000002"), ("1.00100000000", "1.0010000000000000001"), ("1.01", "1.1"), ): - assert vkls(f"da/ba-{v2}") > vkls(f"da/ba-{v1}") + assert newer(f"{v2}", f"{v1}") for x in (18, 36, 100): s = "0" * x - assert vkls(f"da/ba-10{s}1") > vkls(f"da/ba-1{s}1") + assert newer(f"10{s}1", f"1{s}1") for x in (18, 36, 100): s = "0" * x - assert vkls(f"da/ba-1-r10{s}1") > vkls(f"da/ba-1-r1{s}1") + assert newer(f"1-r10{s}1", f"1-r1{s}1") - assert vkls("sys-apps/net-tools-1.60_p2010081516093") > vkls( - "sys-apps/net-tools-1.60_p2009072801401" - ) + assert newer("1.60_p2010081516093", "1.60_p2009072801401") - assert vkls("sys-apps/net-tools-1.60_p20100815160931") > vkls( - "sys-apps/net-tools-1.60_p20090728014017" - ) + assert newer("1.60_p20100815160931", "1.60_p20090728014017") - assert vkls("sys-apps/net-tools-1.60_p20100815160931") > vkls( - "sys-apps/net-tools-1.60_p20090728014017-r1" - ) + assert newer("1.60_p20100815160931", "1.60_p20090728014017-r1") # Regression test: python does comparison slightly differently # if the classes do not match exactly (it prefers rich # comparison over __cmp__). - class DummySubclass(cpv.CPV): - pass - - assert DummySubclass("da/ba-6.0_alpha0_p1", versioned=True) != vkls("da/ba-6.0_alpha") - assert DummySubclass("da/ba-6.0_alpha0", versioned=True) == vkls("da/ba-6.0_alpha") - - assert DummySubclass("da/ba-6.0", versioned=True) != "foon" - assert DummySubclass("da/ba-6.0", versioned=True) == DummySubclass( - "da/ba-6.0-r0", versioned=True - ) - - def test_no_init(self): - """Test if the cpv is in a somewhat sane state if __init__ fails. - - IPython used to segfault when showing a verbose traceback for - a subclass of CPV which raised cpv.InvalidCPV. This checks - if such uninitialized objects survive some basic poking. - """ - uninited = cpv.CPV.__new__(cpv.CPV) - broken = cpv.CPV.__new__(cpv.CPV) - with pytest.raises(cpv.InvalidCPV): - broken.__init__("broken", versioned=True) - for thing in (uninited, broken): - # the c version returns None, the py version does not have the attr - getattr(thing, "cpvstr", None) - repr(thing) - str(thing) - # The c version returns a constant, the py version raises - try: - hash(thing) - except AttributeError: - pass - - def test_r0_revisions(self): - # single '0' - obj = cpv.CPV("dev-util/diffball-1.0-r0", versioned=True) - assert obj.cpvstr == "dev-util/diffball-1.0" - assert str(obj) == "dev-util/diffball-1.0" - assert obj.fullver == "1.0-r0" - assert obj.revision == 0 - - # multiple '0' - obj = cpv.CPV("dev-util/diffball-1.0-r000", versioned=True) - assert obj.cpvstr == "dev-util/diffball-1.0" - assert str(obj) == "dev-util/diffball-1.0" - assert obj.fullver == "1.0-r000" - assert obj.revision == 0 - - # single '0' prefix - obj = cpv.CPV("dev-util/diffball-1.0-r01", versioned=True) - assert obj.cpvstr == "dev-util/diffball-1.0-r1" - assert str(obj) == "dev-util/diffball-1.0-r1" - assert obj.fullver == "1.0-r01" - assert obj.revision == 1 - - # multiple '0' prefixes - obj = cpv.CPV("dev-util/diffball-1.0-r0001", versioned=True) - assert obj.cpvstr == "dev-util/diffball-1.0-r1" - assert str(obj) == "dev-util/diffball-1.0-r1" - assert obj.fullver == "1.0-r0001" - assert obj.revision == 1 - - def test_attribute_errors(self): - obj = cpv.VersionedCPV("foo/bar-0") - assert not obj == 0 - assert obj != 0 - with pytest.raises(TypeError): - assert obj < 0 - with pytest.raises(TypeError): - assert obj <= 0 - with pytest.raises(TypeError): - assert obj > 0 - with pytest.raises(TypeError): - assert obj >= 0 + + assert not equal("6.0_alpha0_p1", "6.0_alpha") + assert equal("6.0_alpha0", "6.0_alpha") + + assert not equal("6.0", "") + assert equal("6.0", "6.0-r0") + + def test_r0_revisions_with_single_0(self): + ver, rev = parse_version_and_revision("1.0-r0") + assert ver == "1.0" + assert rev == 0 + + def test_r0_revisions_with_multiple_0(self): + ver, rev = parse_version_and_revision("1.0-r000") + assert ver == "1.0" + assert rev == 0 + + def test_r0_revisions_with_single_0_prefix(self): + ver, rev = parse_version_and_revision("1.0-r01") + assert ver == "1.0" + assert rev == 1 + + def test_r0_revisions_with_multiple_0_prefix(self): + ver, rev = parse_version_and_revision("11.0-r0001") + assert ver == "11.0" + assert rev == 1 From 9ed4609bda9bf7ec1c142a9b008233ccec0db8f2 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 5 Oct 2021 10:02:11 +0200 Subject: [PATCH 133/160] Format code and add comments Signed-off-by: Philippe Ombredanne --- src/univers/versions.py | 43 +++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/src/univers/versions.py b/src/univers/versions.py index 1177cc6d..737d9e11 100644 --- a/src/univers/versions.py +++ b/src/univers/versions.py @@ -4,7 +4,6 @@ # # Visit https://aboutcode.org and https://github.com/nexB/ for support and download. - import re import attr from functools import total_ordering @@ -26,11 +25,16 @@ class InvalidVersion(ValueError): class BaseVersion: - # each version value should be comparable e.g. implement functools.total_ordering + """ + Base version object to subclass for each version scheme. - scheme = attr.ib() - value = attr.ib() - version_string = attr.ib() + Each version value should be comparable e.g., implement + functools.total_ordering + """ + + # the version scheme is a class attribute + scheme = None + value = attr.ib(type=str) def validate(self): """ @@ -39,14 +43,11 @@ def validate(self): raise NotImplementedError def __str__(self): - return f"{self.scheme}:{self.version_string}" - - def __repr__(self) -> str: - return f"{self.__class__.__name__} <{self.__str__()}>" + return f"{self.scheme}:{self.value}" @total_ordering -@attr.s(frozen=True, init=False, order=False, eq=False, hash=True, repr=False) +@attr.s(frozen=True, init=False, order=False, hash=True) class PYPIVersion(BaseVersion): scheme = "pypi" @@ -81,9 +82,13 @@ def validate(self): Validate that the version is valid for its scheme """ # generic implementation ... - # use 1. of https://github.com/repology/libversion/blob/master/doc/ALGORITHM.md#core-algorithm - # Version is split into separate all-alphabetic or all-numeric components. All other characters are treated as separators. Empty components are not generated. - # 10.2alpha3..patch.4. → 10, 2, alpha, 3, patch, 4 + # TODO: Should use + # https://github.com/repology/libversion/blob/master/doc/ALGORITHM.md#core-algorithm + # Version is split into separate all-alphabetic or all-numeric + # components. + # All other characters are treated as separators. Empty components are + # not generated. + # 10.2alpha3..patch.4. → 10, 2, alpha, 3, patch, 4 @attr.s(frozen=True, init=False, order=False, eq=False, hash=True, repr=False) @@ -236,6 +241,7 @@ def __lt__(self, other): # TODO : Should these be upper case global constants ? + version_class_by_scheme = { "generic": GenericVersion, "semver": SemverVersion, @@ -247,18 +253,25 @@ def __lt__(self, other): "ebuild": GentooVersion, } -# TODO: This is messed up + version_class_by_package_type = { "deb": DebianVersion, "pypi": PYPIVersion, "maven": MavenVersion, "nuget": NugetVersion, + # TODO: composer may need its own scheme see https://github.com/nexB/univers/issues/5 + # and https://getcomposer.org/doc/articles/versions.md "composer": SemverVersion, - "npm": SemverVersion, + # TODO: gem may need its own scheme see https://github.com/nexB/univers/issues/5 + # and https://snyk.io/blog/differences-in-version-handling-gems-and-npm/ + # https://semver.org/spec/v2.0.0.html#spec-item-11 "gem": SemverVersion, + "npm": SemverVersion, "rpm": RPMVersion, "golang": SemverVersion, "generic": SemverVersion, + # apache is not semver at large. And in particular we may have schemes that + # are package name-specific "apache": SemverVersion, "hex": SemverVersion, "cargo": SemverVersion, From 2ed3ce34dfebf728fa678d31ff2bf1b543cec087 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 5 Oct 2021 10:17:50 +0200 Subject: [PATCH 134/160] Use correct location for pytest Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9a4c950c..01fcc812 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,7 +13,7 @@ jobs: image_name: ubuntu-16.04 python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: - all: tmp/bin/pytest -vvs + all: venv/bin/pytest -vvs - template: etc/ci/azure-linux.yml parameters: @@ -21,7 +21,7 @@ jobs: image_name: ubuntu-18.04 python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: - all: tmp/bin/pytest -n 2 -vvs + all: venv/bin/pytest -n 2 -vvs - template: etc/ci/azure-linux.yml parameters: @@ -29,7 +29,7 @@ jobs: image_name: ubuntu-20.04 python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: - all: tmp/bin/pytest -n 2 -vvs + all: venv/bin/pytest -n 2 -vvs - template: etc/ci/azure-mac.yml parameters: @@ -37,7 +37,7 @@ jobs: image_name: macos-10.14 python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: - all: tmp/bin/pytest -n 2 -vvs + all: venv/bin/pytest -n 2 -vvs - template: etc/ci/azure-mac.yml parameters: @@ -45,7 +45,7 @@ jobs: image_name: macos-10.15 python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: - all: tmp/bin/pytest -n 2 -vvs + all: venv/bin/pytest -n 2 -vvs - template: etc/ci/azure-win.yml parameters: @@ -53,7 +53,7 @@ jobs: image_name: vs2017-win2016 python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: - all: tmp\Scripts\pytest -n 2 -vvs + all: venv\Scripts\pytest -n 2 -vvs - template: etc/ci/azure-win.yml parameters: @@ -61,4 +61,4 @@ jobs: image_name: windows-2019 python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: - all: tmp\Scripts\pytest -n 2 -vvs + all: venv\Scripts\pytest -n 2 -vvs From f45cbc5e6aebfb20cc3c9a981b26b444e7098356 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 5 Oct 2021 10:18:59 +0200 Subject: [PATCH 135/160] Use correct command to install Signed-off-by: Philippe Ombredanne --- .travis.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 524f507d..aab9b6f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ # This is a skeleton Travis CI config file that provides a starting point for adding CI -# to a Python project. Since we primarily develop in python3, this skeleton config file -# will be specific to that language. +# to a Python project. # # See https://config.travis-ci.com/ for a full list of configuration options. @@ -14,8 +13,6 @@ python: - "3.7" - "3.8" -# Scripts to run at install stage -install: pip install ./ && pip install -r requirements-dev.txt +install: ./configure --dev -# Scripts to run at script stage -script: python -m pytest +script: venv/bin/pytest -vvs From f282a33d22c5a6b561e33745de9367e4b8a5253e Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 5 Oct 2021 10:40:47 +0200 Subject: [PATCH 136/160] Bump poackaging Signed-off-by: Philippe Ombredanne --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8b6e451a..03df547c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -packaging==20.9 +packaging==21.0 semantic-version==2.8.5 attrs==20.3.0 From 567156396f81d533ee3d4085fe2030d58b8ebd2f Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 5 Oct 2021 12:48:28 +0200 Subject: [PATCH 137/160] Treat text files as text And not a possible binaries Also Ensure that we craft a minimally parsable license expression, even if not correct. Signed-off-by: Philippe Ombredanne --- etc/scripts/utils_thirdparty.py | 51 +++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index d77afc39..7613a0c7 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -24,13 +24,14 @@ import attr import license_expression import packageurl -import utils_pip_compatibility_tags -import utils_pypi_supported_tags import requests import saneyaml +import utils_pip_compatibility_tags +import utils_pypi_supported_tags from commoncode import fileutils from commoncode.hash import multi_checksums +from commoncode.text import python_safe_name from packaging import tags as packaging_tags from packaging import version as packaging_version from utils_requirements import load_requirements @@ -172,11 +173,20 @@ def fetch_wheels( else: force_pinned = False - rrp = list(get_required_remote_packages( - requirements_file=requirements_file, - force_pinned=force_pinned, - remote_links_url=remote_links_url, - )) + try: + rrp = list(get_required_remote_packages( + requirements_file=requirements_file, + force_pinned=force_pinned, + remote_links_url=remote_links_url, + )) + except Exception as e: + raise Exception( + dict( + requirements_file=requirements_file, + force_pinned=force_pinned, + remote_links_url=remote_links_url, + ) + ) from e fetched_filenames = set() for name, version, package in rrp: @@ -211,6 +221,7 @@ def fetch_wheels( print(f'Missed package {nv} in remote repo, has only:') for pv in rr.get_versions(n): print(' ', pv) + raise Exception('Missed some packages in remote repo') def fetch_sources( @@ -261,6 +272,8 @@ def fetch_sources( fetched = package.fetch_sdist(dest_dir=dest_dir) error = f'Failed to fetch' if not fetched else None yield package, error + if missed: + raise Exception(f'Missing source packages in {remote_links_url}', missed) ################################################################################ # @@ -693,8 +706,7 @@ def save_if_modified(location, content): return False if TRACE: print(f'Saving ABOUT (and NOTICE) files for: {self}') - wmode = 'wb' if isinstance(content, bytes) else 'w' - with open(location, wmode, encoding="utf-8") as fo: + with open(location, 'w') as fo: fo.write(content) return True @@ -905,8 +917,8 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): other_classifiers = [c for c in classifiers if not c.startswith('License')] holder = raw_data['Author'] - holder_contact=raw_data['Author-email'] - copyright = f'Copyright (c) {holder} <{holder_contact}>' + holder_contact = raw_data['Author-email'] + copyright_statement = f'Copyright (c) {holder} <{holder_contact}>' pkginfo_data = dict( name=raw_data['Name'], @@ -914,7 +926,7 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): version=raw_data['Version'], description=raw_data['Summary'], homepage_url=raw_data['Home-page'], - copyright=copyright, + copyright=copyright_statement, license_expression=license_expression, holder=holder, holder_contact=holder_contact, @@ -1845,7 +1857,7 @@ def get(self, path_or_url, as_text=True): if not os.path.exists(cached): content = get_file_content(path_or_url=path_or_url, as_text=as_text) wmode = 'w' if as_text else 'wb' - with open(cached, wmode, encoding="utf-8") as fo: + with open(cached, wmode) as fo: fo.write(content) return content else: @@ -1857,7 +1869,7 @@ def put(self, filename, content): """ cached = os.path.join(self.directory, filename) wmode = 'wb' if isinstance(content, bytes) else 'w' - with open(cached, wmode, encoding="utf-8") as fo: + with open(cached, wmode) as fo: fo.write(content) @@ -2331,7 +2343,7 @@ def get_required_remote_packages( repo = get_remote_repo(remote_links_url=remote_links_url) else: # a local path - assert os.path.exists(remote_links_url) + assert os.path.exists(remote_links_url), f'Path does not exist: {remote_links_url}' repo = get_local_repo(directory=remote_links_url) for name, version in required_name_versions: @@ -2365,7 +2377,7 @@ def update_requirements(name, version=None, requirements_file='requirements.txt' updated_name_versions = sorted(updated_name_versions) nvs = '\n'.join(f'{name}=={version}' for name, version in updated_name_versions) - with open(requirements_file, 'w', encoding="utf-8") as fo: + with open(requirements_file, 'w') as fo: fo.write(nvs) @@ -2383,7 +2395,7 @@ def hash_requirements(dest_dir=THIRDPARTY_DIR, requirements_file='requirements.t raise Exception(f'Missing required package {name}=={version}') hashed.append(package.specifier_with_hashes) - with open(requirements_file, 'w', encoding="utf-8") as fo: + with open(requirements_file, 'w') as fo: fo.write('\n'.join(hashed)) ################################################################################ @@ -2961,5 +2973,6 @@ def compute_normalized_license_expression(declared_licenses): from packagedcode import pypi return pypi.compute_normalized_license(declared_licenses) except ImportError: - # Scancode is not installed, we join all license strings and return it - return ' '.join(declared_licenses) + # Scancode is not installed, clean and join all the licenses + lics = [python_safe_name(l).lower() for l in declared_licenses] + return ' AND '.join(lics).lower() From 14f6a2da068bcaa9eed1809bfdafd0656afb47d0 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 5 Oct 2021 12:50:40 +0200 Subject: [PATCH 138/160] Add helper to publish files in GH releases The upload is otherwise shaky. Signed-off-by: Philippe Ombredanne --- etc/scripts/publish_files.py | 204 +++++++++++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 etc/scripts/publish_files.py diff --git a/etc/scripts/publish_files.py b/etc/scripts/publish_files.py new file mode 100644 index 00000000..f343cb3c --- /dev/null +++ b/etc/scripts/publish_files.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import hashlib +import os +import sys + +from pathlib import Path + +import click +import requests +import utils_thirdparty + +from github_release_retry import github_release_retry as grr + +""" +Create GitHub releases and upload files there. +""" + + +def get_files(location): + """ + Return an iterable of (filename, Path, md5) tuples for files in the `location` + directory tree recursively. + """ + for top, _dirs, files in os.walk(location): + for filename in files: + pth = Path(os.path.join(top, filename)) + with open(pth, 'rb') as fi: + md5 = hashlib.md5(fi.read()).hexdigest() + yield filename, pth, md5 + + +def get_etag_md5(url): + """ + Return the cleaned etag of URL `url` or None. + """ + headers = utils_thirdparty.get_remote_headers(url) + headers = {k.lower(): v for k, v in headers.items()} + etag = headers .get('etag') + if etag: + etag = etag.strip('"').lower() + return etag + + +def create_or_update_release_and_upload_directory( + user, + repo, + tag_name, + token, + directory, + retry_limit=10, + description=None, +): + """ + Create or update a GitHub release at https://github.com// for + `tag_name` tag using the optional `description` for this release. + Use the provided `token` as a GitHub token for API calls authentication. + Upload all files found in the `directory` tree to that GitHub release. + Retry API calls up to `retry_limit` time to work around instability the + GitHub API. + + Remote files that are not the same as the local files are deleted and re- + uploaded. + """ + release_homepage_url = f'https://github.com/{user}/{repo}/releases/{tag_name}' + + # scrape release page HTML for links + urls_by_filename = {os.path.basename(l): l + for l in utils_thirdparty.get_paths_or_urls(links_url=release_homepage_url) + } + + # compute what is new, modified or unchanged + print(f'Compute which files is new, modified or unchanged in {release_homepage_url}') + + new_to_upload = [] + unchanged_to_skip = [] + modified_to_delete_and_reupload = [] + for filename, pth, md5 in get_files(directory): + url = urls_by_filename.get(filename) + if not url: + print(f'{filename} content is NEW, will upload') + new_to_upload.append(pth) + continue + + out_of_date = get_etag_md5(url) != md5 + if out_of_date: + print(f'{url} content is CHANGED based on md5 etag, will re-upload') + modified_to_delete_and_reupload.append(pth) + else: + # print(f'{url} content is IDENTICAL, skipping upload based on Etag') + unchanged_to_skip.append(pth) + print('.') + + ghapi = grr.GithubApi( + github_api_url='https://api.github.com', + user=user, + repo=repo, + token=token, + retry_limit=retry_limit, + ) + + # yank modified + print( + f'Unpublishing {len(modified_to_delete_and_reupload)} published but ' + f'locally modified files in {release_homepage_url}') + + release = ghapi.get_release_by_tag(tag_name) + + for pth in modified_to_delete_and_reupload: + filename = os.path.basename(pth) + asset_id = ghapi.find_asset_id_by_file_name(filename, release) + print (f' Unpublishing file: {filename}).') + response = ghapi.delete_asset(asset_id) + if response.status_code != requests.codes.no_content: # NOQA + raise Exception(f'failed asset deletion: {response}') + + # finally upload new and modified + to_upload = new_to_upload + modified_to_delete_and_reupload + print(f'Publishing with {len(to_upload)} files to {release_homepage_url}') + release = grr.Release(tag_name=tag_name, body=description) + grr.make_release(ghapi, release, to_upload) + + +TOKEN_HELP = ( + 'The Github personal acess token is used to authenticate API calls. ' + 'Required unless you set the GITHUB_TOKEN environment variable as an alternative. ' + 'See for details: https://github.com/settings/tokens and ' + 'https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token' +) + + +@click.command() + +@click.option( + '--user-repo-tag', + help='The GitHub qualified repository user/name/tag in which ' + 'to create the release such as in nexB/thirdparty/pypi', + type=str, + required=True, +) +@click.option( + '-d', '--directory', + help='The directory that contains files to upload to the release.', + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True), + required=True, +) +@click.option( + '--token', + help=TOKEN_HELP, + default=os.environ.get('GITHUB_TOKEN', None), + type=str, + required=False, +) +@click.option( + '--description', + help='Text description for the release. Ignored if the release exists.', + default=None, + type=str, + required=False, +) +@click.option( + '--retry_limit', + help='Number of retries when making failing GitHub API calls. ' + 'Retrying helps work around transient failures of the GitHub API.', + type=int, + default=10, +) +@click.help_option('-h', '--help') +def publish_files( + user_repo_tag, + directory, + retry_limit=10, token=None, description=None, +): + """ + Publish all the files in DIRECTORY as assets to a GitHub release. + Either create or update/replace remote files' + """ + if not token: + click.secho('--token required option is missing.') + click.secho(TOKEN_HELP) + sys.exit(1) + + user, repo, tag_name = user_repo_tag.split('/') + + create_or_update_release_and_upload_directory( + user=user, + repo=repo, + tag_name=tag_name, + description=description, + retry_limit=retry_limit, + token=token, + directory=directory, + ) + + +if __name__ == '__main__': + publish_files() From 1a2a144005dc1831223f64c36dc470f3265659bd Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Wed, 6 Oct 2021 09:02:50 +0800 Subject: [PATCH 139/160] Add code to use curl if wget is not installed Signed-off-by: Chin Yeung Li --- configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure b/configure index b965692d..a141735f 100755 --- a/configure +++ b/configure @@ -127,7 +127,7 @@ create_virtualenv() { VIRTUALENV_PYZ="$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz" else VIRTUALENV_PYZ="$CFG_ROOT_DIR/$VENV_DIR/virtualenv.pyz" - wget -O "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL" + wget -O "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL" 2>/dev/null || curl -o "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL" fi $PYTHON_EXECUTABLE "$VIRTUALENV_PYZ" \ From 7aa7d4c08977128a24f029bea1d587f48842210d Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 8 Oct 2021 14:39:38 +0200 Subject: [PATCH 140/160] Do not issue warning if thirdparty dir is missing Signed-off-by: Philippe Ombredanne --- configure | 5 ++++- configure.bat | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/configure b/configure index b965692d..13ee98e8 100755 --- a/configure +++ b/configure @@ -51,7 +51,10 @@ CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org -PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty --find-links https://thirdparty.aboutcode.org/pypi" +if [ -f "$CFG_ROOT_DIR/thirdparty" ]; then + PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty " +fi +PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi" ################################ diff --git a/configure.bat b/configure.bat index 0c824a47..46ed4b36 100644 --- a/configure.bat +++ b/configure.bat @@ -49,7 +49,11 @@ set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" @rem ################################ @rem # Thirdparty package locations and index handling -set "PIP_EXTRA_ARGS=--find-links %CFG_ROOT_DIR%\thirdparty --find-links https://thirdparty.aboutcode.org/pypi" & %INDEX_ARG% +if exist ""%CFG_ROOT_DIR%\thirdparty"" ( + set "PIP_EXTRA_ARGS=--find-links %CFG_ROOT_DIR%\thirdparty " +) + +set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% --find-links https://thirdparty.aboutcode.org/pypi" & %INDEX_ARG% @rem ################################ From 8639d92cc80732af625612a2390dc0de8e4818c9 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 8 Oct 2021 14:40:28 +0200 Subject: [PATCH 141/160] Update dependencies Signed-off-by: Philippe Ombredanne --- requirements-dev.txt | 14 +++++++++++--- requirements.txt | 3 ++- setup.cfg | 4 +++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index d0f170bd..ab646c1e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,11 @@ -pytest==6.2.2 -black==20.8b1 -twine==3.4.1 +execnet==1.9.0 +importlib-metadata==4.8.1 +iniconfig==1.1.1 +pluggy==1.0.0 +py==1.10.0 +pytest==6.2.5 +pytest-forked==1.3.0 +pytest-xdist==2.4.0 +toml==0.10.2 +typing-extensions==3.10.0.2 +zipp==3.6.0 diff --git a/requirements.txt b/requirements.txt index 03df547c..938c44bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +attrs==21.2.0 packaging==21.0 +pyparsing==2.4.7 semantic-version==2.8.5 -attrs==20.3.0 diff --git a/setup.cfg b/setup.cfg index 627b5713..0b2d1b75 100644 --- a/setup.cfg +++ b/setup.cfg @@ -47,10 +47,12 @@ package_dir= packages=find: include_package_data = true zip_safe = false + install_requires = + attrs packaging semantic-version - attrs + setup_requires = setuptools_scm[toml] >= 4 [options.packages.find] From 90334008e296b41f43755091061124b9f8c050aa Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 8 Oct 2021 14:40:48 +0200 Subject: [PATCH 142/160] Remove unused skeleton artifacts Signed-off-by: Philippe Ombredanne --- src/README.rst | 2 -- tests/README.rst | 2 -- thirdparty/README.rst | 2 -- 3 files changed, 6 deletions(-) delete mode 100644 src/README.rst delete mode 100644 tests/README.rst delete mode 100644 thirdparty/README.rst diff --git a/src/README.rst b/src/README.rst deleted file mode 100644 index ec651fcf..00000000 --- a/src/README.rst +++ /dev/null @@ -1,2 +0,0 @@ -Put your Python source code (and installable data) in this directory. - diff --git a/tests/README.rst b/tests/README.rst deleted file mode 100644 index d94783e6..00000000 --- a/tests/README.rst +++ /dev/null @@ -1,2 +0,0 @@ -Put your Python test modules in this directory. - diff --git a/thirdparty/README.rst b/thirdparty/README.rst deleted file mode 100644 index b31482f8..00000000 --- a/thirdparty/README.rst +++ /dev/null @@ -1,2 +0,0 @@ -Put your Python dependency wheels to be vendored in this directory. - From b46d84f6ae633105ce0b1ff51714e34778d608f5 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 11 Oct 2021 22:29:48 +0200 Subject: [PATCH 143/160] Handle as_text correctly in cache Signed-off-by: Philippe Ombredanne --- etc/scripts/utils_thirdparty.py | 40 ++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 15 deletions(-) mode change 100644 => 100755 etc/scripts/utils_thirdparty.py diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py old mode 100644 new mode 100755 index d77afc39..99b9c0e5 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -172,11 +172,20 @@ def fetch_wheels( else: force_pinned = False - rrp = list(get_required_remote_packages( - requirements_file=requirements_file, - force_pinned=force_pinned, - remote_links_url=remote_links_url, - )) + try: + rrp = list(get_required_remote_packages( + requirements_file=requirements_file, + force_pinned=force_pinned, + remote_links_url=remote_links_url, + )) + except Exception as e: + raise Exception( + dict( + requirements_file=requirements_file, + force_pinned=force_pinned, + remote_links_url=remote_links_url, + ) + ) from e fetched_filenames = set() for name, version, package in rrp: @@ -211,6 +220,7 @@ def fetch_wheels( print(f'Missed package {nv} in remote repo, has only:') for pv in rr.get_versions(n): print(' ', pv) + raise Exception('Missed some packages in remote repo') def fetch_sources( @@ -261,6 +271,8 @@ def fetch_sources( fetched = package.fetch_sdist(dest_dir=dest_dir) error = f'Failed to fetch' if not fetched else None yield package, error + if missed: + raise Exception(f'Missing source packages in {remote_links_url}', missed) ################################################################################ # @@ -693,8 +705,7 @@ def save_if_modified(location, content): return False if TRACE: print(f'Saving ABOUT (and NOTICE) files for: {self}') - wmode = 'wb' if isinstance(content, bytes) else 'w' - with open(location, wmode, encoding="utf-8") as fo: + with open(location, 'w') as fo: fo.write(content) return True @@ -1845,7 +1856,7 @@ def get(self, path_or_url, as_text=True): if not os.path.exists(cached): content = get_file_content(path_or_url=path_or_url, as_text=as_text) wmode = 'w' if as_text else 'wb' - with open(cached, wmode, encoding="utf-8") as fo: + with open(cached, wmode) as fo: fo.write(content) return content else: @@ -1857,7 +1868,7 @@ def put(self, filename, content): """ cached = os.path.join(self.directory, filename) wmode = 'wb' if isinstance(content, bytes) else 'w' - with open(cached, wmode, encoding="utf-8") as fo: + with open(cached, wmode) as fo: fo.write(content) @@ -2331,7 +2342,7 @@ def get_required_remote_packages( repo = get_remote_repo(remote_links_url=remote_links_url) else: # a local path - assert os.path.exists(remote_links_url) + assert os.path.exists(remote_links_url), f'Path does not exist: {remote_links_url}' repo = get_local_repo(directory=remote_links_url) for name, version in required_name_versions: @@ -2365,7 +2376,7 @@ def update_requirements(name, version=None, requirements_file='requirements.txt' updated_name_versions = sorted(updated_name_versions) nvs = '\n'.join(f'{name}=={version}' for name, version in updated_name_versions) - with open(requirements_file, 'w', encoding="utf-8") as fo: + with open(requirements_file, 'w') as fo: fo.write(nvs) @@ -2383,7 +2394,7 @@ def hash_requirements(dest_dir=THIRDPARTY_DIR, requirements_file='requirements.t raise Exception(f'Missing required package {name}=={version}') hashed.append(package.specifier_with_hashes) - with open(requirements_file, 'w', encoding="utf-8") as fo: + with open(requirements_file, 'w') as fo: fo.write('\n'.join(hashed)) ################################################################################ @@ -2915,7 +2926,7 @@ def fetch_package_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR): def check_about(dest_dir=THIRDPARTY_DIR): try: - subprocess.check_output(f'venv/bin/about check {dest_dir}'.split()) + subprocess.check_output(f'about check {dest_dir}'.split()) except subprocess.CalledProcessError as cpe: print() print('Invalid ABOUT files:') @@ -2953,7 +2964,6 @@ def find_problems( check_about(dest_dir=dest_dir) - def compute_normalized_license_expression(declared_licenses): if not declared_licenses: return @@ -2962,4 +2972,4 @@ def compute_normalized_license_expression(declared_licenses): return pypi.compute_normalized_license(declared_licenses) except ImportError: # Scancode is not installed, we join all license strings and return it - return ' '.join(declared_licenses) + return ' '.join(declared_licenses).lower() From 255a898ceb3c440ad38c38d08dab7e2e9463771b Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 11 Oct 2021 22:29:48 +0200 Subject: [PATCH 144/160] Handle as_text correctly in cache Signed-off-by: Philippe Ombredanne --- etc/scripts/utils_thirdparty.py | 1 - 1 file changed, 1 deletion(-) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 7613a0c7..6b268cae 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -2965,7 +2965,6 @@ def find_problems( check_about(dest_dir=dest_dir) - def compute_normalized_license_expression(declared_licenses): if not declared_licenses: return From af4a2caf1dd258bef3e42a67911223b8b97c8696 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 15 Oct 2021 15:45:02 +0200 Subject: [PATCH 145/160] Add missing version constraint Signed-off-by: Philippe Ombredanne --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index ab646c1e..240e5a20 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -9,3 +9,4 @@ pytest-xdist==2.4.0 toml==0.10.2 typing-extensions==3.10.0.2 zipp==3.6.0 +black==21.9b0 From a46e34e2a9f8a13ab58093306ae19c4671593079 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 15 Oct 2021 15:45:15 +0200 Subject: [PATCH 146/160] Improve README Signed-off-by: Philippe Ombredanne --- README.rst | 94 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 69 insertions(+), 25 deletions(-) diff --git a/README.rst b/README.rst index 83ea7f2d..921fa736 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -Univers: Ecosystem specific version comparision and conversion -============================================================== +Univers: mostly universal version and version ranges comparison and conversion +=============================================================================== |Build Status| |License| |Python 3.6+| @@ -11,43 +11,76 @@ Univers: Ecosystem specific version comparision and conversion -Why ecosystem specific ? -======================== +univers was born out of the need for a mostly univeral way to perform software +package version comparisons in VulnerableCode. -Univers was born out of the need for version comparision at VulnerableCode. Existing -tools follow a particular algorithm to evaluate and compare versions. This is not -accurate across different ecosystems, since they follow different versioning rules. For -example there's no concept of 'epoch' in semver based ecosystem like npm or ruby gems, but -epochs do exist in debian ecosystem. The tools solely based on semver or dpkg version spec therefore -give different and wrong results in both cases. +Package version ranges and version constraints are useful and essential: -Univers is different, it considers the ecosystem of version. +- When resolving the dependencies of a package to express which subset of the + versions are supported. For instance a dependency requirement statement such + as "I require package foo, version 2.0 and later versions" defines a range of + acceptable foo versions. -How Univers works ? -=================== +- When relating a known vulnerability or bug to a range of affected package + versions. For instance a statement such as "vulnerability 123 affects + package bar, version 3.1 and version 4.2 but not version 5" also defines a + range of affected bar versions. -Univers, can be considered as a wrapper around many version comparision libraries, each of which -solving the problem for the respective ecosystem. It delegates the actual comparision to these libraries -depending upon the ecosystem. +Existing tools support typically a single algorithm to parse and compare +versions and this is not accurate across different ecosystems, since each +follow different versioning rules. For example there's no concept of 'epoch' in +semver versioning as used in package types and ecosystem such as npm or +rubygems, but epochs do exist in debian versions. A tool designed for semver or +dpkg versions processing would not be able to handle correctly the other version +scheme. +univers is different and considers the ecosystem-specific version scheme used. -The supported ecosystems and underlying libraries are: -- npm, golang, php-composer, ruby-gems and others which follow the semver spec. These use `semantic_version `_ library. -- debian, this is handled by `debian-inspector `_ library. -- pypi, this is handled by Python's ``packaging.version`` module. -- maven, this is handled by `rpm_vercmp `_ library. -- ebuild/gentoo, this is handled by `gentoo_vercmp `_ module. +How does univers work ? +========================= + +Univers wraps, embeds or implements multiple version comparision libraries, each +focused on specific ecosystem version scheme. + +It also implements an experimental unified syntax for version ranges specifier +and can parse and convert existing version range strings to this unified syntax. + + +The supported package ecosystems versioning schemes and underlying libraries are: + +- semver: npm, golang, PHP composer, rubygems and others that follow the semver + spec, using `semantic_version `_ library. +- debian: handled by the + `debian-inspector `_ + library. +- pypi: handled by Python's packaging library and the standard ``packaging.version`` module. +- maven: handled by the embedded `rpm_vercmp `_ library. +- ebuild/gentoo: handled by the embedded `gentoo_vercmp `_ module. + +As we grow, new schemes will be implemented accordingly. + + +Alternative +============ + +Rather than using ecosystem-specific version schemes and code, another approach +is to use a single procedure for all the versions as implemented in `libversion +`_. This works in the most common case +but may not work correctly for specific tasks that demand accurate version +comparison such as for dependency resolution and vulnerabilities checks. + Installation ============ - $ pip install univers + $ pip install univers + Examples ======== -Comparing discrete versions +Compare two versions using the Python comparison operators: .. code:: python @@ -57,7 +90,7 @@ Comparing discrete versions assert v1 < v2 == True -Evaluating version ranges +Test if a version is within or outside of a version range: .. code:: python @@ -79,3 +112,14 @@ Starting from a git clone of https://github.com/nexB/univers run these:: $ configure --dev $ source venv/bin/active + $ pytest -vvs + + +We use the same development process as other AboutCode projects. + +Visit https://github.com/nexB/univers and +https://gitter.im/aboutcode-org/vulnerablecode and +https://gitter.im/aboutcode-org/aboutcode for support and chat. + +Primary license: Apache-2.0 +SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause AND MIT From 0b787d8f320a97ce1b63912b6f92146307a1e89e Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 15 Oct 2021 15:45:33 +0200 Subject: [PATCH 147/160] Add black to test/devel utils Signed-off-by: Philippe Ombredanne --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 0b2d1b75..b369de13 100644 --- a/setup.cfg +++ b/setup.cfg @@ -63,6 +63,7 @@ testing = # upstream pytest >= 6 pytest-xdist >= 2 + black docs= Sphinx>=3.3.1 sphinx-rtd-theme>=0.5.0 From e5833d13d8493af3ee385c63b76d4bd55aacbe15 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 18 Oct 2021 13:53:58 +0200 Subject: [PATCH 148/160] Add support for Python 3.10 Signed-off-by: Philippe Ombredanne --- etc/scripts/README.rst | 12 ++++-------- etc/scripts/utils_thirdparty.py | 7 ++++++- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/etc/scripts/README.rst b/etc/scripts/README.rst index 4cb6ec7d..d8b00f98 100755 --- a/etc/scripts/README.rst +++ b/etc/scripts/README.rst @@ -1,10 +1,6 @@ -This directory contains the tools to: - -- manage a directory of thirdparty Python package source, wheels and metadata: - pin, build, update, document and publish to a PyPI-like repo (GitHub release) - -- build and publish scancode releases as wheel, sources and OS-specific bundles. - +This directory contains the tools to manage a directory of thirdparty Python +package source, wheels and metadata pin, build, update, document and publish to +a PyPI-like repo (GitHub release). NOTE: These are tested to run ONLY on Linux. @@ -38,7 +34,7 @@ Scripts ~~~~~~~ **gen_requirements.py**: create/update requirements files from currently - installed requirements. + installed requirements. **gen_requirements_dev.py** does the same but can subtract the main requirements to get extra requirements used in only development. diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 5cac5364..444b20dd 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -87,13 +87,14 @@ TRACE = False # Supported environments -PYTHON_VERSIONS = '36', '37', '38', '39', +PYTHON_VERSIONS = '36', '37', '38', '39', '310' ABIS_BY_PYTHON_VERSION = { '36':['cp36', 'cp36m'], '37':['cp37', 'cp37m'], '38':['cp38', 'cp38m'], '39':['cp39', 'cp39m'], + '310':['cp310', 'cp310m'], } PLATFORMS_BY_OS = { @@ -102,6 +103,7 @@ 'manylinux1_x86_64', 'manylinux2014_x86_64', 'manylinux2010_x86_64', + 'manylinux_2_12_x86_64', ], 'macos': [ 'macosx_10_6_intel', 'macosx_10_6_x86_64', @@ -112,6 +114,9 @@ 'macosx_10_13_intel', 'macosx_10_13_x86_64', 'macosx_10_14_intel', 'macosx_10_14_x86_64', 'macosx_10_15_intel', 'macosx_10_15_x86_64', + 'macosx_10_15_x86_64', + 'macosx_11_0_x86_64', + # 'macosx_11_0_arm64', ], 'windows': [ 'win_amd64', From 0a0ef125bfad078529070e7ee0e6caf6af70b331 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 29 Oct 2021 17:37:15 +0200 Subject: [PATCH 149/160] Adopt black style Signed-off-by: Philippe Ombredanne --- docs/source/conf.py | 23 +- etc/scripts/bootstrap.py | 122 ++- etc/scripts/build_wheels.py | 60 +- etc/scripts/check_thirdparty.py | 11 +- etc/scripts/fetch_requirements.py | 80 +- etc/scripts/fix_thirdparty.py | 38 +- etc/scripts/gen_pypi_simple.py | 20 +- etc/scripts/gen_requirements.py | 21 +- etc/scripts/gen_requirements_dev.py | 35 +- etc/scripts/publish_files.py | 90 +- .../test_utils_pip_compatibility_tags.py | 70 +- etc/scripts/test_utils_pypi_supported_tags.py | 1 + etc/scripts/utils_dejacode.py | 78 +- etc/scripts/utils_pip_compatibility_tags.py | 28 +- etc/scripts/utils_pypi_supported_tags.py | 6 +- etc/scripts/utils_requirements.py | 28 +- etc/scripts/utils_thirdparty.py | 960 ++++++++++-------- 17 files changed, 930 insertions(+), 741 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 529cae39..b792d9f2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,9 +17,9 @@ # -- Project information ----------------------------------------------------- -project = 'nexb-skeleton' -copyright = 'nexb Inc.' -author = 'nexb Inc.' +project = "nexb-skeleton" +copyright = "nexb Inc." +author = "nexb Inc." # -- General configuration --------------------------------------------------- @@ -27,11 +27,10 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = [ -] +extensions = [] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -44,20 +43,20 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] html_context = { - 'css_files': [ - '_static/theme_overrides.css', # override wide tables in RTD theme - ], + "css_files": [ + "_static/theme_overrides.css", # override wide tables in RTD theme + ], "display_github": True, "github_user": "nexB", "github_repo": "nexb-skeleton", "github_version": "develop", # branch "conf_py_path": "/docs/source/", # path in the checkout to the docs root - } \ No newline at end of file +} diff --git a/etc/scripts/bootstrap.py b/etc/scripts/bootstrap.py index fde505bc..31f2f553 100644 --- a/etc/scripts/bootstrap.py +++ b/etc/scripts/bootstrap.py @@ -19,52 +19,63 @@ @click.command() - -@click.option('-r', '--requirements-file', +@click.option( + "-r", + "--requirements-file", type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), - metavar='FILE', + metavar="FILE", multiple=True, - default=['requirements.txt'], + default=["requirements.txt"], show_default=True, - help='Path to the requirements file(s) to use for thirdparty packages.', + help="Path to the requirements file(s) to use for thirdparty packages.", ) -@click.option('-d', '--thirdparty-dir', +@click.option( + "-d", + "--thirdparty-dir", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar='DIR', + metavar="DIR", default=utils_thirdparty.THIRDPARTY_DIR, show_default=True, - help='Path to the thirdparty directory where wheels are built and ' - 'sources, ABOUT and LICENSE files fetched.', + help="Path to the thirdparty directory where wheels are built and " + "sources, ABOUT and LICENSE files fetched.", ) -@click.option('-p', '--python-version', +@click.option( + "-p", + "--python-version", type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), - metavar='PYVER', + metavar="PYVER", default=utils_thirdparty.PYTHON_VERSIONS, show_default=True, multiple=True, - help='Python version(s) to use for this build.', + help="Python version(s) to use for this build.", ) -@click.option('-o', '--operating-system', +@click.option( + "-o", + "--operating-system", type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), - metavar='OS', + metavar="OS", default=tuple(utils_thirdparty.PLATFORMS_BY_OS), multiple=True, show_default=True, - help='OS(ses) to use for this build: one of linux, mac or windows.', + help="OS(ses) to use for this build: one of linux, mac or windows.", ) -@click.option('-l', '--latest-version', +@click.option( + "-l", + "--latest-version", is_flag=True, - help='Get the latest version of all packages, ignoring version specifiers.', + help="Get the latest version of all packages, ignoring version specifiers.", ) -@click.option('--sync-dejacode', +@click.option( + "--sync-dejacode", is_flag=True, - help='Synchronize packages with DejaCode.', + help="Synchronize packages with DejaCode.", ) -@click.option('--with-deps', +@click.option( + "--with-deps", is_flag=True, - help='Also include all dependent wheels.', + help="Also include all dependent wheels.", ) -@click.help_option('-h', '--help') +@click.help_option("-h", "--help") def bootstrap( requirements_file, thirdparty_dir, @@ -105,18 +116,19 @@ def bootstrap( required_name_versions = set() for req_file in requirements_files: - nvs = utils_thirdparty.load_requirements( - requirements_file=req_file, force_pinned=False) + nvs = utils_thirdparty.load_requirements(requirements_file=req_file, force_pinned=False) required_name_versions.update(nvs) if latest_version: required_name_versions = set((name, None) for name, _ver in required_name_versions) - print(f'PROCESSING {len(required_name_versions)} REQUIREMENTS in {len(requirements_files)} FILES') + print( + f"PROCESSING {len(required_name_versions)} REQUIREMENTS in {len(requirements_files)} FILES" + ) # fetch all available wheels, keep track of missing # start with local, then remote, then PyPI - print('==> COLLECTING ALREADY LOCALLY AVAILABLE REQUIRED WHEELS') + print("==> COLLECTING ALREADY LOCALLY AVAILABLE REQUIRED WHEELS") # list of all the wheel filenames either pre-existing, fetched or built # updated as we progress available_wheel_filenames = [] @@ -131,19 +143,32 @@ def bootstrap( # start with a local check for (name, version), envt in itertools.product(required_name_versions, environments): - local_pack = local_packages_by_namever.get((name, version,)) + local_pack = local_packages_by_namever.get( + ( + name, + version, + ) + ) if local_pack: supported_wheels = list(local_pack.get_supported_wheels(environment=envt)) if supported_wheels: available_wheel_filenames.extend(w.filename for w in supported_wheels) - print(f'====> No fetch or build needed. ' - f'Local wheel already available for {name}=={version} ' - f'on os: {envt.operating_system} for Python: {envt.python_version}') + print( + f"====> No fetch or build needed. " + f"Local wheel already available for {name}=={version} " + f"on os: {envt.operating_system} for Python: {envt.python_version}" + ) continue - name_version_envt_to_fetch.append((name, version, envt,)) + name_version_envt_to_fetch.append( + ( + name, + version, + envt, + ) + ) - print(f'==> TRYING TO FETCH #{len(name_version_envt_to_fetch)} REQUIRED WHEELS') + print(f"==> TRYING TO FETCH #{len(name_version_envt_to_fetch)} REQUIRED WHEELS") # list of (name, version, environment) not fetch and to build name_version_envt_to_build = [] @@ -161,46 +186,53 @@ def bootstrap( if fetched_fwn: available_wheel_filenames.append(fetched_fwn) else: - name_version_envt_to_build.append((name, version, envt,)) + name_version_envt_to_build.append( + ( + name, + version, + envt, + ) + ) # At this stage we have all the wheels we could obtain without building for name, version, envt in name_version_envt_to_build: - print(f'====> Need to build wheels for {name}=={version} on os: ' - f'{envt.operating_system} for Python: {envt.python_version}') + print( + f"====> Need to build wheels for {name}=={version} on os: " + f"{envt.operating_system} for Python: {envt.python_version}" + ) packages_and_envts_to_build = [ - (PypiPackage(name, version), envt) - for name, version, envt in name_version_envt_to_build + (PypiPackage(name, version), envt) for name, version, envt in name_version_envt_to_build ] - print(f'==> BUILDING #{len(packages_and_envts_to_build)} MISSING WHEELS') + print(f"==> BUILDING #{len(packages_and_envts_to_build)} MISSING WHEELS") package_envts_not_built, wheel_filenames_built = utils_thirdparty.build_missing_wheels( packages_and_envts=packages_and_envts_to_build, build_remotely=build_remotely, with_deps=with_deps, dest_dir=thirdparty_dir, -) + ) if wheel_filenames_built: available_wheel_filenames.extend(available_wheel_filenames) for pack, envt in package_envts_not_built: print( - f'====> FAILED to build any wheel for {pack.name}=={pack.version} ' - f'on os: {envt.operating_system} for Python: {envt.python_version}' + f"====> FAILED to build any wheel for {pack.name}=={pack.version} " + f"on os: {envt.operating_system} for Python: {envt.python_version}" ) - print(f'==> FETCHING SOURCE DISTRIBUTIONS') + print(f"==> FETCHING SOURCE DISTRIBUTIONS") # fetch all sources, keep track of missing # This is a list of (name, version) utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir) - print(f'==> FETCHING ABOUT AND LICENSE FILES') + print(f"==> FETCHING ABOUT AND LICENSE FILES") utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir) ############################################################################ if sync_dejacode: - print(f'==> SYNC WITH DEJACODE') + print(f"==> SYNC WITH DEJACODE") # try to fetch from DejaCode any missing ABOUT # create all missing DejaCode packages pass @@ -208,5 +240,5 @@ def bootstrap( utils_thirdparty.find_problems(dest_dir=thirdparty_dir) -if __name__ == '__main__': +if __name__ == "__main__": bootstrap() diff --git a/etc/scripts/build_wheels.py b/etc/scripts/build_wheels.py index 352b7055..5a39c78a 100644 --- a/etc/scripts/build_wheels.py +++ b/etc/scripts/build_wheels.py @@ -14,55 +14,67 @@ @click.command() - -@click.option('-n', '--name', +@click.option( + "-n", + "--name", type=str, - metavar='PACKAGE_NAME', + metavar="PACKAGE_NAME", required=True, - help='Python package name to add or build.', + help="Python package name to add or build.", ) -@click.option('-v', '--version', +@click.option( + "-v", + "--version", type=str, default=None, - metavar='VERSION', - help='Python package version to add or build.', + metavar="VERSION", + help="Python package version to add or build.", ) -@click.option('-d', '--thirdparty-dir', +@click.option( + "-d", + "--thirdparty-dir", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar='DIR', + metavar="DIR", default=utils_thirdparty.THIRDPARTY_DIR, show_default=True, - help='Path to the thirdparty directory where wheels are built.', + help="Path to the thirdparty directory where wheels are built.", ) -@click.option('-p', '--python-version', +@click.option( + "-p", + "--python-version", type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), - metavar='PYVER', + metavar="PYVER", default=utils_thirdparty.PYTHON_VERSIONS, show_default=True, multiple=True, - help='Python version to use for this build.', + help="Python version to use for this build.", ) -@click.option('-o', '--operating-system', +@click.option( + "-o", + "--operating-system", type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), - metavar='OS', + metavar="OS", default=tuple(utils_thirdparty.PLATFORMS_BY_OS), multiple=True, show_default=True, - help='OS to use for this build: one of linux, mac or windows.', + help="OS to use for this build: one of linux, mac or windows.", ) -@click.option('--build-remotely', +@click.option( + "--build-remotely", is_flag=True, - help='Build missing wheels remotely.', + help="Build missing wheels remotely.", ) -@click.option('--with-deps', +@click.option( + "--with-deps", is_flag=True, - help='Also include all dependent wheels.', + help="Also include all dependent wheels.", ) -@click.option('--verbose', +@click.option( + "--verbose", is_flag=True, - help='Provide verbose output.', + help="Provide verbose output.", ) -@click.help_option('-h', '--help') +@click.help_option("-h", "--help") def build_wheels( name, version, @@ -93,5 +105,5 @@ def build_wheels( ) -if __name__ == '__main__': +if __name__ == "__main__": build_wheels() diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index e48cfce3..4fea16c5 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -14,13 +14,14 @@ @click.command() - -@click.option('-d', '--thirdparty-dir', +@click.option( + "-d", + "--thirdparty-dir", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), required=True, - help='Path to the thirdparty directory to check.', + help="Path to the thirdparty directory to check.", ) -@click.help_option('-h', '--help') +@click.help_option("-h", "--help") def check_thirdparty_dir(thirdparty_dir): """ Check a thirdparty directory for problems. @@ -28,5 +29,5 @@ def check_thirdparty_dir(thirdparty_dir): utils_thirdparty.find_problems(dest_dir=thirdparty_dir) -if __name__ == '__main__': +if __name__ == "__main__": check_thirdparty_dir() diff --git a/etc/scripts/fetch_requirements.py b/etc/scripts/fetch_requirements.py index 21de865b..9da9ce96 100644 --- a/etc/scripts/fetch_requirements.py +++ b/etc/scripts/fetch_requirements.py @@ -16,64 +16,78 @@ @click.command() - -@click.option('-r', '--requirements-file', +@click.option( + "-r", + "--requirements-file", type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), - metavar='FILE', + metavar="FILE", multiple=True, - default=['requirements.txt'], + default=["requirements.txt"], show_default=True, - help='Path to the requirements file to use for thirdparty packages.', + help="Path to the requirements file to use for thirdparty packages.", ) -@click.option('-d', '--thirdparty-dir', +@click.option( + "-d", + "--thirdparty-dir", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar='DIR', + metavar="DIR", default=utils_thirdparty.THIRDPARTY_DIR, show_default=True, - help='Path to the thirdparty directory.', + help="Path to the thirdparty directory.", ) -@click.option('-p', '--python-version', +@click.option( + "-p", + "--python-version", type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), - metavar='INT', + metavar="INT", multiple=True, - default=['36'], + default=["36"], show_default=True, - help='Python version to use for this build.', + help="Python version to use for this build.", ) -@click.option('-o', '--operating-system', +@click.option( + "-o", + "--operating-system", type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), - metavar='OS', + metavar="OS", multiple=True, - default=['linux'], + default=["linux"], show_default=True, - help='OS to use for this build: one of linux, mac or windows.', + help="OS to use for this build: one of linux, mac or windows.", ) -@click.option('-s', '--with-sources', +@click.option( + "-s", + "--with-sources", is_flag=True, - help='Fetch the corresponding source distributions.', + help="Fetch the corresponding source distributions.", ) -@click.option('-a', '--with-about', +@click.option( + "-a", + "--with-about", is_flag=True, - help='Fetch the corresponding ABOUT and LICENSE files.', + help="Fetch the corresponding ABOUT and LICENSE files.", ) -@click.option('--allow-unpinned', +@click.option( + "--allow-unpinned", is_flag=True, - help='Allow requirements without pinned versions.', + help="Allow requirements without pinned versions.", ) -@click.option('-s', '--only-sources', +@click.option( + "-s", + "--only-sources", is_flag=True, - help='Fetch only the corresponding source distributions.', + help="Fetch only the corresponding source distributions.", ) -@click.option('-u', '--remote-links-url', +@click.option( + "-u", + "--remote-links-url", type=str, - metavar='URL', + metavar="URL", default=utils_thirdparty.REMOTE_LINKS_URL, show_default=True, - help='URL to a PyPI-like links web site. ' - 'Or local path to a directory with wheels.', + help="URL to a PyPI-like links web site. " "Or local path to a directory with wheels.", ) - -@click.help_option('-h', '--help') +@click.help_option("-h", "--help") def fetch_requirements( requirements_file, thirdparty_dir, @@ -117,7 +131,7 @@ def fetch_requirements( remote_links_url=remote_links_url, ): if error: - print('Failed to fetch wheel:', package, ':', error) + print("Failed to fetch wheel:", package, ":", error) # optionally fetch sources if with_sources or only_sources: @@ -130,7 +144,7 @@ def fetch_requirements( remote_links_url=remote_links_url, ): if error: - print('Failed to fetch source:', package, ':', error) + print("Failed to fetch source:", package, ":", error) if with_about: utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir) @@ -141,5 +155,5 @@ def fetch_requirements( ) -if __name__ == '__main__': +if __name__ == "__main__": fetch_requirements() diff --git a/etc/scripts/fix_thirdparty.py b/etc/scripts/fix_thirdparty.py index 061d3fac..9b1cbc49 100644 --- a/etc/scripts/fix_thirdparty.py +++ b/etc/scripts/fix_thirdparty.py @@ -14,21 +14,24 @@ @click.command() - -@click.option('-d', '--thirdparty-dir', +@click.option( + "-d", + "--thirdparty-dir", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), required=True, - help='Path to the thirdparty directory to fix.', + help="Path to the thirdparty directory to fix.", ) -@click.option('--build-wheels', +@click.option( + "--build-wheels", is_flag=True, - help='Build all missing wheels .', + help="Build all missing wheels .", ) -@click.option('--build-remotely', +@click.option( + "--build-remotely", is_flag=True, - help='Build missing wheels remotely.', + help="Build missing wheels remotely.", ) -@click.help_option('-h', '--help') +@click.help_option("-h", "--help") def fix_thirdparty_dir( thirdparty_dir, build_wheels, @@ -47,35 +50,36 @@ def fix_thirdparty_dir( Optionally build missing binary wheels for all supported OS and Python version combos locally or remotely. """ - print('***FETCH*** MISSING WHEELS') + print("***FETCH*** MISSING WHEELS") package_envts_not_fetched = utils_thirdparty.fetch_missing_wheels(dest_dir=thirdparty_dir) - print('***FETCH*** MISSING SOURCES') + print("***FETCH*** MISSING SOURCES") src_name_ver_not_fetched = utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir) package_envts_not_built = [] if build_wheels: - print('***BUILD*** MISSING WHEELS') + print("***BUILD*** MISSING WHEELS") package_envts_not_built, _wheel_filenames_built = utils_thirdparty.build_missing_wheels( packages_and_envts=package_envts_not_fetched, build_remotely=build_remotely, dest_dir=thirdparty_dir, ) - print('***ADD*** ABOUT AND LICENSES') + print("***ADD*** ABOUT AND LICENSES") utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir) # report issues for name, version in src_name_ver_not_fetched: - print(f'{name}=={version}: Failed to fetch source distribution.') + print(f"{name}=={version}: Failed to fetch source distribution.") for package, envt in package_envts_not_built: print( - f'{package.name}=={package.version}: Failed to build wheel ' - f'on {envt.operating_system} for Python {envt.python_version}') + f"{package.name}=={package.version}: Failed to build wheel " + f"on {envt.operating_system} for Python {envt.python_version}" + ) - print('***FIND PROBLEMS***') + print("***FIND PROBLEMS***") utils_thirdparty.find_problems(dest_dir=thirdparty_dir) -if __name__ == '__main__': +if __name__ == "__main__": fix_thirdparty_dir() diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py index 887e407a..53db9b0a 100644 --- a/etc/scripts/gen_pypi_simple.py +++ b/etc/scripts/gen_pypi_simple.py @@ -45,10 +45,15 @@ r"""^(?P(?P.+?)-(?P.*?)) ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) \.whl)$""", - re.VERBOSE + re.VERBOSE, ).match -sdist_exts = ".tar.gz", ".tar.bz2", ".zip", ".tar.xz", +sdist_exts = ( + ".tar.gz", + ".tar.bz2", + ".zip", + ".tar.xz", +) wheel_ext = ".whl" app_ext = ".pyz" dist_exts = sdist_exts + (wheel_ext, app_ext) @@ -98,7 +103,7 @@ def get_package_name_from_filename(filename, normalize=True): if not extension or not name_ver: raise InvalidDistributionFilename(filename) - name, _, version = name_ver.rpartition('-') + name, _, version = name_ver.rpartition("-") if not (name and version): raise InvalidDistributionFilename(filename) @@ -110,8 +115,8 @@ def get_package_name_from_filename(filename, normalize=True): if not wheel_info: raise InvalidDistributionFilename(filename) - name = wheel_info.group('name') - version = wheel_info.group('version') + name = wheel_info.group("name") + version = wheel_info.group("version") if not (name and version): raise InvalidDistributionFilename(filename) @@ -120,7 +125,7 @@ def get_package_name_from_filename(filename, normalize=True): name_ver, extension, _ = filename.rpartition(".pyz") if "-" in filename: - name, _, version = name_ver.rpartition('-') + name, _, version = name_ver.rpartition("-") else: name = name_ver @@ -128,7 +133,7 @@ def get_package_name_from_filename(filename, normalize=True): raise InvalidDistributionFilename(filename) if normalize: - name = name.lower().replace('_', '-') + name = name.lower().replace("_", "-") return name @@ -187,5 +192,6 @@ def build_pypi_index(directory, write_index=False): if __name__ == "__main__": import sys + pkg_dir = sys.argv[1] build_pypi_index(pkg_dir) diff --git a/etc/scripts/gen_requirements.py b/etc/scripts/gen_requirements.py index 3be974cc..6f17a75f 100644 --- a/etc/scripts/gen_requirements.py +++ b/etc/scripts/gen_requirements.py @@ -13,21 +13,24 @@ @click.command() - -@click.option('-s', '--site-packages-dir', +@click.option( + "-s", + "--site-packages-dir", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True), required=True, - metavar='DIR', + metavar="DIR", help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages', ) -@click.option('-r', '--requirements-file', +@click.option( + "-r", + "--requirements-file", type=click.Path(path_type=str, dir_okay=False), - metavar='FILE', - default='requirements.txt', + metavar="FILE", + default="requirements.txt", show_default=True, - help='Path to the requirements file to update or create.', + help="Path to the requirements file to update or create.", ) -@click.help_option('-h', '--help') +@click.help_option("-h", "--help") def gen_requirements(site_packages_dir, requirements_file): """ Create or replace the `--requirements-file` file FILE requirements file with all @@ -39,5 +42,5 @@ def gen_requirements(site_packages_dir, requirements_file): ) -if __name__ == '__main__': +if __name__ == "__main__": gen_requirements() diff --git a/etc/scripts/gen_requirements_dev.py b/etc/scripts/gen_requirements_dev.py index ff4ce500..ef804554 100644 --- a/etc/scripts/gen_requirements_dev.py +++ b/etc/scripts/gen_requirements_dev.py @@ -13,29 +13,34 @@ @click.command() - -@click.option('-s', '--site-packages-dir', +@click.option( + "-s", + "--site-packages-dir", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True), required=True, - metavar='DIR', + metavar="DIR", help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages', ) -@click.option('-d', '--dev-requirements-file', +@click.option( + "-d", + "--dev-requirements-file", type=click.Path(path_type=str, dir_okay=False), - metavar='FILE', - default='requirements-dev.txt', + metavar="FILE", + default="requirements-dev.txt", show_default=True, - help='Path to the dev requirements file to update or create.', + help="Path to the dev requirements file to update or create.", ) -@click.option('-r', '--main-requirements-file', +@click.option( + "-r", + "--main-requirements-file", type=click.Path(path_type=str, dir_okay=False), - default='requirements.txt', - metavar='FILE', + default="requirements.txt", + metavar="FILE", show_default=True, - help='Path to the main requirements file. Its requirements will be excluded ' - 'from the generated dev requirements.', + help="Path to the main requirements file. Its requirements will be excluded " + "from the generated dev requirements.", ) -@click.help_option('-h', '--help') +@click.help_option("-h", "--help") def gen_dev_requirements(site_packages_dir, dev_requirements_file, main_requirements_file): """ Create or overwrite the `--dev-requirements-file` pip requirements FILE with @@ -47,9 +52,9 @@ def gen_dev_requirements(site_packages_dir, dev_requirements_file, main_requirem utils_requirements.lock_dev_requirements( dev_requirements_file=dev_requirements_file, main_requirements_file=main_requirements_file, - site_packages_dir=site_packages_dir + site_packages_dir=site_packages_dir, ) -if __name__ == '__main__': +if __name__ == "__main__": gen_dev_requirements() diff --git a/etc/scripts/publish_files.py b/etc/scripts/publish_files.py index f343cb3c..86693631 100644 --- a/etc/scripts/publish_files.py +++ b/etc/scripts/publish_files.py @@ -32,7 +32,7 @@ def get_files(location): for top, _dirs, files in os.walk(location): for filename in files: pth = Path(os.path.join(top, filename)) - with open(pth, 'rb') as fi: + with open(pth, "rb") as fi: md5 = hashlib.md5(fi.read()).hexdigest() yield filename, pth, md5 @@ -43,20 +43,20 @@ def get_etag_md5(url): """ headers = utils_thirdparty.get_remote_headers(url) headers = {k.lower(): v for k, v in headers.items()} - etag = headers .get('etag') + etag = headers.get("etag") if etag: etag = etag.strip('"').lower() return etag def create_or_update_release_and_upload_directory( - user, - repo, - tag_name, - token, - directory, - retry_limit=10, - description=None, + user, + repo, + tag_name, + token, + directory, + retry_limit=10, + description=None, ): """ Create or update a GitHub release at https://github.com// for @@ -69,15 +69,16 @@ def create_or_update_release_and_upload_directory( Remote files that are not the same as the local files are deleted and re- uploaded. """ - release_homepage_url = f'https://github.com/{user}/{repo}/releases/{tag_name}' + release_homepage_url = f"https://github.com/{user}/{repo}/releases/{tag_name}" # scrape release page HTML for links - urls_by_filename = {os.path.basename(l): l + urls_by_filename = { + os.path.basename(l): l for l in utils_thirdparty.get_paths_or_urls(links_url=release_homepage_url) } # compute what is new, modified or unchanged - print(f'Compute which files is new, modified or unchanged in {release_homepage_url}') + print(f"Compute which files is new, modified or unchanged in {release_homepage_url}") new_to_upload = [] unchanged_to_skip = [] @@ -85,21 +86,21 @@ def create_or_update_release_and_upload_directory( for filename, pth, md5 in get_files(directory): url = urls_by_filename.get(filename) if not url: - print(f'{filename} content is NEW, will upload') + print(f"{filename} content is NEW, will upload") new_to_upload.append(pth) continue out_of_date = get_etag_md5(url) != md5 if out_of_date: - print(f'{url} content is CHANGED based on md5 etag, will re-upload') + print(f"{url} content is CHANGED based on md5 etag, will re-upload") modified_to_delete_and_reupload.append(pth) else: # print(f'{url} content is IDENTICAL, skipping upload based on Etag') unchanged_to_skip.append(pth) - print('.') + print(".") ghapi = grr.GithubApi( - github_api_url='https://api.github.com', + github_api_url="https://api.github.com", user=user, repo=repo, token=token, @@ -108,86 +109,89 @@ def create_or_update_release_and_upload_directory( # yank modified print( - f'Unpublishing {len(modified_to_delete_and_reupload)} published but ' - f'locally modified files in {release_homepage_url}') + f"Unpublishing {len(modified_to_delete_and_reupload)} published but " + f"locally modified files in {release_homepage_url}" + ) release = ghapi.get_release_by_tag(tag_name) for pth in modified_to_delete_and_reupload: filename = os.path.basename(pth) asset_id = ghapi.find_asset_id_by_file_name(filename, release) - print (f' Unpublishing file: {filename}).') + print(f" Unpublishing file: {filename}).") response = ghapi.delete_asset(asset_id) if response.status_code != requests.codes.no_content: # NOQA - raise Exception(f'failed asset deletion: {response}') + raise Exception(f"failed asset deletion: {response}") # finally upload new and modified to_upload = new_to_upload + modified_to_delete_and_reupload - print(f'Publishing with {len(to_upload)} files to {release_homepage_url}') + print(f"Publishing with {len(to_upload)} files to {release_homepage_url}") release = grr.Release(tag_name=tag_name, body=description) grr.make_release(ghapi, release, to_upload) TOKEN_HELP = ( - 'The Github personal acess token is used to authenticate API calls. ' - 'Required unless you set the GITHUB_TOKEN environment variable as an alternative. ' - 'See for details: https://github.com/settings/tokens and ' - 'https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token' + "The Github personal acess token is used to authenticate API calls. " + "Required unless you set the GITHUB_TOKEN environment variable as an alternative. " + "See for details: https://github.com/settings/tokens and " + "https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token" ) @click.command() - @click.option( - '--user-repo-tag', - help='The GitHub qualified repository user/name/tag in which ' - 'to create the release such as in nexB/thirdparty/pypi', + "--user-repo-tag", + help="The GitHub qualified repository user/name/tag in which " + "to create the release such as in nexB/thirdparty/pypi", type=str, required=True, ) @click.option( - '-d', '--directory', - help='The directory that contains files to upload to the release.', + "-d", + "--directory", + help="The directory that contains files to upload to the release.", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True), required=True, ) @click.option( - '--token', + "--token", help=TOKEN_HELP, - default=os.environ.get('GITHUB_TOKEN', None), + default=os.environ.get("GITHUB_TOKEN", None), type=str, required=False, ) @click.option( - '--description', - help='Text description for the release. Ignored if the release exists.', + "--description", + help="Text description for the release. Ignored if the release exists.", default=None, type=str, required=False, ) @click.option( - '--retry_limit', - help='Number of retries when making failing GitHub API calls. ' - 'Retrying helps work around transient failures of the GitHub API.', + "--retry_limit", + help="Number of retries when making failing GitHub API calls. " + "Retrying helps work around transient failures of the GitHub API.", type=int, default=10, ) -@click.help_option('-h', '--help') +@click.help_option("-h", "--help") def publish_files( user_repo_tag, directory, - retry_limit=10, token=None, description=None, + retry_limit=10, + token=None, + description=None, ): """ Publish all the files in DIRECTORY as assets to a GitHub release. Either create or update/replace remote files' """ if not token: - click.secho('--token required option is missing.') + click.secho("--token required option is missing.") click.secho(TOKEN_HELP) sys.exit(1) - user, repo, tag_name = user_repo_tag.split('/') + user, repo, tag_name = user_repo_tag.split("/") create_or_update_release_and_upload_directory( user=user, @@ -200,5 +204,5 @@ def publish_files( ) -if __name__ == '__main__': +if __name__ == "__main__": publish_files() diff --git a/etc/scripts/test_utils_pip_compatibility_tags.py b/etc/scripts/test_utils_pip_compatibility_tags.py index 30c4ddab..722fa705 100644 --- a/etc/scripts/test_utils_pip_compatibility_tags.py +++ b/etc/scripts/test_utils_pip_compatibility_tags.py @@ -33,23 +33,25 @@ import utils_pip_compatibility_tags -@pytest.mark.parametrize('version_info, expected', [ - ((2,), '2'), - ((2, 8), '28'), - ((3,), '3'), - ((3, 6), '36'), - # Test a tuple of length 3. - ((3, 6, 5), '36'), - # Test a 2-digit minor version. - ((3, 10), '310'), -]) +@pytest.mark.parametrize( + "version_info, expected", + [ + ((2,), "2"), + ((2, 8), "28"), + ((3,), "3"), + ((3, 6), "36"), + # Test a tuple of length 3. + ((3, 6, 5), "36"), + # Test a 2-digit minor version. + ((3, 10), "310"), + ], +) def test_version_info_to_nodot(version_info, expected): actual = pip_compatibility_tags.version_info_to_nodot(version_info) assert actual == expected class Testcompatibility_tags(object): - def mock_get_config_var(self, **kwd): """ Patch sysconfig.get_config_var for arbitrary keys. @@ -69,23 +71,25 @@ def test_no_hyphen_tag(self): """ import pip._internal.utils.compatibility_tags - mock_gcf = self.mock_get_config_var(SOABI='cpython-35m-darwin') + mock_gcf = self.mock_get_config_var(SOABI="cpython-35m-darwin") - with patch('sysconfig.get_config_var', mock_gcf): + with patch("sysconfig.get_config_var", mock_gcf): supported = pip._internal.utils.compatibility_tags.get_supported() for tag in supported: - assert '-' not in tag.interpreter - assert '-' not in tag.abi - assert '-' not in tag.platform + assert "-" not in tag.interpreter + assert "-" not in tag.abi + assert "-" not in tag.platform class TestManylinux2010Tags(object): - - @pytest.mark.parametrize("manylinux2010,manylinux1", [ - ("manylinux2010_x86_64", "manylinux1_x86_64"), - ("manylinux2010_i686", "manylinux1_i686"), - ]) + @pytest.mark.parametrize( + "manylinux2010,manylinux1", + [ + ("manylinux2010_x86_64", "manylinux1_x86_64"), + ("manylinux2010_i686", "manylinux1_i686"), + ], + ) def test_manylinux2010_implies_manylinux1(self, manylinux2010, manylinux1): """ Specifying manylinux2010 implies manylinux1. @@ -93,22 +97,22 @@ def test_manylinux2010_implies_manylinux1(self, manylinux2010, manylinux1): groups = {} supported = pip_compatibility_tags.get_supported(platforms=[manylinux2010]) for tag in supported: - groups.setdefault( - (tag.interpreter, tag.abi), [] - ).append(tag.platform) + groups.setdefault((tag.interpreter, tag.abi), []).append(tag.platform) for arches in groups.values(): - if arches == ['any']: + if arches == ["any"]: continue assert arches[:2] == [manylinux2010, manylinux1] class TestManylinux2014Tags(object): - - @pytest.mark.parametrize("manylinuxA,manylinuxB", [ - ("manylinux2014_x86_64", ["manylinux2010_x86_64", "manylinux1_x86_64"]), - ("manylinux2014_i686", ["manylinux2010_i686", "manylinux1_i686"]), - ]) + @pytest.mark.parametrize( + "manylinuxA,manylinuxB", + [ + ("manylinux2014_x86_64", ["manylinux2010_x86_64", "manylinux1_x86_64"]), + ("manylinux2014_i686", ["manylinux2010_i686", "manylinux1_i686"]), + ], + ) def test_manylinuxA_implies_manylinuxB(self, manylinuxA, manylinuxB): """ Specifying manylinux2014 implies manylinux2010/manylinux1. @@ -116,13 +120,11 @@ def test_manylinuxA_implies_manylinuxB(self, manylinuxA, manylinuxB): groups = {} supported = pip_compatibility_tags.get_supported(platforms=[manylinuxA]) for tag in supported: - groups.setdefault( - (tag.interpreter, tag.abi), [] - ).append(tag.platform) + groups.setdefault((tag.interpreter, tag.abi), []).append(tag.platform) expected_arches = [manylinuxA] expected_arches.extend(manylinuxB) for arches in groups.values(): - if arches == ['any']: + if arches == ["any"]: continue assert arches[:3] == expected_arches diff --git a/etc/scripts/test_utils_pypi_supported_tags.py b/etc/scripts/test_utils_pypi_supported_tags.py index 9ad68b21..d291572d 100644 --- a/etc/scripts/test_utils_pypi_supported_tags.py +++ b/etc/scripts/test_utils_pypi_supported_tags.py @@ -29,6 +29,7 @@ def validate_wheel_filename_for_pypi(filename): an empty list if all tags are supported. """ from utils_thirdparty import Wheel + wheel = Wheel.from_filename(filename) return validate_platforms_for_pypi(wheel.platforms) diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index 8b6e5d20..f28e2479 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -21,19 +21,19 @@ Utility to create and retrieve package and ABOUT file data from DejaCode. """ -DEJACODE_API_KEY = os.environ.get('DEJACODE_API_KEY', '') -DEJACODE_API_URL = os.environ.get('DEJACODE_API_URL', '') +DEJACODE_API_KEY = os.environ.get("DEJACODE_API_KEY", "") +DEJACODE_API_URL = os.environ.get("DEJACODE_API_URL", "") -DEJACODE_API_URL_PACKAGES = f'{DEJACODE_API_URL}packages/' +DEJACODE_API_URL_PACKAGES = f"{DEJACODE_API_URL}packages/" DEJACODE_API_HEADERS = { - 'Authorization': 'Token {}'.format(DEJACODE_API_KEY), - 'Accept': 'application/json; indent=4', + "Authorization": "Token {}".format(DEJACODE_API_KEY), + "Accept": "application/json; indent=4", } def can_do_api_calls(): if not DEJACODE_API_KEY and DEJACODE_API_URL: - print('DejaCode DEJACODE_API_KEY and DEJACODE_API_URL not configured. Doing nothing') + print("DejaCode DEJACODE_API_KEY and DEJACODE_API_URL not configured. Doing nothing") return False else: return True @@ -53,7 +53,7 @@ def fetch_dejacode_packages(params): headers=DEJACODE_API_HEADERS, ) - return response.json()['results'] + return response.json()["results"] def get_package_data(distribution): @@ -68,9 +68,9 @@ def get_package_data(distribution): return results[0] elif len_results > 1: - print(f'More than 1 entry exists, review at: {DEJACODE_API_URL_PACKAGES}') + print(f"More than 1 entry exists, review at: {DEJACODE_API_URL_PACKAGES}") else: - print('Could not find package:', distribution.download_url) + print("Could not find package:", distribution.download_url) def update_with_dejacode_data(distribution): @@ -82,7 +82,7 @@ def update_with_dejacode_data(distribution): if package_data: return distribution.update(package_data, keep_extra=False) - print(f'No package found for: {distribution}') + print(f"No package found for: {distribution}") def update_with_dejacode_about_data(distribution): @@ -92,19 +92,19 @@ def update_with_dejacode_about_data(distribution): """ package_data = get_package_data(distribution) if package_data: - package_api_url = package_data['api_url'] - about_url = f'{package_api_url}about' + package_api_url = package_data["api_url"] + about_url = f"{package_api_url}about" response = requests.get(about_url, headers=DEJACODE_API_HEADERS) # note that this is YAML-formatted - about_text = response.json()['about_data'] + about_text = response.json()["about_data"] about_data = saneyaml.load(about_text) return distribution.update(about_data, keep_extra=True) - print(f'No package found for: {distribution}') + print(f"No package found for: {distribution}") -def fetch_and_save_about_files(distribution, dest_dir='thirdparty'): +def fetch_and_save_about_files(distribution, dest_dir="thirdparty"): """ Fetch and save in `dest_dir` the .ABOUT, .LICENSE and .NOTICE files fetched from DejaCode for a Distribution `distribution`. Return True if files were @@ -112,8 +112,8 @@ def fetch_and_save_about_files(distribution, dest_dir='thirdparty'): """ package_data = get_package_data(distribution) if package_data: - package_api_url = package_data['api_url'] - about_url = f'{package_api_url}about_files' + package_api_url = package_data["api_url"] + about_url = f"{package_api_url}about_files" response = requests.get(about_url, headers=DEJACODE_API_HEADERS) about_zip = response.content with io.BytesIO(about_zip) as zf: @@ -121,7 +121,7 @@ def fetch_and_save_about_files(distribution, dest_dir='thirdparty'): zi.extractall(path=dest_dir) return True - print(f'No package found for: {distribution}') + print(f"No package found for: {distribution}") def find_latest_dejacode_package(distribution): @@ -138,9 +138,9 @@ def find_latest_dejacode_package(distribution): for package_data in packages: matched = ( - package_data['download_url'] == distribution.download_url - and package_data['version'] == distribution.version - and package_data['filename'] == distribution.filename + package_data["download_url"] == distribution.download_url + and package_data["version"] == distribution.version + and package_data["filename"] == distribution.filename ) if matched: @@ -149,12 +149,11 @@ def find_latest_dejacode_package(distribution): # there was no exact match, find the latest version # TODO: consider the closest version rather than the latest # or the version that has the best data - with_versions = [(packaging_version.parse(p['version']), p) for p in packages] + with_versions = [(packaging_version.parse(p["version"]), p) for p in packages] with_versions = sorted(with_versions) latest_version, latest_package_version = sorted(with_versions)[-1] print( - f'Found DejaCode latest version: {latest_version} ' - f'for dist: {distribution.package_url}', + f"Found DejaCode latest version: {latest_version} " f"for dist: {distribution.package_url}", ) return latest_package_version @@ -172,27 +171,26 @@ def create_dejacode_package(distribution): if existing_package_data: return existing_package_data - print(f'Creating new DejaCode package for: {distribution}') + print(f"Creating new DejaCode package for: {distribution}") new_package_payload = { # Trigger data collection, scan, and purl - 'collect_data': 1, + "collect_data": 1, } fields_to_carry_over = [ - 'download_url' - 'type', - 'namespace', - 'name', - 'version', - 'qualifiers', - 'subpath', - 'license_expression', - 'copyright', - 'description', - 'homepage_url', - 'primary_language', - 'notice_text', + "download_url" "type", + "namespace", + "name", + "version", + "qualifiers", + "subpath", + "license_expression", + "copyright", + "description", + "homepage_url", + "primary_language", + "notice_text", ] for field in fields_to_carry_over: @@ -207,7 +205,7 @@ def create_dejacode_package(distribution): ) new_package_data = response.json() if response.status_code != 201: - raise Exception(f'Error, cannot create package for: {distribution}') + raise Exception(f"Error, cannot create package for: {distribution}") print(f'New Package created at: {new_package_data["absolute_url"]}') return new_package_data diff --git a/etc/scripts/utils_pip_compatibility_tags.py b/etc/scripts/utils_pip_compatibility_tags.py index 4c6529b1..5d5eb34c 100644 --- a/etc/scripts/utils_pip_compatibility_tags.py +++ b/etc/scripts/utils_pip_compatibility_tags.py @@ -36,13 +36,13 @@ mac_platforms, ) -_osx_arch_pat = re.compile(r'(.+)_(\d+)_(\d+)_(.+)') +_osx_arch_pat = re.compile(r"(.+)_(\d+)_(\d+)_(.+)") def version_info_to_nodot(version_info): # type: (Tuple[int, ...]) -> str # Only use up to the first two numbers. - return ''.join(map(str, version_info[:2])) + return "".join(map(str, version_info[:2])) def _mac_platforms(arch): @@ -57,7 +57,7 @@ def _mac_platforms(arch): # actual prefix provided by the user in case they provided # something like "macosxcustom_". It may be good to remove # this as undocumented or deprecate it in the future. - '{}_{}'.format(name, arch[len('macosx_'):]) + "{}_{}".format(name, arch[len("macosx_") :]) for arch in mac_platforms(mac_version, actual_arch) ] else: @@ -69,31 +69,31 @@ def _mac_platforms(arch): def _custom_manylinux_platforms(arch): # type: (str) -> List[str] arches = [arch] - arch_prefix, arch_sep, arch_suffix = arch.partition('_') - if arch_prefix == 'manylinux2014': + arch_prefix, arch_sep, arch_suffix = arch.partition("_") + if arch_prefix == "manylinux2014": # manylinux1/manylinux2010 wheels run on most manylinux2014 systems # with the exception of wheels depending on ncurses. PEP 599 states # manylinux1/manylinux2010 wheels should be considered # manylinux2014 wheels: # https://www.python.org/dev/peps/pep-0599/#backwards-compatibility-with-manylinux2010-wheels - if arch_suffix in {'i686', 'x86_64'}: - arches.append('manylinux2010' + arch_sep + arch_suffix) - arches.append('manylinux1' + arch_sep + arch_suffix) - elif arch_prefix == 'manylinux2010': + if arch_suffix in {"i686", "x86_64"}: + arches.append("manylinux2010" + arch_sep + arch_suffix) + arches.append("manylinux1" + arch_sep + arch_suffix) + elif arch_prefix == "manylinux2010": # manylinux1 wheels run on most manylinux2010 systems with the # exception of wheels depending on ncurses. PEP 571 states # manylinux1 wheels should be considered manylinux2010 wheels: # https://www.python.org/dev/peps/pep-0571/#backwards-compatibility-with-manylinux1-wheels - arches.append('manylinux1' + arch_sep + arch_suffix) + arches.append("manylinux1" + arch_sep + arch_suffix) return arches def _get_custom_platforms(arch): # type: (str) -> List[str] - arch_prefix, _arch_sep, _arch_suffix = arch.partition('_') - if arch.startswith('macosx'): + arch_prefix, _arch_sep, _arch_suffix = arch.partition("_") + if arch.startswith("macosx"): arches = _mac_platforms(arch) - elif arch_prefix in ['manylinux2014', 'manylinux2010']: + elif arch_prefix in ["manylinux2014", "manylinux2010"]: arches = _custom_manylinux_platforms(arch) else: arches = [arch] @@ -139,7 +139,7 @@ def get_supported( version=None, # type: Optional[str] platforms=None, # type: Optional[List[str]] impl=None, # type: Optional[str] - abis=None # type: Optional[List[str]] + abis=None, # type: Optional[List[str]] ): # type: (...) -> List[Tag] """Return a list of supported tags for each version specified in diff --git a/etc/scripts/utils_pypi_supported_tags.py b/etc/scripts/utils_pypi_supported_tags.py index 8dcb70fb..de9f21b6 100644 --- a/etc/scripts/utils_pypi_supported_tags.py +++ b/etc/scripts/utils_pypi_supported_tags.py @@ -82,11 +82,7 @@ def is_supported_platform_tag(platform_tag): if platform_tag in _allowed_platforms: return True m = _macosx_platform_re.match(platform_tag) - if ( - m - and m.group("major") in _macosx_major_versions - and m.group("arch") in _macosx_arches - ): + if m and m.group("major") in _macosx_major_versions and m.group("arch") in _macosx_arches: return True m = _manylinux_platform_re.match(platform_tag) if m and m.group("arch") in _manylinux_arches: diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index ddbed612..9545db5e 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -16,7 +16,7 @@ """ -def load_requirements(requirements_file='requirements.txt', force_pinned=True): +def load_requirements(requirements_file="requirements.txt", force_pinned=True): """ Yield package (name, version) tuples for each requirement in a `requirement` file. Every requirement versions must be pinned if `force_pinned` is True. @@ -36,14 +36,14 @@ def get_required_name_versions(requirement_lines, force_pinned=True): """ for req_line in requirement_lines: req_line = req_line.strip() - if not req_line or req_line.startswith('#'): + if not req_line or req_line.startswith("#"): continue - if '==' not in req_line and force_pinned: - raise Exception(f'Requirement version is not pinned: {req_line}') + if "==" not in req_line and force_pinned: + raise Exception(f"Requirement version is not pinned: {req_line}") name = req_line version = None else: - name, _, version = req_line.partition('==') + name, _, version = req_line.partition("==") name = name.lower().strip() version = version.lower().strip() yield name, version @@ -58,22 +58,22 @@ def parse_requires(requires): if not requires: return [] - requires = [''.join(r.split()) for r in requires if r and r.strip()] + requires = ["".join(r.split()) for r in requires if r and r.strip()] return sorted(requires) -def lock_requirements(requirements_file='requirements.txt', site_packages_dir=None): +def lock_requirements(requirements_file="requirements.txt", site_packages_dir=None): """ Freeze and lock current installed requirements and save this to the `requirements_file` requirements file. """ - with open(requirements_file, 'w') as fo: + with open(requirements_file, "w") as fo: fo.write(get_installed_reqs(site_packages_dir=site_packages_dir)) def lock_dev_requirements( - dev_requirements_file='requirements-dev.txt', - main_requirements_file='requirements.txt', + dev_requirements_file="requirements-dev.txt", + main_requirements_file="requirements.txt", site_packages_dir=None, ): """ @@ -89,8 +89,8 @@ def lock_dev_requirements( all_req_nvs = get_required_name_versions(all_req_lines) dev_only_req_nvs = {n: v for n, v in all_req_nvs if n not in main_names} - new_reqs = '\n'.join(f'{n}=={v}' for n, v in sorted(dev_only_req_nvs.items())) - with open(dev_requirements_file, 'w') as fo: + new_reqs = "\n".join(f"{n}=={v}" for n, v in sorted(dev_only_req_nvs.items())) + with open(dev_requirements_file, "w") as fo: fo.write(new_reqs) @@ -99,5 +99,5 @@ def get_installed_reqs(site_packages_dir): Return the installed pip requirements as text found in `site_packages_dir` as a text. """ # Also include these packages in the output with --all: wheel, distribute, setuptools, pip - args = ['pip', 'freeze', '--exclude-editable', '--all', '--path', site_packages_dir] - return subprocess.check_output(args, encoding='utf-8') + args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] + return subprocess.check_output(args, encoding="utf-8") diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 444b20dd..e2778fe7 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -87,56 +87,73 @@ TRACE = False # Supported environments -PYTHON_VERSIONS = '36', '37', '38', '39', '310' +PYTHON_VERSIONS = "36", "37", "38", "39", "310" ABIS_BY_PYTHON_VERSION = { - '36':['cp36', 'cp36m'], - '37':['cp37', 'cp37m'], - '38':['cp38', 'cp38m'], - '39':['cp39', 'cp39m'], - '310':['cp310', 'cp310m'], + "36": ["cp36", "cp36m"], + "37": ["cp37", "cp37m"], + "38": ["cp38", "cp38m"], + "39": ["cp39", "cp39m"], + "310": ["cp310", "cp310m"], } PLATFORMS_BY_OS = { - 'linux': [ - 'linux_x86_64', - 'manylinux1_x86_64', - 'manylinux2014_x86_64', - 'manylinux2010_x86_64', - 'manylinux_2_12_x86_64', + "linux": [ + "linux_x86_64", + "manylinux1_x86_64", + "manylinux2014_x86_64", + "manylinux2010_x86_64", + "manylinux_2_12_x86_64", ], - 'macos': [ - 'macosx_10_6_intel', 'macosx_10_6_x86_64', - 'macosx_10_9_intel', 'macosx_10_9_x86_64', - 'macosx_10_10_intel', 'macosx_10_10_x86_64', - 'macosx_10_11_intel', 'macosx_10_11_x86_64', - 'macosx_10_12_intel', 'macosx_10_12_x86_64', - 'macosx_10_13_intel', 'macosx_10_13_x86_64', - 'macosx_10_14_intel', 'macosx_10_14_x86_64', - 'macosx_10_15_intel', 'macosx_10_15_x86_64', - 'macosx_10_15_x86_64', - 'macosx_11_0_x86_64', + "macos": [ + "macosx_10_6_intel", + "macosx_10_6_x86_64", + "macosx_10_9_intel", + "macosx_10_9_x86_64", + "macosx_10_10_intel", + "macosx_10_10_x86_64", + "macosx_10_11_intel", + "macosx_10_11_x86_64", + "macosx_10_12_intel", + "macosx_10_12_x86_64", + "macosx_10_13_intel", + "macosx_10_13_x86_64", + "macosx_10_14_intel", + "macosx_10_14_x86_64", + "macosx_10_15_intel", + "macosx_10_15_x86_64", + "macosx_10_15_x86_64", + "macosx_11_0_x86_64", # 'macosx_11_0_arm64', ], - 'windows': [ - 'win_amd64', + "windows": [ + "win_amd64", ], } -THIRDPARTY_DIR = 'thirdparty' -CACHE_THIRDPARTY_DIR = '.cache/thirdparty' - -REMOTE_LINKS_URL = 'https://thirdparty.aboutcode.org/pypi' - -EXTENSIONS_APP = '.pyz', -EXTENSIONS_SDIST = '.tar.gz', '.tar.bz2', '.zip', '.tar.xz', -EXTENSIONS_INSTALLABLE = EXTENSIONS_SDIST + ('.whl',) -EXTENSIONS_ABOUT = '.ABOUT', '.LICENSE', '.NOTICE', +THIRDPARTY_DIR = "thirdparty" +CACHE_THIRDPARTY_DIR = ".cache/thirdparty" + +REMOTE_LINKS_URL = "https://thirdparty.aboutcode.org/pypi" + +EXTENSIONS_APP = (".pyz",) +EXTENSIONS_SDIST = ( + ".tar.gz", + ".tar.bz2", + ".zip", + ".tar.xz", +) +EXTENSIONS_INSTALLABLE = EXTENSIONS_SDIST + (".whl",) +EXTENSIONS_ABOUT = ( + ".ABOUT", + ".LICENSE", + ".NOTICE", +) EXTENSIONS = EXTENSIONS_INSTALLABLE + EXTENSIONS_ABOUT + EXTENSIONS_APP -PYPI_SIMPLE_URL = 'https://pypi.org/simple' +PYPI_SIMPLE_URL = "https://pypi.org/simple" -LICENSEDB_API_URL = 'https://scancode-licensedb.aboutcode.org' +LICENSEDB_API_URL = "https://scancode-licensedb.aboutcode.org" LICENSING = license_expression.Licensing() @@ -149,7 +166,7 @@ def fetch_wheels( environment=None, - requirements_file='requirements.txt', + requirements_file="requirements.txt", allow_unpinned=False, dest_dir=THIRDPARTY_DIR, remote_links_url=REMOTE_LINKS_URL, @@ -179,11 +196,13 @@ def fetch_wheels( force_pinned = False try: - rrp = list(get_required_remote_packages( - requirements_file=requirements_file, - force_pinned=force_pinned, - remote_links_url=remote_links_url, - )) + rrp = list( + get_required_remote_packages( + requirements_file=requirements_file, + force_pinned=force_pinned, + remote_links_url=remote_links_url, + ) + ) except Exception as e: raise Exception( dict( @@ -196,9 +215,14 @@ def fetch_wheels( fetched_filenames = set() for name, version, package in rrp: if not package: - missed.append((name, version,)) - nv = f'{name}=={version}' if version else name - yield None, f'fetch_wheels: Missing package in remote repo: {nv}' + missed.append( + ( + name, + version, + ) + ) + nv = f"{name}=={version}" if version else name + yield None, f"fetch_wheels: Missing package in remote repo: {nv}" else: fetched_filename = package.fetch_wheel( @@ -214,23 +238,23 @@ def fetch_wheels( if fetched_filename in fetched_filenames: error = None else: - error = f'Failed to fetch' + error = f"Failed to fetch" yield package, error if missed: rr = get_remote_repo() print() - print(f'===> fetch_wheels: Missed some packages') + print(f"===> fetch_wheels: Missed some packages") for n, v in missed: - nv = f'{n}=={v}' if v else n - print(f'Missed package {nv} in remote repo, has only:') + nv = f"{n}=={v}" if v else n + print(f"Missed package {nv} in remote repo, has only:") for pv in rr.get_versions(n): - print(' ', pv) - raise Exception('Missed some packages in remote repo') + print(" ", pv) + raise Exception("Missed some packages in remote repo") def fetch_sources( - requirements_file='requirements.txt', + requirements_file="requirements.txt", allow_unpinned=False, dest_dir=THIRDPARTY_DIR, remote_links_url=REMOTE_LINKS_URL, @@ -258,27 +282,35 @@ def fetch_sources( else: force_pinned = False - rrp = list(get_required_remote_packages( - requirements_file=requirements_file, - force_pinned=force_pinned, - remote_links_url=remote_links_url, - )) + rrp = list( + get_required_remote_packages( + requirements_file=requirements_file, + force_pinned=force_pinned, + remote_links_url=remote_links_url, + ) + ) for name, version, package in rrp: if not package: - missed.append((name, name,)) - nv = f'{name}=={version}' if version else name - yield None, f'fetch_sources: Missing package in remote repo: {nv}' + missed.append( + ( + name, + name, + ) + ) + nv = f"{name}=={version}" if version else name + yield None, f"fetch_sources: Missing package in remote repo: {nv}" elif not package.sdist: - yield package, f'Missing sdist in links' + yield package, f"Missing sdist in links" else: fetched = package.fetch_sdist(dest_dir=dest_dir) - error = f'Failed to fetch' if not fetched else None + error = f"Failed to fetch" if not fetched else None yield package, error if missed: - raise Exception(f'Missing source packages in {remote_links_url}', missed) + raise Exception(f"Missing source packages in {remote_links_url}", missed) + ################################################################################ # @@ -291,12 +323,12 @@ def fetch_sources( class NameVer: name = attr.ib( type=str, - metadata=dict(help='Python package name, lowercase and normalized.'), + metadata=dict(help="Python package name, lowercase and normalized."), ) version = attr.ib( type=str, - metadata=dict(help='Python package version string.'), + metadata=dict(help="Python package version string."), ) @property @@ -320,7 +352,7 @@ def standardize_name(name): @property def name_ver(self): - return f'{self.name}-{self.version}' + return f"{self.name}-{self.version}" def sortable_name_version(self): """ @@ -339,146 +371,146 @@ class Distribution(NameVer): # field names that can be updated from another dist of mapping updatable_fields = [ - 'license_expression', - 'copyright', - 'description', - 'homepage_url', - 'primary_language', - 'notice_text', - 'extra_data', + "license_expression", + "copyright", + "description", + "homepage_url", + "primary_language", + "notice_text", + "extra_data", ] filename = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='File name.'), + default="", + metadata=dict(help="File name."), ) path_or_url = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='Path or download URL.'), + default="", + metadata=dict(help="Path or download URL."), ) sha256 = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='SHA256 checksum.'), + default="", + metadata=dict(help="SHA256 checksum."), ) sha1 = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='SHA1 checksum.'), + default="", + metadata=dict(help="SHA1 checksum."), ) md5 = attr.ib( repr=False, type=int, default=0, - metadata=dict(help='MD5 checksum.'), + metadata=dict(help="MD5 checksum."), ) type = attr.ib( repr=False, type=str, - default='pypi', - metadata=dict(help='Package type'), + default="pypi", + metadata=dict(help="Package type"), ) namespace = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='Package URL namespace'), + default="", + metadata=dict(help="Package URL namespace"), ) qualifiers = attr.ib( repr=False, type=dict, default=attr.Factory(dict), - metadata=dict(help='Package URL qualifiers'), + metadata=dict(help="Package URL qualifiers"), ) subpath = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='Package URL subpath'), + default="", + metadata=dict(help="Package URL subpath"), ) size = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='Size in bytes.'), + default="", + metadata=dict(help="Size in bytes."), ) primary_language = attr.ib( repr=False, type=str, - default='Python', - metadata=dict(help='Primary Programming language.'), + default="Python", + metadata=dict(help="Primary Programming language."), ) description = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='Description.'), + default="", + metadata=dict(help="Description."), ) homepage_url = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='Homepage URL'), + default="", + metadata=dict(help="Homepage URL"), ) notes = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='Notes.'), + default="", + metadata=dict(help="Notes."), ) copyright = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='Copyright.'), + default="", + metadata=dict(help="Copyright."), ) license_expression = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='License expression'), + default="", + metadata=dict(help="License expression"), ) licenses = attr.ib( repr=False, type=list, default=attr.Factory(list), - metadata=dict(help='List of license mappings.'), + metadata=dict(help="List of license mappings."), ) notice_text = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='Notice text'), + default="", + metadata=dict(help="Notice text"), ) extra_data = attr.ib( repr=False, type=dict, default=attr.Factory(dict), - metadata=dict(help='Extra data'), + metadata=dict(help="Extra data"), ) @property @@ -490,14 +522,14 @@ def package_url(self): @property def download_url(self): - if self.path_or_url and self.path_or_url.startswith('https://'): + if self.path_or_url and self.path_or_url.startswith("https://"): return self.path_or_url else: return self.get_best_download_url() @property def about_filename(self): - return f'{self.filename}.ABOUT' + return f"{self.filename}.ABOUT" def has_about_file(self, dest_dir=THIRDPARTY_DIR): return os.path.exists(os.path.join(dest_dir, self.about_filename)) @@ -508,7 +540,7 @@ def about_download_url(self): @property def notice_filename(self): - return f'{self.filename}.NOTICE' + return f"{self.filename}.NOTICE" @property def notice_download_url(self): @@ -521,16 +553,21 @@ def from_path_or_url(cls, path_or_url): `path_or_url` string. Raise an exception if this is not a valid filename. """ - filename = os.path.basename(path_or_url.strip('/')) + filename = os.path.basename(path_or_url.strip("/")) dist = cls.from_filename(filename) dist.path_or_url = path_or_url return dist @classmethod def get_dist_class(cls, filename): - if filename.endswith('.whl'): + if filename.endswith(".whl"): return Wheel - elif filename.endswith(('.zip', '.tar.gz',)): + elif filename.endswith( + ( + ".zip", + ".tar.gz", + ) + ): return Sdist raise InvalidDistributionFilename(filename) @@ -548,7 +585,7 @@ def from_data(cls, data, keep_extra=False): """ Return a distribution built from a `data` mapping. """ - filename = data['filename'] + filename = data["filename"] dist = cls.from_filename(filename) dist.update(data, keep_extra=keep_extra) return dist @@ -560,16 +597,20 @@ def from_dist(cls, data, dist): from another dist Distribution. Return None if it cannot be created """ # We can only create from a dist of the same package - has_same_key_fields = all(data.get(kf) == getattr(dist, kf, None) - for kf in ('type', 'namespace', 'name') + has_same_key_fields = all( + data.get(kf) == getattr(dist, kf, None) for kf in ("type", "namespace", "name") ) if not has_same_key_fields: - print(f'Missing key fields: Cannot derive a new dist from data: {data} and dist: {dist}') + print( + f"Missing key fields: Cannot derive a new dist from data: {data} and dist: {dist}" + ) return - has_key_field_values = all(data.get(kf) for kf in ('type', 'name', 'version')) + has_key_field_values = all(data.get(kf) for kf in ("type", "name", "version")) if not has_key_field_values: - print(f'Missing key field values: Cannot derive a new dist from data: {data} and dist: {dist}') + print( + f"Missing key field values: Cannot derive a new dist from data: {data} and dist: {dist}" + ) return data = dict(data) @@ -583,7 +624,7 @@ def build_remote_download_url(cls, filename, base_url=REMOTE_LINKS_URL): """ Return a direct download URL for a file in our remote repo """ - return f'{base_url}/{filename}' + return f"{base_url}/{filename}" def get_best_download_url(self): """ @@ -656,7 +697,7 @@ def has_key_metadata(self): """ Return True if this distribution has key metadata required for basic attribution. """ - if self.license_expression == 'public-domain': + if self.license_expression == "public-domain": # copyright not needed return True return self.license_expression and self.copyright and self.path_or_url @@ -677,7 +718,7 @@ def to_about(self): name=self.name, namespace=self.namespace, notes=self.notes, - notice_file=self.notice_filename if self.notice_text else '', + notice_file=self.notice_filename if self.notice_text else "", package_url=self.package_url, primary_language=self.primary_language, qualifiers=self.qualifiers, @@ -695,7 +736,7 @@ def to_dict(self): """ Return a mapping data from this distribution. """ - return {k: v for k, v in attr.asdict(self).items() if v} + return {k: v for k, v in attr.asdict(self).items() if v} def save_about_and_notice_files(self, dest_dir=THIRDPARTY_DIR): """ @@ -710,8 +751,9 @@ def save_if_modified(location, content): if existing_content == content: return False - if TRACE: print(f'Saving ABOUT (and NOTICE) files for: {self}') - with open(location, 'w') as fo: + if TRACE: + print(f"Saving ABOUT (and NOTICE) files for: {self}") + with open(location, "w") as fo: fo.write(content) return True @@ -750,26 +792,26 @@ def load_about_data(self, about_filename_or_data=None, dest_dir=THIRDPARTY_DIR): else: about_data = about_filename_or_data - md5 = about_data.pop('checksum_md5', None) + md5 = about_data.pop("checksum_md5", None) if md5: - about_data['md5'] = md5 - sha1 = about_data.pop('checksum_sha1', None) + about_data["md5"] = md5 + sha1 = about_data.pop("checksum_sha1", None) if sha1: - about_data['sha1'] = sha1 - sha256 = about_data.pop('checksum_sha256', None) + about_data["sha1"] = sha1 + sha256 = about_data.pop("checksum_sha256", None) if sha256: - about_data['sha256'] = sha256 + about_data["sha256"] = sha256 - about_data.pop('about_resource', None) - notice_text = about_data.pop('notice_text', None) - notice_file = about_data.pop('notice_file', None) + about_data.pop("about_resource", None) + notice_text = about_data.pop("notice_text", None) + notice_file = about_data.pop("notice_file", None) if notice_text: - about_data['notice_text'] = notice_text + about_data["notice_text"] = notice_text elif notice_file: notice_loc = os.path.join(dest_dir, notice_file) if os.path.exists(notice_loc): with open(notice_loc) as fi: - about_data['notice_text'] = fi.read() + about_data["notice_text"] = fi.read() return self.update(about_data, keep_extra=True) def load_remote_about_data(self): @@ -786,14 +828,14 @@ def load_remote_about_data(self): return False about_data = saneyaml.load(about_text) - notice_file = about_data.pop('notice_file', None) + notice_file = about_data.pop("notice_file", None) if notice_file: try: notice_text = fetch_content_from_path_or_url_through_cache(self.notice_download_url) if notice_text: - about_data['notice_text'] = notice_text + about_data["notice_text"] = notice_text except RemoteNotFetchedException: - print(f'Failed to fetch NOTICE file: {self.notice_download_url}') + print(f"Failed to fetch NOTICE file: {self.notice_download_url}") return self.load_about_data(about_data) def get_checksums(self, dest_dir=THIRDPARTY_DIR): @@ -803,7 +845,7 @@ def get_checksums(self, dest_dir=THIRDPARTY_DIR): """ dist_loc = os.path.join(dest_dir, self.filename) if os.path.exists(dist_loc): - return multi_checksums(dist_loc, checksum_names=('md5', 'sha1', 'sha256')) + return multi_checksums(dist_loc, checksum_names=("md5", "sha1", "sha256")) else: return {} @@ -819,7 +861,7 @@ def validate_checksums(self, dest_dir=THIRDPARTY_DIR): checksums computed for this dist filename is `dest_dir`. """ real_checksums = self.get_checksums(dest_dir) - for csk in ('md5', 'sha1', 'sha256'): + for csk in ("md5", "sha1", "sha256"): csv = getattr(self, csk) rcv = real_checksums.get(csk) if csv and rcv and csv != rcv: @@ -830,14 +872,14 @@ def get_pip_hash(self): """ Return a pip hash option string as used in requirements for this dist. """ - assert self.sha256, f'Missinh SHA256 for dist {self}' - return f'--hash=sha256:{self.sha256}' + assert self.sha256, f"Missinh SHA256 for dist {self}" + return f"--hash=sha256:{self.sha256}" def get_license_keys(self): try: keys = LICENSING.license_keys(self.license_expression, unique=True, simple=True) except license_expression.ExpressionParseError: - return ['unknown'] + return ["unknown"] return keys def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): @@ -847,19 +889,18 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): """ paths_or_urls = get_remote_repo().links errors = [] - extra_lic_names = [l.get('file') for l in self.extra_data.get('licenses', {})] - extra_lic_names += [self.extra_data.get('license_file')] - extra_lic_names = [ln for ln in extra_lic_names if ln] - lic_names = [ f'{key}.LICENSE' for key in self.get_license_keys()] - for filename in lic_names + extra_lic_names: + extra_lic_names = [l.get("file") for l in self.extra_data.get("licenses", {})] + extra_lic_names += [self.extra_data.get("license_file")] + extra_lic_names = [ln for ln in extra_lic_names if ln] + lic_names = [f"{key}.LICENSE" for key in self.get_license_keys()] + for filename in lic_names + extra_lic_names: floc = os.path.join(dest_dir, filename) if os.path.exists(floc): continue try: # try remotely first - lic_url = get_link_for_filename( - filename=filename, paths_or_urls=paths_or_urls) + lic_url = get_link_for_filename(filename=filename, paths_or_urls=paths_or_urls) fetch_and_save_path_or_url( filename=filename, @@ -867,19 +908,21 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): path_or_url=lic_url, as_text=True, ) - if TRACE: print(f'Fetched license from remote: {lic_url}') + if TRACE: + print(f"Fetched license from remote: {lic_url}") except: try: # try licensedb second - lic_url = f'{LICENSEDB_API_URL}/{filename}' + lic_url = f"{LICENSEDB_API_URL}/{filename}" fetch_and_save_path_or_url( filename=filename, dest_dir=dest_dir, path_or_url=lic_url, as_text=True, ) - if TRACE: print(f'Fetched license from licensedb: {lic_url}') + if TRACE: + print(f"Fetched license from licensedb: {lic_url}") except: msg = f'No text for license {filename} in expression "{self.license_expression}" from {self}' @@ -893,14 +936,19 @@ def extract_pkginfo(self, dest_dir=THIRDPARTY_DIR): Return the text of the first PKG-INFO or METADATA file found in the archive of this Distribution in `dest_dir`. Return None if not found. """ - fmt = 'zip' if self.filename.endswith('.whl') else None + fmt = "zip" if self.filename.endswith(".whl") else None dist = os.path.join(dest_dir, self.filename) - with tempfile.TemporaryDirectory(prefix='pypi-tmp-extract') as td: + with tempfile.TemporaryDirectory(prefix="pypi-tmp-extract") as td: shutil.unpack_archive(filename=dist, extract_dir=td, format=fmt) # NOTE: we only care about the first one found in the dist # which may not be 100% right for pi in fileutils.resource_iter(location=td, with_dirs=False): - if pi.endswith(('PKG-INFO', 'METADATA',)): + if pi.endswith( + ( + "PKG-INFO", + "METADATA", + ) + ): with open(pi) as fi: return fi.read() @@ -911,31 +959,33 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): """ pkginfo_text = self.extract_pkginfo(dest_dir=dest_dir) if not pkginfo_text: - print(f'!!!!PKG-INFO not found in {self.filename}') + print(f"!!!!PKG-INFO not found in {self.filename}") return raw_data = email.message_from_string(pkginfo_text) - classifiers = raw_data.get_all('Classifier') or [] + classifiers = raw_data.get_all("Classifier") or [] - declared_license = [raw_data['License']] + [c for c in classifiers if c.startswith('License')] + declared_license = [raw_data["License"]] + [ + c for c in classifiers if c.startswith("License") + ] license_expression = compute_normalized_license_expression(declared_license) - other_classifiers = [c for c in classifiers if not c.startswith('License')] + other_classifiers = [c for c in classifiers if not c.startswith("License")] - holder = raw_data['Author'] - holder_contact = raw_data['Author-email'] - copyright_statement = f'Copyright (c) {holder} <{holder_contact}>' + holder = raw_data["Author"] + holder_contact = raw_data["Author-email"] + copyright_statement = f"Copyright (c) {holder} <{holder_contact}>" pkginfo_data = dict( - name=raw_data['Name'], + name=raw_data["Name"], declared_license=declared_license, - version=raw_data['Version'], - description=raw_data['Summary'], - homepage_url=raw_data['Home-page'], + version=raw_data["Version"], + description=raw_data["Summary"], + homepage_url=raw_data["Home-page"], copyright=copyright_statement, license_expression=license_expression, holder=holder, holder_contact=holder_contact, - keywords=raw_data['Keywords'], + keywords=raw_data["Keywords"], classifiers=other_classifiers, ) @@ -949,10 +999,7 @@ def update_from_other_dist(self, dist): def get_updatable_data(self, data=None): data = data or self.to_dict() - return { - k: v for k, v in data.items() - if v and k in self.updatable_fields - } + return {k: v for k, v in data.items() if v and k in self.updatable_fields} def update(self, data, overwrite=False, keep_extra=True): """ @@ -961,20 +1008,21 @@ def update(self, data, overwrite=False, keep_extra=True): Return True if any data was updated, False otherwise. Raise an exception if there are key data conflicts. """ - package_url = data.get('package_url') + package_url = data.get("package_url") if package_url: purl_from_data = packageurl.PackageURL.from_string(package_url) purl_from_self = packageurl.PackageURL.from_string(self.package_url) if purl_from_data != purl_from_self: print( - f'Invalid dist update attempt, no same same purl with dist: ' - f'{self} using data {data}.') + f"Invalid dist update attempt, no same same purl with dist: " + f"{self} using data {data}." + ) return - data.pop('about_resource', None) - dl = data.pop('download_url', None) + data.pop("about_resource", None) + dl = data.pop("download_url", None) if dl: - data['path_or_url'] = dl + data["path_or_url"] = dl updated = False extra = {} @@ -990,7 +1038,7 @@ def update(self, data, overwrite=False, keep_extra=True): try: setattr(self, k, v) except Exception as e: - raise Exception(f'{self}, {k}, {v}') from e + raise Exception(f"{self}, {k}, {v}") from e updated = True elif keep_extra: @@ -1013,8 +1061,8 @@ class Sdist(Distribution): extension = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='File extension, including leading dot.'), + default="", + metadata=dict(help="File extension, including leading dot."), ) @classmethod @@ -1034,13 +1082,13 @@ def from_filename(cls, filename): if not extension or not name_ver: raise InvalidDistributionFilename(filename) - name, _, version = name_ver.rpartition('-') + name, _, version = name_ver.rpartition("-") if not name or not version: raise InvalidDistributionFilename(filename) return cls( - type='pypi', + type="pypi", name=name, version=version, extension=extension, @@ -1052,7 +1100,7 @@ def to_filename(self): Return an sdist filename reconstructed from its fields (that may not be the same as the original filename.) """ - return f'{self.name}-{self.version}.{self.extension}' + return f"{self.name}-{self.version}.{self.extension}" @attr.attributes @@ -1097,38 +1145,38 @@ class Wheel(Distribution): r"""^(?P(?P.+?)-(?P.*?)) ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) \.whl)$""", - re.VERBOSE + re.VERBOSE, ).match build = attr.ib( type=str, - default='', - metadata=dict(help='Python wheel build.'), + default="", + metadata=dict(help="Python wheel build."), ) python_versions = attr.ib( type=list, default=attr.Factory(list), - metadata=dict(help='List of wheel Python version tags.'), + metadata=dict(help="List of wheel Python version tags."), ) abis = attr.ib( type=list, default=attr.Factory(list), - metadata=dict(help='List of wheel ABI tags.'), + metadata=dict(help="List of wheel ABI tags."), ) platforms = attr.ib( type=list, default=attr.Factory(list), - metadata=dict(help='List of wheel platform tags.'), + metadata=dict(help="List of wheel platform tags."), ) tags = attr.ib( repr=False, type=set, default=attr.Factory(set), - metadata=dict(help='Set of all tags for this wheel.'), + metadata=dict(help="Set of all tags for this wheel."), ) @classmethod @@ -1141,24 +1189,23 @@ def from_filename(cls, filename): if not wheel_info: raise InvalidDistributionFilename(filename) - name = wheel_info.group('name').replace('_', '-') + name = wheel_info.group("name").replace("_", "-") # we'll assume "_" means "-" due to wheel naming scheme # (https://github.com/pypa/pip/issues/1150) - version = wheel_info.group('ver').replace('_', '-') - build = wheel_info.group('build') - python_versions = wheel_info.group('pyvers').split('.') - abis = wheel_info.group('abis').split('.') - platforms = wheel_info.group('plats').split('.') + version = wheel_info.group("ver").replace("_", "-") + build = wheel_info.group("build") + python_versions = wheel_info.group("pyvers").split(".") + abis = wheel_info.group("abis").split(".") + platforms = wheel_info.group("plats").split(".") # All the tag combinations from this file tags = { - packaging_tags.Tag(x, y, z) for x in python_versions - for y in abis for z in platforms + packaging_tags.Tag(x, y, z) for x in python_versions for y in abis for z in platforms } return cls( filename=filename, - type='pypi', + type="pypi", name=name, version=version, build=build, @@ -1179,18 +1226,18 @@ def is_supported_by_environment(self, environment): Return True if this wheel is compatible with the Environment `environment`. """ - return not self.is_supported_by_tags(environment.tags) + return not self.is_supported_by_tags(environment.tags) def to_filename(self): """ Return a wheel filename reconstructed from its fields (that may not be the same as the original filename.) """ - build = f'-{self.build}' if self.build else '' - pyvers = '.'.join(self.python_versions) - abis = '.'.join(self.abis) - plats = '.'.join(self.platforms) - return f'{self.name}-{self.version}{build}-{pyvers}-{abis}-{plats}.whl' + build = f"-{self.build}" if self.build else "" + pyvers = ".".join(self.python_versions) + abis = ".".join(self.abis) + plats = ".".join(self.platforms) + return f"{self.name}-{self.version}{build}-{pyvers}-{abis}-{plats}.whl" def is_pure(self): """ @@ -1216,11 +1263,7 @@ def is_pure(self): >>> Wheel.from_filename('future-0.16.0-py3-cp36m-any.whl').is_pure() False """ - return ( - 'py3' in self.python_versions - and 'none' in self.abis - and 'any' in self.platforms - ) + return "py3" in self.python_versions and "none" in self.abis and "any" in self.platforms def is_pure_wheel(filename): @@ -1236,18 +1279,19 @@ class PypiPackage(NameVer): A Python package with its "distributions", e.g. wheels and source distribution , ABOUT files and licenses or notices. """ + sdist = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='Sdist source distribution for this package.'), + default="", + metadata=dict(help="Sdist source distribution for this package."), ) wheels = attr.ib( repr=False, type=list, default=attr.Factory(list), - metadata=dict(help='List of Wheel for this package'), + metadata=dict(help="List of Wheel for this package"), ) @property @@ -1256,7 +1300,7 @@ def specifier(self): A requirement specifier for this package """ if self.version: - return f'{self.name}=={self.version}' + return f"{self.name}=={self.version}" else: return self.name @@ -1268,7 +1312,7 @@ def specifier_with_hashes(self): """ items = [self.specifier] items += [d.get_pip_hashes() for d in self.get_distributions()] - return ' \\\n '.join(items) + return " \\\n ".join(items) def get_supported_wheels(self, environment): """ @@ -1314,7 +1358,7 @@ def package_from_dists(cls, dists): if dist.normalized_name != normalized_name or dist.version != version: if TRACE: print( - f' Skipping inconsistent dist name and version: {dist} ' + f" Skipping inconsistent dist name and version: {dist} " f'Expected instead package name: {normalized_name} and version: "{version}"' ) continue @@ -1326,7 +1370,7 @@ def package_from_dists(cls, dists): package.wheels.append(dist) else: - raise Exception(f'Unknown distribution type: {dist}') + raise Exception(f"Unknown distribution type: {dist}") return package @@ -1348,7 +1392,8 @@ def packages_from_many_paths_or_urls(cls, paths_or_urls): dists = NameVer.sorted(dists) for _projver, dists_of_package in itertools.groupby( - dists, key=NameVer.sortable_name_version, + dists, + key=NameVer.sortable_name_version, ): yield PypiPackage.package_from_dists(dists_of_package) @@ -1409,7 +1454,7 @@ def get_name_version(cls, name, version, packages): if len(nvs) == 1: return nvs[0] - raise Exception(f'More than one PypiPackage with {name}=={version}') + raise Exception(f"More than one PypiPackage with {name}=={version}") def fetch_wheel( self, @@ -1457,17 +1502,19 @@ def fetch_sdist(self, dest_dir=THIRDPARTY_DIR): """ if self.sdist: assert self.sdist.filename - if TRACE: print('Fetching source for package:', self.name, self.version) + if TRACE: + print("Fetching source for package:", self.name, self.version) fetch_and_save_path_or_url( filename=self.sdist.filename, dest_dir=dest_dir, path_or_url=self.sdist.path_or_url, as_text=False, ) - if TRACE: print(' --> file:', self.sdist.filename) + if TRACE: + print(" --> file:", self.sdist.filename) return self.sdist.filename else: - print(f'Missing sdist for: {self.name}=={self.version}') + print(f"Missing sdist for: {self.name}=={self.version}") return False def delete_files(self, dest_dir=THIRDPARTY_DIR): @@ -1481,10 +1528,10 @@ def delete_files(self, dest_dir=THIRDPARTY_DIR): if not to_delete: continue tdfn = to_delete.filename - for deletable in [tdfn, f'{tdfn}.ABOUT', f'{tdfn}.NOTICE']: + for deletable in [tdfn, f"{tdfn}.ABOUT", f"{tdfn}.NOTICE"]: target = os.path.join(dest_dir, deletable) if os.path.exists(target): - print(f'Deleting outdated {target}') + print(f"Deleting outdated {target}") fileutils.delete(target) @classmethod @@ -1528,7 +1575,7 @@ def get_dists(cls, paths_or_urls): yield Distribution.from_path_or_url(path_or_url) except InvalidDistributionFilename: if TRACE: - print(f'Skipping invalid distribution from: {path_or_url}') + print(f"Skipping invalid distribution from: {path_or_url}") continue def get_distributions(self): @@ -1562,42 +1609,42 @@ class Environment: python_version = attr.ib( type=str, - default='', - metadata=dict(help='Python version supported by this environment.'), + default="", + metadata=dict(help="Python version supported by this environment."), ) operating_system = attr.ib( type=str, - default='', - metadata=dict(help='operating system supported by this environment.'), + default="", + metadata=dict(help="operating system supported by this environment."), ) implementation = attr.ib( type=str, - default='cp', - metadata=dict(help='Python implementation supported by this environment.'), + default="cp", + metadata=dict(help="Python implementation supported by this environment."), ) abis = attr.ib( type=list, default=attr.Factory(list), - metadata=dict(help='List of ABI tags supported by this environment.'), + metadata=dict(help="List of ABI tags supported by this environment."), ) platforms = attr.ib( type=list, default=attr.Factory(list), - metadata=dict(help='List of platform tags supported by this environment.'), + metadata=dict(help="List of platform tags supported by this environment."), ) @classmethod def from_pyver_and_os(cls, python_version, operating_system): - if '.' in python_version: - python_version = ''.join(python_version.split('.')) + if "." in python_version: + python_version = "".join(python_version.split(".")) return cls( python_version=python_version, - implementation='cp', + implementation="cp", abis=ABIS_BY_PYTHON_VERSION[python_version], platforms=PLATFORMS_BY_OS[operating_system], operating_system=operating_system, @@ -1608,24 +1655,30 @@ def get_pip_cli_options(self): Return a list of pip command line options for this environment. """ options = [ - '--python-version', self.python_version, - '--implementation', self.implementation, - '--abi', self.abi, + "--python-version", + self.python_version, + "--implementation", + self.implementation, + "--abi", + self.abi, ] for platform in self.platforms: - options.extend(['--platform', platform]) + options.extend(["--platform", platform]) return options def tags(self): """ Return a set of all the PEP425 tags supported by this environment. """ - return set(utils_pip_compatibility_tags.get_supported( - version=self.python_version or None, - impl=self.implementation or None, - platforms=self.platforms or None, - abis=self.abis or None, - )) + return set( + utils_pip_compatibility_tags.get_supported( + version=self.python_version or None, + impl=self.implementation or None, + platforms=self.platforms or None, + abis=self.abis or None, + ) + ) + ################################################################################ # @@ -1643,15 +1696,13 @@ class Repository: packages_by_normalized_name = attr.ib( type=dict, default=attr.Factory(lambda: defaultdict(list)), - metadata=dict(help= - 'Mapping of {package name: [package objects]} available in this repo'), + metadata=dict(help="Mapping of {package name: [package objects]} available in this repo"), ) packages_by_normalized_name_version = attr.ib( type=dict, default=attr.Factory(dict), - metadata=dict(help= - 'Mapping of {(name, version): package object} available in this repo'), + metadata=dict(help="Mapping of {(name, version): package object} available in this repo"), ) def get_links(self, *args, **kwargs): @@ -1684,16 +1735,17 @@ class LinksRepository(Repository): Python wheels and sdist or a remote URL to an HTML with links to these. (e.g. suitable for use with pip --find-links). """ + path_or_url = attr.ib( type=str, - default='', - metadata=dict(help='Package directory path or URL'), + default="", + metadata=dict(help="Package directory path or URL"), ) links = attr.ib( type=list, default=attr.Factory(list), - metadata=dict(help='List of links available in this repo'), + metadata=dict(help="List of links available in this repo"), ) def __attrs_post_init__(self): @@ -1725,16 +1777,17 @@ class PypiRepository(Repository): Represents the public PyPI simple index. It is populated lazily based on requested packages names """ + simple_url = attr.ib( type=str, default=PYPI_SIMPLE_URL, - metadata=dict(help='Base PyPI simple URL for this index.'), + metadata=dict(help="Base PyPI simple URL for this index."), ) links_by_normalized_name = attr.ib( type=dict, default=attr.Factory(lambda: defaultdict(list)), - metadata=dict(help='Mapping of {package name: [links]} available in this repo'), + metadata=dict(help="Mapping of {package name: [links]} available in this repo"), ) def _fetch_links(self, name): @@ -1759,7 +1812,7 @@ def _populate_links_and_packages(self, name): def get_links(self, name, *args, **kwargs): name = name and NameVer.normalize_name(name) self._populate_links_and_packages(name) - return self.links_by_normalized_name.get(name, []) + return self.links_by_normalized_name.get(name, []) def get_versions(self, name): name = name and NameVer.normalize_name(name) @@ -1772,6 +1825,7 @@ def get_latest_version(self, name): def get_package(self, name, version): return PypiPackage.get_name_version(name, version, self.get_versions(name)) + ################################################################################ # Globals for remote repos to be lazily created and cached on first use for the # life of the session together with some convenience functions. @@ -1807,7 +1861,7 @@ def get_remote_package(name, version, remote_links_url=REMOTE_LINKS_URL): try: return get_remote_repo(remote_links_url).get_package(name, version) except RemoteNotFetchedException as e: - print(f'Failed to fetch remote package info: {e}') + print(f"Failed to fetch remote package info: {e}") _PYPI_REPO = None @@ -1827,7 +1881,8 @@ def get_pypi_package(name, version, pypi_simple_url=PYPI_SIMPLE_URL): try: return get_pypi_repo(pypi_simple_url).get_package(name, version) except RemoteNotFetchedException as e: - print(f'Failed to fetch remote package info: {e}') + print(f"Failed to fetch remote package info: {e}") + ################################################################################ # @@ -1856,12 +1911,12 @@ def get(self, path_or_url, as_text=True): Get a file from a `path_or_url` through the cache. `path_or_url` can be a path or a URL to a file. """ - filename = os.path.basename(path_or_url.strip('/')) + filename = os.path.basename(path_or_url.strip("/")) cached = os.path.join(self.directory, filename) if not os.path.exists(cached): content = get_file_content(path_or_url=path_or_url, as_text=as_text) - wmode = 'w' if as_text else 'wb' + wmode = "w" if as_text else "wb" with open(cached, wmode) as fo: fo.write(content) return content @@ -1873,7 +1928,7 @@ def put(self, filename, content): Put in the cache the `content` of `filename`. """ cached = os.path.join(self.directory, filename) - wmode = 'wb' if isinstance(content, bytes) else 'w' + wmode = "wb" if isinstance(content, bytes) else "w" with open(cached, wmode) as fo: fo.write(content) @@ -1883,18 +1938,19 @@ def get_file_content(path_or_url, as_text=True): Fetch and return the content at `path_or_url` from either a local path or a remote URL. Return the content as bytes is `as_text` is False. """ - if (path_or_url.startswith('file://') - or (path_or_url.startswith('/') and os.path.exists(path_or_url)) + if path_or_url.startswith("file://") or ( + path_or_url.startswith("/") and os.path.exists(path_or_url) ): return get_local_file_content(path=path_or_url, as_text=as_text) - elif path_or_url.startswith('https://'): - if TRACE: print(f'Fetching: {path_or_url}') + elif path_or_url.startswith("https://"): + if TRACE: + print(f"Fetching: {path_or_url}") _headers, content = get_remote_file_content(url=path_or_url, as_text=as_text) return content else: - raise Exception(f'Unsupported URL scheme: {path_or_url}') + raise Exception(f"Unsupported URL scheme: {path_or_url}") def get_local_file_content(path, as_text=True): @@ -1902,10 +1958,10 @@ def get_local_file_content(path, as_text=True): Return the content at `url` as text. Return the content as bytes is `as_text` is False. """ - if path.startswith('file://'): + if path.startswith("file://"): path = path[7:] - mode = 'r' if as_text else 'rb' + mode = "r" if as_text else "rb" with open(path, mode) as fo: return fo.read() @@ -1914,7 +1970,13 @@ class RemoteNotFetchedException(Exception): pass -def get_remote_file_content(url, as_text=True, headers_only=False, headers=None, _delay=0,): +def get_remote_file_content( + url, + as_text=True, + headers_only=False, + headers=None, + _delay=0, +): """ Fetch and return a tuple of (headers, content) at `url`. Return content as a text string if `as_text` is True. Otherwise return the content as bytes. @@ -1944,7 +2006,7 @@ def get_remote_file_content(url, as_text=True, headers_only=False, headers=None, ) else: - raise RemoteNotFetchedException(f'Failed HTTP request from {url} with {status}') + raise RemoteNotFetchedException(f"Failed HTTP request from {url} with {status}") if headers_only: return response.headers, None @@ -1952,7 +2014,11 @@ def get_remote_file_content(url, as_text=True, headers_only=False, headers=None, return response.headers, response.text if as_text else response.content -def get_url_content_if_modified(url, md5, _delay=0,): +def get_url_content_if_modified( + url, + md5, + _delay=0, +): """ Return fetched content bytes at `url` or None if the md5 has not changed. Retries multiple times to fetch if there is a HTTP 429 throttling response @@ -1962,7 +2028,7 @@ def get_url_content_if_modified(url, md5, _delay=0,): headers = None if md5: etag = f'"{md5}"' - headers = {'If-None-Match': f'{etag}'} + headers = {"If-None-Match": f"{etag}"} # using a GET with stream=True ensure we get the the final header from # several redirects and that we can ignore content there. A HEAD request may @@ -1979,7 +2045,7 @@ def get_url_content_if_modified(url, md5, _delay=0,): return None elif status != requests.codes.ok: # NOQA - raise RemoteNotFetchedException(f'Failed HTTP request from {url} with {status}') + raise RemoteNotFetchedException(f"Failed HTTP request from {url} with {status}") return response.content @@ -2045,11 +2111,12 @@ def fetch_and_save_path_or_url(filename, dest_dir, path_or_url, as_text=True, th content = fetch_content_from_path_or_url_through_cache(path_or_url, as_text, cache=None) output = os.path.join(dest_dir, filename) - wmode = 'w' if as_text else 'wb' + wmode = "w" if as_text else "wb" with open(output, wmode) as fo: fo.write(content) return content + ################################################################################ # # Sync and fix local thirdparty directory for various issues and gaps @@ -2070,29 +2137,34 @@ def fetch_missing_sources(dest_dir=THIRDPARTY_DIR): for package in local_packages: if not package.sdist: - print(f'Finding sources for: {package.name}=={package.version}: ', end='') + print(f"Finding sources for: {package.name}=={package.version}: ", end="") try: - pypi_package = pypi_repo.get_package( - name=package.name, version=package.version) + pypi_package = pypi_repo.get_package(name=package.name, version=package.version) if pypi_package and pypi_package.sdist: - print(f'Fetching sources from Pypi') + print(f"Fetching sources from Pypi") pypi_package.fetch_sdist(dest_dir=dest_dir) continue else: remote_package = remote_repo.get_package( - name=package.name, version=package.version) + name=package.name, version=package.version + ) if remote_package and remote_package.sdist: - print(f'Fetching sources from Remote') + print(f"Fetching sources from Remote") remote_package.fetch_sdist(dest_dir=dest_dir) continue except RemoteNotFetchedException as e: - print(f'Failed to fetch remote package info: {e}') + print(f"Failed to fetch remote package info: {e}") - print(f'No sources found') - not_found.append((package.name, package.version,)) + print(f"No sources found") + not_found.append( + ( + package.name, + package.version, + ) + ) return not_found @@ -2125,7 +2197,12 @@ def fetch_missing_wheels( if filename: fetched_filenames.add(filename) else: - not_fetched.append((package, envt,)) + not_fetched.append( + ( + package, + envt, + ) + ) return not_fetched @@ -2145,8 +2222,7 @@ def build_missing_wheels( not_built = [] built_filenames = [] - packages_and_envts = itertools.groupby( - sorted(packages_and_envts), key=operator.itemgetter(0)) + packages_and_envts = itertools.groupby(sorted(packages_and_envts), key=operator.itemgetter(0)) for package, pkg_envts in packages_and_envts: @@ -2164,25 +2240,27 @@ def build_missing_wheels( verbose=False, dest_dir=dest_dir, ) - print('.') + print(".") except Exception as e: import traceback - print('#############################################################') - print('############# WHEEL BUILD FAILED ######################') + + print("#############################################################") + print("############# WHEEL BUILD FAILED ######################") traceback.print_exc() print() - print('#############################################################') + print("#############################################################") if not built: for envt in pkg_envts: not_built.append((package, envt)) else: for bfn in built: - print(f' --> Built wheel: {bfn}') + print(f" --> Built wheel: {bfn}") built_filenames.append(bfn) return not_built, built_filenames + ################################################################################ # # Functions to handle remote or local repo used to "find-links" @@ -2191,7 +2269,7 @@ def build_missing_wheels( def get_paths_or_urls(links_url): - if links_url.startswith('https:'): + if links_url.startswith("https:"): paths_or_urls = find_links_from_release_url(links_url) else: paths_or_urls = find_links_from_dir(links_url) @@ -2217,14 +2295,15 @@ def find_links_from_release_url(links_url=REMOTE_LINKS_URL): URL that starts with the `prefix` string and ends with any of the extension in the list of `extensions` strings. Use the `base_url` to prefix the links. """ - if TRACE: print(f'Finding links for {links_url}') + if TRACE: + print(f"Finding links for {links_url}") plinks_url = urllib.parse.urlparse(links_url) - base_url = urllib.parse.SplitResult( - plinks_url.scheme, plinks_url.netloc, '', '', '').geturl() + base_url = urllib.parse.SplitResult(plinks_url.scheme, plinks_url.netloc, "", "", "").geturl() - if TRACE: print(f'Base URL {base_url}') + if TRACE: + print(f"Base URL {base_url}") _headers, text = get_remote_file_content(links_url) links = [] @@ -2238,19 +2317,21 @@ def find_links_from_release_url(links_url=REMOTE_LINKS_URL): # full URL kept as-is url = link - if plink.path.startswith('/'): + if plink.path.startswith("/"): # absolute link - url = f'{base_url}{link}' + url = f"{base_url}{link}" else: # relative link - url = f'{links_url}/{link}' + url = f"{links_url}/{link}" - if TRACE: print(f'Adding URL: {url}') + if TRACE: + print(f"Adding URL: {url}") links.append(url) - if TRACE: print(f'Found {len(links)} links at {links_url}') + if TRACE: + print(f"Found {len(links)} links at {links_url}") return links @@ -2259,19 +2340,20 @@ def find_pypi_links(name, simple_url=PYPI_SIMPLE_URL): Return a list of download link URLs found in a PyPI simple index for package name. with the list of `extensions` strings. Use the `simple_url` PyPI url. """ - if TRACE: print(f'Finding links for {simple_url}') + if TRACE: + print(f"Finding links for {simple_url}") name = name and NameVer.normalize_name(name) - simple_url = simple_url.strip('/') - simple_url = f'{simple_url}/{name}' + simple_url = simple_url.strip("/") + simple_url = f"{simple_url}/{name}" _headers, text = get_remote_file_content(simple_url) links = get_links(text) # TODO: keep sha256 - links = [l.partition('#sha256=') for l in links] + links = [l.partition("#sha256=") for l in links] links = [url for url, _, _sha256 in links] links = [l for l in links if l.endswith(EXTENSIONS)] - return links + return links def get_link_for_filename(filename, paths_or_urls): @@ -2280,13 +2362,14 @@ def get_link_for_filename(filename, paths_or_urls): exception if no link is found or if there are more than one link for that file name. """ - path_or_url = [l for l in paths_or_urls if l.endswith(f'/{filename}')] + path_or_url = [l for l in paths_or_urls if l.endswith(f"/{filename}")] if not path_or_url: - raise Exception(f'Missing link to file: {filename}') + raise Exception(f"Missing link to file: {filename}") if not len(path_or_url) == 1: - raise Exception(f'Multiple links to file: {filename}: \n' + '\n'.join(path_or_url)) + raise Exception(f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url)) return path_or_url[0] + ################################################################################ # # Requirements processing @@ -2308,12 +2391,16 @@ def get_required_packages(required_name_versions): """ remote_repo = get_remote_repo() - remote_packages = {(name, version): remote_repo.get_package(name, version) - for name, version in required_name_versions} + remote_packages = { + (name, version): remote_repo.get_package(name, version) + for name, version in required_name_versions + } pypi_repo = get_pypi_repo() - pypi_packages = {(name, version): pypi_repo.get_package(name, version) - for name, version in required_name_versions} + pypi_packages = { + (name, version): pypi_repo.get_package(name, version) + for name, version in required_name_versions + } # remove any empty package (e.g. that do not exist in some place) remote_packages = {nv: p for nv, p in remote_packages.items() if p} @@ -2329,7 +2416,7 @@ def get_required_packages(required_name_versions): def get_required_remote_packages( - requirements_file='requirements.txt', + requirements_file="requirements.txt", force_pinned=True, remote_links_url=REMOTE_LINKS_URL, ): @@ -2344,11 +2431,11 @@ def get_required_remote_packages( force_pinned=force_pinned, ) - if remote_links_url.startswith('https://'): + if remote_links_url.startswith("https://"): repo = get_remote_repo(remote_links_url=remote_links_url) else: # a local path - assert os.path.exists(remote_links_url), f'Path does not exist: {remote_links_url}' + assert os.path.exists(remote_links_url), f"Path does not exist: {remote_links_url}" repo = get_local_repo(directory=remote_links_url) for name, version in required_name_versions: @@ -2358,7 +2445,7 @@ def get_required_remote_packages( yield name, version, repo.get_latest_version(name) -def update_requirements(name, version=None, requirements_file='requirements.txt'): +def update_requirements(name, version=None, requirements_file="requirements.txt"): """ Upgrade or add `package_name` with `new_version` to the `requirements_file` requirements file. Write back requirements sorted with name and version @@ -2376,17 +2463,22 @@ def update_requirements(name, version=None, requirements_file='requirements.txt' if normalized_name == existing_normalized_name: if version != existing_version: is_updated = True - updated_name_versions.append((existing_normalized_name, existing_version,)) + updated_name_versions.append( + ( + existing_normalized_name, + existing_version, + ) + ) if is_updated: updated_name_versions = sorted(updated_name_versions) - nvs = '\n'.join(f'{name}=={version}' for name, version in updated_name_versions) + nvs = "\n".join(f"{name}=={version}" for name, version in updated_name_versions) - with open(requirements_file, 'w') as fo: + with open(requirements_file, "w") as fo: fo.write(nvs) -def hash_requirements(dest_dir=THIRDPARTY_DIR, requirements_file='requirements.txt'): +def hash_requirements(dest_dir=THIRDPARTY_DIR, requirements_file="requirements.txt"): """ Hash all the requirements found in the `requirements_file` requirements file based on distributions available in `dest_dir` @@ -2397,11 +2489,12 @@ def hash_requirements(dest_dir=THIRDPARTY_DIR, requirements_file='requirements.t for name, version in load_requirements(requirements_file, force_pinned=True): package = packages_by_normalized_name_version.get((name, version)) if not package: - raise Exception(f'Missing required package {name}=={version}') + raise Exception(f"Missing required package {name}=={version}") hashed.append(package.specifier_with_hashes) - with open(requirements_file, 'w') as fo: - fo.write('\n'.join(hashed)) + with open(requirements_file, "w") as fo: + fo.write("\n".join(hashed)) + ################################################################################ # @@ -2462,7 +2555,8 @@ def get_other_dists(_package, _dist): # try to get a latest version of the same package that is not our version other_local_packages = [ - p for p in local_repo.get_versions(local_package.name) + p + for p in local_repo.get_versions(local_package.name) if p.version != local_package.version ] @@ -2498,7 +2592,8 @@ def get_other_dists(_package, _dist): # try to get a latest version of the same package that is not our version other_remote_packages = [ - p for p in remote_repo.get_versions(local_package.name) + p + for p in remote_repo.get_versions(local_package.name) if p.version != local_package.version ] @@ -2534,10 +2629,11 @@ def get_other_dists(_package, _dist): # TODO: try to get data from dejacode if not local_dist.has_key_metadata(): - print(f'Unable to add essential ABOUT data for: {local_dist}') + print(f"Unable to add essential ABOUT data for: {local_dist}") if lic_errs: - lic_errs = '\n'.join(lic_errs) - print(f'Failed to fetch some licenses:: {lic_errs}') + lic_errs = "\n".join(lic_errs) + print(f"Failed to fetch some licenses:: {lic_errs}") + ################################################################################ # @@ -2551,19 +2647,18 @@ def call(args): Call args in a subprocess and display output on the fly. Return or raise stdout, stderr, returncode """ - if TRACE: print('Calling:', ' '.join(args)) + if TRACE: + print("Calling:", " ".join(args)) with subprocess.Popen( - args, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - encoding='utf-8' + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8" ) as process: while True: line = process.stdout.readline() if not line and process.poll() is not None: break - if TRACE: print(line.rstrip(), flush=True) + if TRACE: + print(line.rstrip(), flush=True) stdout, stderr = process.communicate() returncode = process.returncode @@ -2595,15 +2690,17 @@ def add_or_upgrade_built_wheels( Include wheels for all dependencies if `with_deps` is True. Build remotely is `build_remotely` is True. """ - assert name, 'Name is required' - ver = version and f'=={version}' or '' - print(f'\nAdding wheels for package: {name}{ver}') + assert name, "Name is required" + ver = version and f"=={version}" or "" + print(f"\nAdding wheels for package: {name}{ver}") wheel_filenames = [] # a mapping of {req specifier: {mapping build_wheels kwargs}} wheels_to_build = {} for python_version, operating_system in itertools.product(python_versions, operating_systems): - print(f' Adding wheels for package: {name}{ver} on {python_version,} and {operating_system}') + print( + f" Adding wheels for package: {name}{ver} on {python_version,} and {operating_system}" + ) environment = Environment.from_pyver_and_os(python_version, operating_system) # Check if requested wheel already exists locally for this version @@ -2617,7 +2714,7 @@ def add_or_upgrade_built_wheels( wheel_filenames.append(wheel.filename) break if has_local_wheel: - print(f' local wheel exists: {wheel.filename}') + print(f" local wheel exists: {wheel.filename}") continue if not version: @@ -2626,17 +2723,18 @@ def add_or_upgrade_built_wheels( # Check if requested wheel already exists remotely or in Pypi for this version wheel_filename = fetch_package_wheel( - name=name, version=version, environment=environment, dest_dir=dest_dir) + name=name, version=version, environment=environment, dest_dir=dest_dir + ) if wheel_filename: wheel_filenames.append(wheel_filename) # the wheel is not available locally, remotely or in Pypi # we need to build binary from sources - requirements_specifier = f'{name}=={version}' + requirements_specifier = f"{name}=={version}" to_build = wheels_to_build.get(requirements_specifier) if to_build: - to_build['python_versions'].append(python_version) - to_build['operating_systems'].append(operating_system) + to_build["python_versions"].append(python_version) + to_build["operating_systems"].append(operating_system) else: wheels_to_build[requirements_specifier] = dict( requirements_specifier=requirements_specifier, @@ -2682,7 +2780,7 @@ def build_wheels( dest_dir=dest_dir, ) for local_build in builds: - print(f'Built wheel: {local_build}') + print(f"Built wheel: {local_build}") if all_pure: return builds @@ -2718,36 +2816,43 @@ def build_wheels_remotely_on_multiple_platforms( """ check_romp_is_configured() pyos_options = get_romp_pyos_options(python_versions, operating_systems) - deps = '' if with_deps else '--no-deps' - verbose = '--verbose' if verbose else '' - - romp_args = ([ - 'romp', - '--interpreter', 'cpython', - '--architecture', 'x86_64', - '--check-period', '5', # in seconds - - ] + pyos_options + [ - - '--artifact-paths', '*.whl', - '--artifact', 'artifacts.tar.gz', - '--command', + deps = "" if with_deps else "--no-deps" + verbose = "--verbose" if verbose else "" + + romp_args = ( + [ + "romp", + "--interpreter", + "cpython", + "--architecture", + "x86_64", + "--check-period", + "5", # in seconds + ] + + pyos_options + + [ + "--artifact-paths", + "*.whl", + "--artifact", + "artifacts.tar.gz", + "--command", # create a virtualenv, upgrade pip -# f'python -m ensurepip --user --upgrade; ' - f'python -m pip {verbose} install --user --upgrade pip setuptools wheel; ' - f'python -m pip {verbose} wheel {deps} {requirements_specifier}', - ]) + # f'python -m ensurepip --user --upgrade; ' + f"python -m pip {verbose} install --user --upgrade pip setuptools wheel; " + f"python -m pip {verbose} wheel {deps} {requirements_specifier}", + ] + ) if verbose: - romp_args.append('--verbose') + romp_args.append("--verbose") - print(f'Building wheels for: {requirements_specifier}') - print(f'Using command:', ' '.join(romp_args)) + print(f"Building wheels for: {requirements_specifier}") + print(f"Using command:", " ".join(romp_args)) call(romp_args) - wheel_filenames = extract_tar('artifacts.tar.gz', dest_dir) + wheel_filenames = extract_tar("artifacts.tar.gz", dest_dir) for wfn in wheel_filenames: - print(f' built wheel: {wfn}') + print(f" built wheel: {wfn}") return wheel_filenames @@ -2763,12 +2868,16 @@ def get_romp_pyos_options( ... '--platform', 'windows'] >>> assert get_romp_pyos_options() == expected """ - python_dot_versions = ['.'.join(pv) for pv in sorted(set(python_versions))] - pyos_options = list(itertools.chain.from_iterable( - ('--version', ver) for ver in python_dot_versions)) + python_dot_versions = [".".join(pv) for pv in sorted(set(python_versions))] + pyos_options = list( + itertools.chain.from_iterable(("--version", ver) for ver in python_dot_versions) + ) - pyos_options += list(itertools.chain.from_iterable( - ('--platform' , plat) for plat in sorted(set(operating_systems)))) + pyos_options += list( + itertools.chain.from_iterable( + ("--platform", plat) for plat in sorted(set(operating_systems)) + ) + ) return pyos_options @@ -2776,17 +2885,18 @@ def get_romp_pyos_options( def check_romp_is_configured(): # these environment variable must be set before has_envt = ( - os.environ.get('ROMP_BUILD_REQUEST_URL') and - os.environ.get('ROMP_DEFINITION_ID') and - os.environ.get('ROMP_PERSONAL_ACCESS_TOKEN') and - os.environ.get('ROMP_USERNAME') + os.environ.get("ROMP_BUILD_REQUEST_URL") + and os.environ.get("ROMP_DEFINITION_ID") + and os.environ.get("ROMP_PERSONAL_ACCESS_TOKEN") + and os.environ.get("ROMP_USERNAME") ) if not has_envt: raise Exception( - 'ROMP_BUILD_REQUEST_URL, ROMP_DEFINITION_ID, ' - 'ROMP_PERSONAL_ACCESS_TOKEN and ROMP_USERNAME ' - 'are required enironment variables.') + "ROMP_BUILD_REQUEST_URL, ROMP_DEFINITION_ID, " + "ROMP_PERSONAL_ACCESS_TOKEN and ROMP_USERNAME " + "are required enironment variables." + ) def build_wheels_locally_if_pure_python( @@ -2804,19 +2914,24 @@ def build_wheels_locally_if_pure_python( Return a tuple of (True if all wheels are "pure", list of built wheel file names) """ - deps = [] if with_deps else ['--no-deps'] - verbose = ['--verbose'] if verbose else [] + deps = [] if with_deps else ["--no-deps"] + verbose = ["--verbose"] if verbose else [] - wheel_dir = tempfile.mkdtemp(prefix='scancode-release-wheels-local-') - cli_args = [ - 'pip', 'wheel', - '--wheel-dir', wheel_dir, - ] + deps + verbose + [ - requirements_specifier - ] + wheel_dir = tempfile.mkdtemp(prefix="scancode-release-wheels-local-") + cli_args = ( + [ + "pip", + "wheel", + "--wheel-dir", + wheel_dir, + ] + + deps + + verbose + + [requirements_specifier] + ) - print(f'Building local wheels for: {requirements_specifier}') - print(f'Using command:', ' '.join(cli_args)) + print(f"Building local wheels for: {requirements_specifier}") + print(f"Using command:", " ".join(cli_args)) call(cli_args) built = os.listdir(wheel_dir) @@ -2826,9 +2941,9 @@ def build_wheels_locally_if_pure_python( all_pure = all(is_pure_wheel(bwfn) for bwfn in built) if not all_pure: - print(f' Some wheels are not pure') + print(f" Some wheels are not pure") - print(f' Copying local wheels') + print(f" Copying local wheels") pure_built = [] for bwfn in built: owfn = os.path.join(dest_dir, bwfn) @@ -2836,7 +2951,7 @@ def build_wheels_locally_if_pure_python( nwfn = os.path.join(wheel_dir, bwfn) fileutils.copyfile(nwfn, owfn) pure_built.append(bwfn) - print(f' Built local wheel: {bwfn}') + print(f" Built local wheel: {bwfn}") return all_pure, pure_built @@ -2848,17 +2963,12 @@ def optimize_wheel(wheel_filename, dest_dir=THIRDPARTY_DIR): name of the new wheel if renamed or the existing new name otherwise. """ if is_pure_wheel(wheel_filename): - print(f'Pure wheel: {wheel_filename}, nothing to do.') + print(f"Pure wheel: {wheel_filename}, nothing to do.") return wheel_filename original_wheel_loc = os.path.join(dest_dir, wheel_filename) - wheel_dir = tempfile.mkdtemp(prefix='scancode-release-wheels-') - awargs = [ - 'auditwheel', - 'addtag', - '--wheel-dir', wheel_dir, - original_wheel_loc - ] + wheel_dir = tempfile.mkdtemp(prefix="scancode-release-wheels-") + awargs = ["auditwheel", "addtag", "--wheel-dir", wheel_dir, original_wheel_loc] call(awargs) audited = os.listdir(wheel_dir) @@ -2882,7 +2992,7 @@ def optimize_wheel(wheel_filename, dest_dir=THIRDPARTY_DIR): non_pypi_plats = utils_pypi_supported_tags.validate_platforms_for_pypi(new_wheel.platforms) new_wheel.platforms = [p for p in new_wheel.platforms if p not in non_pypi_plats] if not new_wheel.platforms: - print(f'Cannot make wheel PyPI compatible: {original_wheel_loc}') + print(f"Cannot make wheel PyPI compatible: {original_wheel_loc}") os.rename(new_wheel_loc, original_wheel_loc) return wheel_filename @@ -2892,18 +3002,20 @@ def optimize_wheel(wheel_filename, dest_dir=THIRDPARTY_DIR): return new_wheel_cleaned_filename -def extract_tar(location, dest_dir=THIRDPARTY_DIR,): +def extract_tar( + location, + dest_dir=THIRDPARTY_DIR, +): """ Extract a tar archive at `location` in the `dest_dir` directory. Return a list of extracted locations (either directories or files). """ - with open(location, 'rb') as fi: + with open(location, "rb") as fi: with tarfile.open(fileobj=fi) as tar: members = list(tar.getmembers()) tar.extractall(dest_dir, members=members) - return [os.path.basename(ti.name) for ti in members - if ti.type == tarfile.REGTYPE] + return [os.path.basename(ti.name) for ti in members if ti.type == tarfile.REGTYPE] def fetch_package_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR): @@ -2918,25 +3030,23 @@ def fetch_package_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR): wheel_filename = None remote_package = get_remote_package(name=name, version=version) if remote_package: - wheel_filename = remote_package.fetch_wheel( - environment=environment, dest_dir=dest_dir) + wheel_filename = remote_package.fetch_wheel(environment=environment, dest_dir=dest_dir) if wheel_filename: return wheel_filename pypi_package = get_pypi_package(name=name, version=version) if pypi_package: - wheel_filename = pypi_package.fetch_wheel( - environment=environment, dest_dir=dest_dir) + wheel_filename = pypi_package.fetch_wheel(environment=environment, dest_dir=dest_dir) return wheel_filename def check_about(dest_dir=THIRDPARTY_DIR): try: - subprocess.check_output(f'about check {dest_dir}'.split()) + subprocess.check_output(f"about check {dest_dir}".split()) except subprocess.CalledProcessError as cpe: print() - print('Invalid ABOUT files:') - print(cpe.output.decode('utf-8', errors='replace')) + print("Invalid ABOUT files:") + print(cpe.output.decode("utf-8", errors="replace")) def find_problems( @@ -2952,31 +3062,33 @@ def find_problems( for package in local_packages: if report_missing_sources and not package.sdist: - print(f'{package.name}=={package.version}: Missing source distribution.') + print(f"{package.name}=={package.version}: Missing source distribution.") if report_missing_wheels and not package.wheels: - print(f'{package.name}=={package.version}: Missing wheels.') + print(f"{package.name}=={package.version}: Missing wheels.") for dist in package.get_distributions(): dist.load_about_data(dest_dir=dest_dir) abpth = os.path.abspath(os.path.join(dest_dir, dist.about_filename)) if not dist.has_key_metadata(): - print(f' Missing key ABOUT data in file://{abpth}') - if 'classifiers' in dist.extra_data: - print(f' Dangling classifiers data in file://{abpth}') + print(f" Missing key ABOUT data in file://{abpth}") + if "classifiers" in dist.extra_data: + print(f" Dangling classifiers data in file://{abpth}") if not dist.validate_checksums(dest_dir): - print(f' Invalid checksums in file://{abpth}') + print(f" Invalid checksums in file://{abpth}") if not dist.sha1 and dist.md5: - print(f' Missing checksums in file://{abpth}') + print(f" Missing checksums in file://{abpth}") check_about(dest_dir=dest_dir) + def compute_normalized_license_expression(declared_licenses): if not declared_licenses: return try: from packagedcode import pypi + return pypi.compute_normalized_license(declared_licenses) except ImportError: # Scancode is not installed, clean and join all the licenses lics = [python_safe_name(l).lower() for l in declared_licenses] - return ' AND '.join(lics).lower() + return " AND ".join(lics).lower() From c8acebe48dccc9a5d3321b55c07ae327b7e21d71 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Wed, 6 Oct 2021 09:02:50 +0800 Subject: [PATCH 150/160] Add code to use curl if wget is not installed Signed-off-by: Chin Yeung Li --- configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure b/configure index b965692d..a141735f 100755 --- a/configure +++ b/configure @@ -127,7 +127,7 @@ create_virtualenv() { VIRTUALENV_PYZ="$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz" else VIRTUALENV_PYZ="$CFG_ROOT_DIR/$VENV_DIR/virtualenv.pyz" - wget -O "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL" + wget -O "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL" 2>/dev/null || curl -o "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL" fi $PYTHON_EXECUTABLE "$VIRTUALENV_PYZ" \ From 9f2b46a9b59c1ff0c7f59bb22802c745fa9dd2ad Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 8 Oct 2021 14:39:38 +0200 Subject: [PATCH 151/160] Do not issue warning if thirdparty dir is missing Signed-off-by: Philippe Ombredanne --- configure | 5 ++++- configure.bat | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/configure b/configure index a141735f..fdfdc855 100755 --- a/configure +++ b/configure @@ -51,7 +51,10 @@ CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org -PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty --find-links https://thirdparty.aboutcode.org/pypi" +if [ -f "$CFG_ROOT_DIR/thirdparty" ]; then + PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty " +fi +PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi" ################################ diff --git a/configure.bat b/configure.bat index 0c824a47..46ed4b36 100644 --- a/configure.bat +++ b/configure.bat @@ -49,7 +49,11 @@ set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" @rem ################################ @rem # Thirdparty package locations and index handling -set "PIP_EXTRA_ARGS=--find-links %CFG_ROOT_DIR%\thirdparty --find-links https://thirdparty.aboutcode.org/pypi" & %INDEX_ARG% +if exist ""%CFG_ROOT_DIR%\thirdparty"" ( + set "PIP_EXTRA_ARGS=--find-links %CFG_ROOT_DIR%\thirdparty " +) + +set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% --find-links https://thirdparty.aboutcode.org/pypi" & %INDEX_ARG% @rem ################################ From f6c1d21b36de6cc79cb889020ed44b5f002b8ccb Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 11 Oct 2021 22:29:48 +0200 Subject: [PATCH 152/160] Handle as_text correctly in cache Signed-off-by: Philippe Ombredanne --- etc/scripts/utils_thirdparty.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 7613a0c7..3ad746dc 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -918,7 +918,7 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): holder = raw_data['Author'] holder_contact = raw_data['Author-email'] - copyright_statement = f'Copyright (c) {holder} <{holder_contact}>' + copyright = f'Copyright (c) {holder} <{holder_contact}>' pkginfo_data = dict( name=raw_data['Name'], @@ -926,7 +926,7 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): version=raw_data['Version'], description=raw_data['Summary'], homepage_url=raw_data['Home-page'], - copyright=copyright_statement, + copyright=copyright, license_expression=license_expression, holder=holder, holder_contact=holder_contact, @@ -2927,7 +2927,7 @@ def fetch_package_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR): def check_about(dest_dir=THIRDPARTY_DIR): try: - subprocess.check_output(f'venv/bin/about check {dest_dir}'.split()) + subprocess.check_output(f'about check {dest_dir}'.split()) except subprocess.CalledProcessError as cpe: print() print('Invalid ABOUT files:') @@ -2965,7 +2965,6 @@ def find_problems( check_about(dest_dir=dest_dir) - def compute_normalized_license_expression(declared_licenses): if not declared_licenses: return From a118aa68a768a28c5d7d9f07d4568c88fd02b862 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 18 Oct 2021 13:53:58 +0200 Subject: [PATCH 153/160] Add support for Python 3.10 Signed-off-by: Philippe Ombredanne --- etc/scripts/README.rst | 12 ++++-------- etc/scripts/utils_thirdparty.py | 7 ++++++- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/etc/scripts/README.rst b/etc/scripts/README.rst index 4cb6ec7d..d8b00f98 100755 --- a/etc/scripts/README.rst +++ b/etc/scripts/README.rst @@ -1,10 +1,6 @@ -This directory contains the tools to: - -- manage a directory of thirdparty Python package source, wheels and metadata: - pin, build, update, document and publish to a PyPI-like repo (GitHub release) - -- build and publish scancode releases as wheel, sources and OS-specific bundles. - +This directory contains the tools to manage a directory of thirdparty Python +package source, wheels and metadata pin, build, update, document and publish to +a PyPI-like repo (GitHub release). NOTE: These are tested to run ONLY on Linux. @@ -38,7 +34,7 @@ Scripts ~~~~~~~ **gen_requirements.py**: create/update requirements files from currently - installed requirements. + installed requirements. **gen_requirements_dev.py** does the same but can subtract the main requirements to get extra requirements used in only development. diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 3ad746dc..090c3161 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -87,13 +87,14 @@ TRACE = False # Supported environments -PYTHON_VERSIONS = '36', '37', '38', '39', +PYTHON_VERSIONS = '36', '37', '38', '39', '310' ABIS_BY_PYTHON_VERSION = { '36':['cp36', 'cp36m'], '37':['cp37', 'cp37m'], '38':['cp38', 'cp38m'], '39':['cp39', 'cp39m'], + '310':['cp310', 'cp310m'], } PLATFORMS_BY_OS = { @@ -102,6 +103,7 @@ 'manylinux1_x86_64', 'manylinux2014_x86_64', 'manylinux2010_x86_64', + 'manylinux_2_12_x86_64', ], 'macos': [ 'macosx_10_6_intel', 'macosx_10_6_x86_64', @@ -112,6 +114,9 @@ 'macosx_10_13_intel', 'macosx_10_13_x86_64', 'macosx_10_14_intel', 'macosx_10_14_x86_64', 'macosx_10_15_intel', 'macosx_10_15_x86_64', + 'macosx_10_15_x86_64', + 'macosx_11_0_x86_64', + # 'macosx_11_0_arm64', ], 'windows': [ 'win_amd64', From 02169362d0fea7c3b306dda95b03aea49d0b9725 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 29 Oct 2021 17:07:01 +0200 Subject: [PATCH 154/160] Add missing dev requirements Signed-off-by: Philippe Ombredanne --- requirements-dev.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 240e5a20..75a17ba8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,3 +10,5 @@ toml==0.10.2 typing-extensions==3.10.0.2 zipp==3.6.0 black==21.9b0 +typed-ast==1.4.3 +pathspec==0.9.0 \ No newline at end of file From 31ed4461437bfb5df3a4c4eef9e559de8d7a07e0 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 29 Oct 2021 18:39:18 +0200 Subject: [PATCH 155/160] Drop Ubuntu 16 add Python 3.10 Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 22c12c43..b788ecb4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -7,19 +7,11 @@ jobs: - - template: etc/ci/azure-posix.yml - parameters: - job_name: ubuntu16_cpython - image_name: ubuntu-16.04 - python_versions: ['3.6', '3.7', '3.8', '3.9'] - test_suites: - all: venv/bin/pytest -vvs - - template: etc/ci/azure-posix.yml parameters: job_name: ubuntu18_cpython image_name: ubuntu-18.04 - python_versions: ['3.6', '3.7', '3.8', '3.9'] + python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -27,7 +19,7 @@ jobs: parameters: job_name: ubuntu20_cpython image_name: ubuntu-20.04 - python_versions: ['3.6', '3.7', '3.8', '3.9'] + python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -35,7 +27,7 @@ jobs: parameters: job_name: macos1014_cpython image_name: macos-10.14 - python_versions: ['3.6', '3.7', '3.8', '3.9'] + python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -43,7 +35,7 @@ jobs: parameters: job_name: macos1015_cpython image_name: macos-10.15 - python_versions: ['3.6', '3.7', '3.8', '3.9'] + python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -51,7 +43,7 @@ jobs: parameters: job_name: win2016_cpython image_name: vs2017-win2016 - python_versions: ['3.6', '3.7', '3.8', '3.9'] + python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -59,6 +51,6 @@ jobs: parameters: job_name: win2019_cpython image_name: windows-2019 - python_versions: ['3.6', '3.7', '3.8', '3.9'] + python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] test_suites: all: venv\Scripts\pytest -n 2 -vvs From fcb402dcf55bff56474ded2c282e4a7cb42f4089 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 29 Oct 2021 18:51:28 +0200 Subject: [PATCH 156/160] Use raw strings for regex Signed-off-by: Philippe Ombredanne --- src/univers/gentoo.py | 2 +- src/univers/versions.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/univers/gentoo.py b/src/univers/gentoo.py index 234db61c..ca6dae47 100644 --- a/src/univers/gentoo.py +++ b/src/univers/gentoo.py @@ -10,7 +10,7 @@ suffix_regexp = re.compile("^(alpha|beta|rc|pre|p)(\\d*)$") -revision_regexp = re.compile(".*(-r\d+)") +revision_regexp = re.compile(r".*(-r\d+)") suffix_value = {"pre": -2, "p": 1, "alpha": -4, "beta": -3, "rc": -1} diff --git a/src/univers/versions.py b/src/univers/versions.py index 737d9e11..9219417f 100644 --- a/src/univers/versions.py +++ b/src/univers/versions.py @@ -216,7 +216,7 @@ def __lt__(self, other): @attr.s(frozen=True, init=False, order=False, eq=False, hash=True, repr=False) class GentooVersion(BaseVersion): scheme = "ebuild" - version_re = re.compile("^(?:\d+)(?:\.\d+)*[a-zA-Z]?(?:_(p(?:re)?|beta|alpha|rc)\d*)*$") + version_re = re.compile(r"^(?:\d+)(?:\.\d+)*[a-zA-Z]?(?:_(p(?:re)?|beta|alpha|rc)\d*)*$") def __init__(self, version_string): version_string = remove_spaces(version_string) From 1a01ec34b3ace3f9f12c2f1bbb494c2805b48c90 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 29 Oct 2021 18:57:52 +0200 Subject: [PATCH 157/160] Use explicit directories for style checks Signed-off-by: Philippe Ombredanne --- tests/test_codestyle.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_codestyle.py b/tests/test_codestyle.py index 3d7d5b5e..53f571e9 100644 --- a/tests/test_codestyle.py +++ b/tests/test_codestyle.py @@ -10,11 +10,11 @@ class BaseTests(unittest.TestCase): def test_codestyle(self): - args = "python -m black --check -l 100 ." + args = "python -m black --check -l 100 *.py etc/ src/ tests/ docs" try: subprocess.check_output(args.split()) except Exception as e: raise Exception( - "Black style check failed; please format the code using: " - "python -m black --check -l 100" + "Black style check failed; please format the code using:\n" + " python -m black -l 100 *.py etc/ src/ tests/ docs" ) from e From 9d44053c7d4cf94c5ba80361ceedb316f85a054d Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 29 Oct 2021 19:06:06 +0200 Subject: [PATCH 158/160] Improve codestyle check Signed-off-by: Philippe Ombredanne --- tests/test_codestyle.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_codestyle.py b/tests/test_codestyle.py index 53f571e9..d089b7e2 100644 --- a/tests/test_codestyle.py +++ b/tests/test_codestyle.py @@ -10,11 +10,15 @@ class BaseTests(unittest.TestCase): def test_codestyle(self): - args = "python -m black --check -l 100 *.py etc/ src/ tests/ docs" + args = "venv/bin/black --check -l 100 setup.py etc src tests docs" try: subprocess.check_output(args.split()) - except Exception as e: + except subprocess.CalledProcessError as e: + print("===========================================================") + print(e.output) + print("===========================================================") raise Exception( "Black style check failed; please format the code using:\n" - " python -m black -l 100 *.py etc/ src/ tests/ docs" + " python -m black -l 100 setup.py etc src tests docs", + e.output, ) from e From 2ce7c7a1f1d4028d163de3a07a61b7c1febb9ae1 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 29 Oct 2021 19:14:08 +0200 Subject: [PATCH 159/160] Disable Python 3.10 tests on macOS 10.14 Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b788ecb4..7cd30254 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -27,7 +27,7 @@ jobs: parameters: job_name: macos1014_cpython image_name: macos-10.14 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: all: venv/bin/pytest -n 2 -vvs From 92c9db1bd952965a7ad8890947592865dddebfeb Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 29 Oct 2021 19:24:18 +0200 Subject: [PATCH 160/160] Use lowercase for univers library name Signed-off-by: Philippe Ombredanne --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 921fa736..55b752ab 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -Univers: mostly universal version and version ranges comparison and conversion +univers: mostly universal version and version ranges comparison and conversion =============================================================================== |Build Status| |License| |Python 3.6+| @@ -40,7 +40,7 @@ univers is different and considers the ecosystem-specific version scheme used. How does univers work ? ========================= -Univers wraps, embeds or implements multiple version comparision libraries, each +univers wraps, embeds or implements multiple version comparision libraries, each focused on specific ecosystem version scheme. It also implements an experimental unified syntax for version ranges specifier