From 52741200b24933d3c4af1f5a7fa5124d29d5daf6 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 27 Mar 2018 00:00:25 +0100 Subject: [PATCH 1/4] Use pipenv also for the default dependencies --- .../dependencies/pinned-requirements.txt | 91 +------------ .../dependencies/pipfiles/default/Pipfile | 55 ++++++++ .../pipfiles/default/pinned-requirements.txt | 125 ++++++++++++++++++ lib/galaxy/dependencies/pipfiles/update.sh | 3 +- lib/galaxy/dependencies/requirements.txt | 70 ---------- 5 files changed, 183 insertions(+), 161 deletions(-) mode change 100644 => 120000 lib/galaxy/dependencies/pinned-requirements.txt create mode 100644 lib/galaxy/dependencies/pipfiles/default/Pipfile create mode 100644 lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt delete mode 100644 lib/galaxy/dependencies/requirements.txt diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt deleted file mode 100644 index 71b0abf4de1..00000000000 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ /dev/null @@ -1,90 +0,0 @@ -# bx-python and dependencies -numpy==1.9.2 -python-lzo==1.11 -bx-python==0.8.1 - -# packages with C extensions -MarkupSafe==1.0 -PyYAML==3.12 -SQLAlchemy==1.2.2 -sqlalchemy-utils==0.32.19 -mercurial==3.7.3; python_version < '3.0' -pycrypto==2.6.1 -uWSGI==2.0.15 -# Flexible BAM index naming is new to core pysam -pysam==0.14 - -# pure Python packages -bdbag==1.1.1 -bleach==2.1.3 -bz2file==0.98; python_version < '3.3' -ipaddress==1.0.18; python_version < '3.3' -boltons==17.1.0 -Paste==2.0.3 -PasteDeploy==1.5.2 -docutils==0.14 -repoze.lru==0.6 -Routes==2.4.1 -WebOb==1.4.1 -Mako==1.0.2 -pytz==2018.3 -Babel==2.5.3 -Beaker==1.7.0 -dictobj==0.3.1 -nose==1.3.7 -Parsley==1.3 -six==1.11.0 -Whoosh==2.7.4 -galaxy_sequence_utils==1.0.2 -h5py==2.7.1 -packaging==17.1 - -# pykwalify and dependencies -pykwalify==1.6.0 -python-dateutil==2.5.3 -docopt==0.6.2 - -# Cheetah and dependencies -Cheetah3==3.0.0 -Markdown==2.6.3 - -# Requests and dependencies -requests==2.18.4 -certifi==2018.1.18 -urllib3==1.22 -chardet==3.0.4 -idna==2.6 - -# BioBlend and dependencies -bioblend==0.7.0 -boto==2.38.0 -requests-toolbelt==0.8.0 - - -# kombu and dependencies -kombu==3.0.30 -amqp==1.4.8 -anyjson==0.3.3 - -# Pulsar requirements -psutil==4.1.0 -pulsar-galaxy-lib==0.8.3 - -# sqlalchemy-migrate and dependencies -sqlalchemy-migrate==0.11.0 -decorator==4.0.2 -Tempita==0.5.3dev -sqlparse==0.1.16 -pbr==3.1.1 - -# svgwrite and dependencies -svgwrite==1.1.6 -pyparsing==2.2.0 - -# Fabric and dependencies -Fabric3==1.13.1.post1 -paramiko==2.2.1 -ecdsa==0.13 - -# GenomeSpace dependencies -python-genomespaceclient==0.1.8 diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt new file mode 120000 index 00000000000..26ec2b5e875 --- /dev/null +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -0,0 +1 @@ +pipfiles/default/pinned-requirements.txt \ No newline at end of file diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile new file mode 100644 index 00000000000..7dc4dda8681 --- /dev/null +++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile @@ -0,0 +1,55 @@ +[[source]] +url = "https://pypi.python.org/simple" +verify_ssl = true +name = "pypi" + +[dev-packages] + +[packages] +numpy = "*" +bx-python = "*" +MarkupSafe = "*" +PyYAML = "*" +SQLAlchemy = "*" +SQLAlchemy-Utils = "*" +Mercurial = {version = "*", markers = "python_version < '3'"} +pycrypto = "*" +uWSGI = "*" +pysam = "*" +bdbag = "*" +bleach = "*" +"bz2file" = {version = "*", markers = "python_version < '3.3'"} +ipaddress = {version = "*", markers = "python_version < '3.3'"} +boltons = "*" +Paste = "*" +docutils = "*" +Routes = "*" +WebOb = "*" +Mako = "*" +Babel = "*" +Beaker = "*" +dictobj = "*" +nose = "*" +Parsley = "*" +six = "*" +Whoosh = "*" +galaxy_sequence_utils = "*" +"h5py" = "*" +pykwalify = "*" +"Cheetah3" = "*" +requests = "*" +bioblend = "*" +boto = "*" +kombu = "*" +psutil = "*" +pulsar-galaxy-lib = "*" +sqlalchemy-migrate = "*" +sqlparse = "*" +svgwrite = "*" +pyparsing = "*" +"Fabric3" = "*" +paramiko = "*" +python-genomespaceclient = "*" + +[requires] +python_version = "2.7" diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt new file mode 100644 index 00000000000..6c55287f03b --- /dev/null +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -0,0 +1,125 @@ +amqp==2.2.2 +appdirs==1.4.3 +asn1crypto==0.24.0 +babel==2.5.3 +bagit==1.6.4 +bcrypt==3.1.4 +bdbag==1.2.4 +beaker==1.9.0 +bioblend==0.10.0 +bleach==2.1.3 +boltons==18.0.0 +boto==2.46.1 +bunch==1.0.1 +bx-python==0.8.1 +bz2file==0.98; python_version < '3.3' +certifi==2018.1.18 +cffi==1.11.5 +chardet==3.0.4 +cheetah3==3.1.0 +cliff==2.11.0 +cloudbridge==0.3.3 +cmd2==0.8.2 +contextlib2==0.5.5 +cryptography==2.2.1 +debtcollector==1.19.0 +decorator==4.2.1 +deprecation==2.0 +dictobj==0.4 +docopt==0.6.2 +docutils==0.14 +dogpile.cache==0.6.5 +enum34==1.1.6; python_version < '3' +fabric3==1.14.post1 +funcsigs==1.0.2 +functools32==3.2.3.post2; python_version == '2.7' +futures==3.2.0; python_version == '2.7' or python_version == '2.6' +galaxy-sequence-utils==1.1.2 +globus-sdk==1.3.0 +h5py==2.7.1 +html5lib==1.0.1 +idna==2.6 +ipaddress==1.0.19; python_version < '3.3' +iso8601==0.1.12 +jmespath==0.9.3 +jsonpatch==1.21 +jsonpointer==2.0 +jsonschema==2.6.0 +keystoneauth1==3.4.0 +kombu==4.1.0 +mako==1.0.7 +markupsafe==1.0 +mercurial==4.5.2; python_version < '3' +monotonic==1.4 +msgpack==0.5.6 +munch==2.2.0 +netaddr==0.7.19 +netifaces==0.10.6 +nose==1.3.7 +numpy==1.14.2 +openstacksdk==0.12.0 +os-client-config==1.29.0 +os-service-types==1.2.0 +osc-lib==1.10.0 +oslo.config==5.2.0 +oslo.i18n==3.20.0 +oslo.serialization==2.25.0 +oslo.utils==3.36.0 +packaging==17.1 +paramiko==2.4.1 +parsley==1.3 +paste==2.0.3 +pastedeploy==1.5.2 +pastescript==2.0.2 +pbr==3.1.1 +positional==1.2.1 +prettytable==0.7.2 +psutil==5.4.3 +pulsar-galaxy-lib==0.8.3 +py2-ipaddress==3.4.1 +pyasn1==0.4.2 +pycparser==2.18 +pycrypto==2.6.1 +pykwalify==1.6.1 +pynacl==1.2.1 +pyparsing==2.2.0 +pyperclip==1.6.0 +pysam==0.14.1 +python-cinderclient==2.0.1 +python-dateutil==2.7.2 +python-genomespaceclient==1.0.0 +python-glanceclient==2.6.0 +python-keystoneclient==3.10.0 +python-lzo==1.11 +python-neutronclient==6.1.0 +python-novaclient==7.0.0 +python-swiftclient==3.3.0 +pytz==2018.3 +pyyaml==3.12 +repoze.lru==0.7 +requests-toolbelt==0.8.0 +requests==2.18.4 +requestsexceptions==1.4.0 +retrying==1.3.3 +rfc3986==1.1.0 +routes==2.4.1 +simplejson==3.13.2 +six==1.11.0 +sqlalchemy-migrate==0.11.0 +sqlalchemy-utils==0.33.1 +sqlalchemy==1.2.5 +sqlparse==0.2.4 +stevedore==1.28.0 +subprocess32==3.2.7 +svgwrite==1.1.12 +tempita==0.5.2 +tzlocal==1.5.1 +unicodecsv==0.14.1; python_version < '3.0' +urllib3==1.22 +uwsgi==2.0.17 +vine==1.1.4 +warlock==1.2.0 +webencodings==0.5.1 +webob==1.7.4 +whoosh==2.7.4 +wrapt==1.10.11 diff --git a/lib/galaxy/dependencies/pipfiles/update.sh b/lib/galaxy/dependencies/pipfiles/update.sh index 4c369e39f24..f6a179e2ae6 100755 --- a/lib/galaxy/dependencies/pipfiles/update.sh +++ b/lib/galaxy/dependencies/pipfiles/update.sh @@ -31,7 +31,8 @@ done THIS_DIRECTORY="$(cd "$(dirname "$0")" > /dev/null && pwd)" ENVS="develop -flake8" +flake8 +default" for env in $ENVS do diff --git a/lib/galaxy/dependencies/requirements.txt b/lib/galaxy/dependencies/requirements.txt deleted file mode 100644 index 6886784794c..00000000000 --- a/lib/galaxy/dependencies/requirements.txt +++ /dev/null @@ -1,70 +0,0 @@ -# bx-python and dependencies -numpy -python-lzo -bx-python - -# packages with C extensions -MarkupSafe -PyYAML -SQLAlchemy -mercurial; python_version < '3.0' -pycrypto -# Flexible BAM index naming is new to main pysam -pysam>=0.13 - -# pure Python packages -bleach -bz2file; python_version < '3.3' -ipaddress; python_version < '3.3' -boltons -Paste -PasteDeploy -docutils -repoze.lru -Routes -WebOb -Mako -pytz -Babel -Beaker -dictobj -nose -Parsley -six -Whoosh -galaxy_sequence_utils -h5py -packaging - -# Cheetah and dependencies -Cheetah3 -Markdown - -# BioBlend and dependencies -bioblend -boto -requests -requests-toolbelt - -# kombu and dependencies -kombu - -# Pulsar requirements -psutil -pulsar-galaxy-lib>=0.8.3 - -# sqlalchemy-migrate and dependencies -sqlalchemy-migrate -decorator -Tempita -sqlparse -pbr - -# svgwrite and dependencies -svgwrite -pyparsing - -# Fabric and dependencies -Fabric3 -paramiko -ecdsa From fb92257bc755ae12a5dad810c152967bfc68d8a1 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 27 Mar 2018 01:25:58 +0100 Subject: [PATCH 2/4] Exclude 2 problematic h5py versions A warning was appearing in the stderr of upload jobs due to https://github.com/h5py/h5py/issues/995 . --- lib/galaxy/dependencies/pipfiles/default/Pipfile | 2 +- .../dependencies/pipfiles/default/pinned-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile index 7dc4dda8681..34de13d0900 100644 --- a/lib/galaxy/dependencies/pipfiles/default/Pipfile +++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile @@ -34,7 +34,7 @@ Parsley = "*" six = "*" Whoosh = "*" galaxy_sequence_utils = "*" -"h5py" = "*" +"h5py" = "!=2.7.0, !=2.7.1" pykwalify = "*" "Cheetah3" = "*" requests = "*" diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 6c55287f03b..2c312bb0840 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -36,7 +36,7 @@ functools32==3.2.3.post2; python_version == '2.7' futures==3.2.0; python_version == '2.7' or python_version == '2.6' galaxy-sequence-utils==1.1.2 globus-sdk==1.3.0 -h5py==2.7.1 +h5py==2.6.0 html5lib==1.0.1 idna==2.6 ipaddress==1.0.19; python_version < '3.3' From f8aa29e2805b932a03a577fd329f881a3d047c98 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 27 Mar 2018 03:40:40 +0100 Subject: [PATCH 3/4] Temporarily pin WebOb to latest compatible release --- lib/galaxy/dependencies/pipfiles/default/Pipfile | 2 +- .../dependencies/pipfiles/default/pinned-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile index 34de13d0900..c7d4196652e 100644 --- a/lib/galaxy/dependencies/pipfiles/default/Pipfile +++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile @@ -24,7 +24,7 @@ boltons = "*" Paste = "*" docutils = "*" Routes = "*" -WebOb = "*" +WebOb = "<=1.4.2" Mako = "*" Babel = "*" Beaker = "*" diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 2c312bb0840..f050bd630e6 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -120,6 +120,6 @@ uwsgi==2.0.17 vine==1.1.4 warlock==1.2.0 webencodings==0.5.1 -webob==1.7.4 +webob==1.4.2 whoosh==2.7.4 wrapt==1.10.11 From e625b649dc0e823d0c92f8c997254215bcf9dcab Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 27 Mar 2018 08:47:38 +0100 Subject: [PATCH 4/4] Temporarily pin Mercurial to latest compatible release --- lib/galaxy/dependencies/pipfiles/default/Pipfile | 2 +- .../dependencies/pipfiles/default/pinned-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile index c7d4196652e..d31a76a4829 100644 --- a/lib/galaxy/dependencies/pipfiles/default/Pipfile +++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile @@ -12,7 +12,7 @@ MarkupSafe = "*" PyYAML = "*" SQLAlchemy = "*" SQLAlchemy-Utils = "*" -Mercurial = {version = "*", markers = "python_version < '3'"} +Mercurial = {version = "<=3.7.3", markers = "python_version < '3'"} pycrypto = "*" uWSGI = "*" pysam = "*" diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index f050bd630e6..495aff6fe84 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -49,7 +49,7 @@ keystoneauth1==3.4.0 kombu==4.1.0 mako==1.0.7 markupsafe==1.0 -mercurial==4.5.2; python_version < '3' +mercurial==3.7.3; python_version < '3' monotonic==1.4 msgpack==0.5.6 munch==2.2.0