diff --git a/.ci/jenkins/api-py3 b/.ci/jenkins/api-py3 new file mode 120000 index 00000000000..9e5bfb42d23 --- /dev/null +++ b/.ci/jenkins/api-py3 @@ -0,0 +1 @@ +api \ No newline at end of file diff --git a/.ci/jenkins/api-py3/run_tests.sh b/.ci/jenkins/api-py3/run_tests.sh deleted file mode 100755 index 24790d0b720..00000000000 --- a/.ci/jenkins/api-py3/run_tests.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -./run_tests.sh --dockerize --python3 --db postgres --clean_pyc --skip_flakey_fails -api "$@" diff --git a/.ci/jenkins/framework-py3 b/.ci/jenkins/framework-py3 new file mode 120000 index 00000000000..e3939d229ce --- /dev/null +++ b/.ci/jenkins/framework-py3 @@ -0,0 +1 @@ +framework \ No newline at end of file diff --git a/.ci/jenkins/framework-py3/run_tests.sh b/.ci/jenkins/framework-py3/run_tests.sh deleted file mode 100755 index f8bfe97335d..00000000000 --- a/.ci/jenkins/framework-py3/run_tests.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -./run_tests.sh --dockerize --python3 --db postgres --clean_pyc --framework "$@" diff --git a/.ci/jenkins/integration-py3 b/.ci/jenkins/integration-py3 new file mode 120000 index 00000000000..c0bfe0d4f94 --- /dev/null +++ b/.ci/jenkins/integration-py3 @@ -0,0 +1 @@ +integration \ No newline at end of file diff --git a/.ci/jenkins/integration-py3/run_tests.sh b/.ci/jenkins/integration-py3/run_tests.sh deleted file mode 100755 index ffc6c87e98c..00000000000 --- a/.ci/jenkins/integration-py3/run_tests.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -DOCKER_RUN_EXTRA_ARGS="--privileged" ./run_tests.sh --dockerize --python3 --db postgres --clean_pyc --integration "$@" diff --git a/.ci/jenkins/main-tools-py3 b/.ci/jenkins/main-tools-py3 new file mode 120000 index 00000000000..49589fdd3a8 --- /dev/null +++ b/.ci/jenkins/main-tools-py3 @@ -0,0 +1 @@ +main-tools \ No newline at end of file diff --git a/.ci/jenkins/main-tools-py3/run_tests.sh b/.ci/jenkins/main-tools-py3/run_tests.sh deleted file mode 100755 index 442b15cdf2d..00000000000 --- a/.ci/jenkins/main-tools-py3/run_tests.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -./run_tests.sh --dockerize --python3 --db postgres --clean_pyc -main "$@" diff --git a/.ci/jenkins/selenium-compose/run_galaxy.bash b/.ci/jenkins/selenium-compose/run_galaxy.bash index aaf10b77401..c16901324c1 100755 --- a/.ci/jenkins/selenium-compose/run_galaxy.bash +++ b/.ci/jenkins/selenium-compose/run_galaxy.bash @@ -20,8 +20,7 @@ virtualenv "$GALAXY_VIRTUAL_ENV" chown -R "$GALAXY_TEST_UID:$GALAXY_TEST_UID" "$GALAXY_VIRTUAL_ENV" cd /galaxy -HOME=/galaxy -sudo -E -u "#${GALAXY_TEST_UID}" ./scripts/common_startup.sh || { echo "common_startup.sh failed"; exit 1; } +sudo -E -H -u "#${GALAXY_TEST_UID}" ./scripts/common_startup.sh || { echo "common_startup.sh failed"; exit 1; } echo "Waiting for postgres to become available" while ! nc -z postgres 5432; @@ -34,7 +33,7 @@ echo "Creating postgres database for Galaxy" createdb -w -U postgres -h postgres galaxy echo "Starting and waiting for Galaxy daemon(s)" -sudo -E -u "#${GALAXY_TEST_UID}" GALAXY_RUN_ALL=1 bash "$GALAXY_ROOT/run.sh" --daemon --wait +sudo -E -H -u "#${GALAXY_TEST_UID}" GALAXY_RUN_ALL=1 bash "$GALAXY_ROOT/run.sh" --daemon --wait echo "Galaxy daemon ready, monitoring Galaxy logs" tail -f "$GALAXY_ROOT/main.log" diff --git a/.ci/jenkins/selenium-py3 b/.ci/jenkins/selenium-py3 new file mode 120000 index 00000000000..954f0db0bc0 --- /dev/null +++ b/.ci/jenkins/selenium-py3 @@ -0,0 +1 @@ +selenium \ No newline at end of file diff --git a/.ci/jenkins/selenium-py3/run_tests.sh b/.ci/jenkins/selenium-py3/run_tests.sh deleted file mode 100755 index 827151835f6..00000000000 --- a/.ci/jenkins/selenium-py3/run_tests.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# Enable retries on tests to reduce chances of transient failures. -: ${GALAXY_TEST_SELENIUM_RETRIES:=1} - -# If in Jenkins environment, use it for artifacts. -if [ -n "$BUILD_NUMBER" ]; -then - : ${GALAXY_TEST_ERRORS_DIRECTORY:=${BUILD_NUMBER}-test-errors} - : ${GALAXY_TEST_SCREENSHOTS_DIRECTORY:=${BUILD_NUMBER}-test-screenshots} -else - : ${GALAXY_TEST_ERRORS_DIRECTORY:=database/test-errors} - : ${GALAXY_TEST_SCREENSHOTS_DIRECTORY:=database/test-screenshots} -fi - -mkdir -p "$GALAXY_TEST_ERRORS_DIRECTORY" -mkdir -p "$GALAXY_TEST_SCREENSHOTS_DIRECTORY" - -mkdir -p ~/.jenkins-yarn-cache -YARN_CACHE_FOLDER=~/.jenkins-yarn-cache - -# Start Selenium server in the test Docker container. -DOCKER_RUN_EXTRA_ARGS="${DOCKER_RUN_EXTRA_ARGS} --shm-size=2g -v $YARN_CACHE_FOLDER:$YARN_CACHE_FOLDER -e YARN_CACHE_FOLDER=$YARN_CACHE_FOLDER -e USE_SELENIUM=1 -e GALAXY_TEST_SELENIUM_RETRIES=${GALAXY_TEST_SELENIUM_RETRIES} -e GALAXY_TEST_ERRORS_DIRECTORY=${GALAXY_TEST_ERRORS_DIRECTORY} -e GALAXY_TEST_SCREENSHOTS_DIRECTORY=${GALAXY_TEST_SCREENSHOTS_DIRECTORY}" -export DOCKER_RUN_EXTRA_ARGS - -./run_tests.sh --dockerize --python3 --db postgres --clean_pyc --skip_flakey_fails --selenium "$@" diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt index a66925ec2f3..5d27d5abf3d 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt @@ -4,20 +4,20 @@ alabaster==0.7.12 atomicwrites==1.3.0 attrs==19.3.0 babel==2.8.0 -certifi==2019.11.28 +certifi==2020.4.5.1 chardet==3.0.4 commonmark==0.9.1 configparser==4.0.2 ; python_version < '3.2' contextlib2==0.6.0.post1 ; python_version < '3.5' -coverage==5.0.3 -deprecated==1.2.7 +coverage==5.0.4 +deprecated==1.2.9 docutils==0.15.2 funcsigs==1.0.2 ; python_version < '3.3' future==0.18.2 gunicorn==19.10.0 idna==2.9 imagesize==1.2.0 -importlib-metadata==1.5.0 ; python_version < '3.8' +importlib-metadata==1.6.0 ; python_version < '3.8' jinja2==2.11.1 lxml==4.5.0 markdown==3.1.1 @@ -27,17 +27,18 @@ mock==3.0.5 more-itertools==5.0.0 nose==1.3.7 nosehtml==0.4.5 -packaging==20.1 +packaging==20.3 pathlib2==2.3.5 ; python_version < '3.6' pathtools==0.1.2 pluggy==0.13.1 port-for==0.4 +prettytable==0.7.2 psutil==5.7.0 py==1.8.1 pygithub==1.45 pygments==2.5.2 pyjwt==1.7.1 -pyparsing==2.4.6 +pyparsing==2.4.7 pytest-cov==2.8.1 pytest-html==1.22.1 pytest-metadata==1.8.0 @@ -61,6 +62,7 @@ twill==2.0 typing==3.7.4.1 ; python_version < '3.5' urllib3==1.25.8 watchdog==0.10.2 -wcwidth==0.1.8 -wrapt==1.12.0 +wcwidth==0.1.9 +wrapt==1.12.1 +xmlrunner==1.7.7 zipp==1.2.0 diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 0d47bdffdc2..ba38a4adb57 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -23,8 +23,8 @@ bcrypt==3.1.7 bdbag==1.5.6 beaker==1.11.0 bioblend==0.13.0 -bleach==3.1.1 -boltons==20.0.0 +bleach==3.1.4 +boltons==20.1.0 boto3==1.9.114 boto==2.49.0 botocore==1.12.253 @@ -32,7 +32,7 @@ bx-python==0.8.8 bz2file==0.98 ; python_version < '3.3' cachecontrol==0.11.7 cachetools==3.1.1 -certifi==2019.11.28 +certifi==2020.4.5.1 cffi==1.14.0 chardet==3.0.4 cheetah3==3.2.4 @@ -43,18 +43,18 @@ cmd2==0.8.9 coloredlogs==14.0 configparser==4.0.2 ; python_version < '3.2' contextlib2==0.6.0.post1 ; python_version < '3.5' -cryptography==2.8 +cryptography==2.9 cwltool==1.0.20191225192155 debtcollector==1.22.0 decorator==4.4.2 -deprecated==1.2.7 +deprecated==1.2.9 deprecation==2.0.7 dictobj==0.4 docopt==0.6.2 docutils==0.15.2 dogpile.cache==0.9.0 ecdsa==0.15 -enum34==1.1.9 ; python_version < '3.4' +enum34==1.1.10 ; python_version < '3.4' fabric3==1.14.post1 funcsigs==1.0.2 ; python_version < '3.3' functools32==3.2.3.post2 ; python_version < '3.2' @@ -63,13 +63,13 @@ futures==3.3.0 ; python_version == '2.6' or python_version == '2.7' galaxy-sequence-utils==1.1.5 google-api-python-client==1.7.8 google-auth-httplib2==0.0.3 -google-auth==1.11.2 -gxformat2==0.10.1 +google-auth==1.13.1 +gxformat2==0.11.1 h5py==2.10.0 -httplib2==0.17.0 -humanfriendly==7.1.1 +httplib2==0.17.2 +humanfriendly==8.1 idna==2.9 -importlib-metadata==1.5.0 ; python_version < '3.8' +importlib-metadata==1.6.0 ; python_version < '3.8' ipaddress==1.0.23 ; python_version < '3.3' isa-rwval==0.10.7 iso8601==0.1.12 @@ -78,14 +78,14 @@ jmespath==0.9.5 jsonpatch==1.25 jsonpointer==2.0 jsonschema==3.2.0 -keystoneauth1==3.18.0 -kombu==4.6.7 +keystoneauth1==4.0.0 +kombu==4.6.8 lockfile==0.12.2 lxml==4.5.0 -mako==1.1.1 +mako==1.1.2 markdown==3.1.1 markupsafe==1.1.1 -mercurial==5.3 +mercurial==5.3.2 mistune==0.8.4 monotonic==1.5 msgpack==1.0.0 @@ -102,7 +102,7 @@ numpy==1.16.6 oauth2client==4.1.3 oauthlib==3.1.0 openstacksdk==0.17.0 -os-client-config==2.0.0 +os-client-config==2.1.0 os-service-types==1.7.0 osc-lib==2.0.0 oslo.config==7.0.0 @@ -111,21 +111,21 @@ oslo.i18n==3.25.1 oslo.log==3.45.2 oslo.serialization==2.29.2 oslo.utils==3.42.1 -packaging==20.1 +packaging==20.3 paramiko==2.7.1 parsley==1.3 paste==3.4.0 pastedeploy==2.1.0 pastescript==3.2.0 pathlib2==2.3.5 ; python_version < '3.6' -pbr==5.4.4 +pbr==5.4.5 prettytable==0.7.2 prov==1.5.1 psutil==5.7.0 pulsar-galaxy-lib==0.14.0.dev1 pyasn1-modules==0.2.8 pyasn1==0.4.8 -pycparser==2.19 +pycparser==2.20 pycryptodome==3.9.7 pyeventsystem==0.1.0 pyinotify==0.9.6 ; sys_platform != 'win32' and sys_platform != 'darwin' and sys_platform != 'sunos5' @@ -133,9 +133,9 @@ pyjwt==1.7.1 pykwalify==1.7.0 pynacl==1.3.0 pyopenssl==19.1.0 -pyparsing==2.4.6 -pyperclip==1.7.0 -pyrsistent==0.15.7 +pyparsing==2.4.7 +pyperclip==1.8.0 +pyrsistent==0.16.0 pysam==0.15.2 pysftp==0.2.9 python-cinderclient==4.0.0 @@ -149,7 +149,7 @@ python-openid==2.2.5 ; python_version < '3.0' python-swiftclient==3.6.0 pytz==2019.3 pyuwsgi==2.0.18.post0 -pyyaml==5.3 +pyyaml==5.3.1 rdflib-jsonld==0.4.0 rdflib==4.2.2 repoze.lru==0.7 @@ -157,7 +157,7 @@ requests-oauthlib==1.3.0 requests-toolbelt==0.9.1 requests==2.23.0 requestsexceptions==1.4.0 -rfc3986==1.3.2 +rfc3986==1.4.0 routes==2.4.1 rsa==4.0 ruamel.ordereddict==0.4.14 ; platform_python_implementation == 'CPython' and python_version <= '2.7' @@ -172,15 +172,15 @@ simplejson==3.17.0 six==1.11.0 social-auth-core[openidconnect]==3.1.0+gx0 sqlalchemy-migrate==0.13.0 -sqlalchemy-utils==0.36.1 -sqlalchemy==1.3.13 +sqlalchemy-utils==0.36.3 +sqlalchemy==1.3.16 sqlparse==0.3.1 stevedore==1.32.0 subprocess32==3.5.4 ; python_version < '3.0' svgwrite==1.3.1 tempita==0.5.2 tenacity==4.12.0 -typing-extensions==3.7.4.1 +typing-extensions==3.7.4.2 typing==3.7.4.1 ; python_version < '3.5' tzlocal==2.0.0 unicodecsv==0.14.1 ; python_version < '3.0' @@ -188,9 +188,9 @@ uritemplate==3.0.1 urllib3==1.25.8 vine==1.3.0 warlock==1.3.3 -wcwidth==0.1.8 +wcwidth==0.1.9 webencodings==0.5.1 webob==1.8.6 whoosh==2.7.4 -wrapt==1.12.0 +wrapt==1.12.1 zipp==1.2.0 diff --git a/lib/galaxy/dependencies/pipfiles/flake8/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/flake8/pinned-requirements.txt index 0a1aafdbde5..aa48e32250e 100644 --- a/lib/galaxy/dependencies/pipfiles/flake8/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/flake8/pinned-requirements.txt @@ -1,7 +1,7 @@ -i https://pypi.python.org/simple configparser==4.0.2 ; python_version < '3.2' entrypoints==0.3 -enum34==1.1.9 ; python_version < '3.4' +enum34==1.1.10 ; python_version < '3.4' flake8-import-order==0.18.1 flake8==3.7.9 functools32==3.2.3.post2 ; python_version < '3.2' diff --git a/lib/galaxy_test/api/embed_test_1.gxwf.yml b/lib/galaxy_test/api/embed_test_1.gxwf.yml index 76461dff6fd..e43f9426582 100644 --- a/lib/galaxy_test/api/embed_test_1.gxwf.yml +++ b/lib/galaxy_test/api/embed_test_1.gxwf.yml @@ -14,7 +14,7 @@ steps: - tool_id: cat1 state: input1: - $link: first_cat#out_file1 + $link: first_cat/out_file1 queries: - input2: - $link: embed1#output1 + $link: embed1/output1 diff --git a/lib/galaxy_test/api/test_workflow_extraction.py b/lib/galaxy_test/api/test_workflow_extraction.py index 048f20aa237..6e94c1839d2 100644 --- a/lib/galaxy_test/api/test_workflow_extraction.py +++ b/lib/galaxy_test/api/test_workflow_extraction.py @@ -228,7 +228,7 @@ steps: - tool_id: cat_collection state: input1: - $link: noop#out_file1 + $link: noop/out_file1 test_data: text_input1: type: "list:paired" @@ -274,11 +274,11 @@ steps: tool_id: collection_split_on_column state: input1: - $link: cat_inputs#out_file1 + $link: cat_inputs/out_file1 - tool_id: cat_list state: input1: - $link: split_up#split_output + $link: split_up/split_output test_data: text_input1: "samp1\t10.0\nsamp2\t20.0\n" text_input2: "samp1\t30.0\nsamp2\t40.0\n" @@ -316,16 +316,16 @@ steps: tool_id: collection_creates_pair state: input1: - $link: cat_inputs#out_file1 + $link: cat_inputs/out_file1 - label: cat_pairs tool_id: cat_collection state: input1: - $link: pair_off#paired_output + $link: pair_off/paired_output - tool_id: cat_list state: input1: - $link: cat_pairs#out_file1 + $link: cat_pairs/out_file1 test_data: text_input1: type: list diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index bb0a662896a..d4e2359c8de 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -40,47 +40,6 @@ from galaxy_test.base.workflow_fixtures import ( from ._framework import ApiTestCase -NESTED_WORKFLOW_AUTO_LABELS_LEGACY_SYNTAX = """ -class: GalaxyWorkflow -inputs: - outer_input: data -outputs: - outer_output: - outputSource: second_cat/out_file1 -steps: - first_cat: - tool_id: cat1 - in: - input1: outer_input - nested_workflow: - run: - class: GalaxyWorkflow - inputs: - - id: inner_input - outputs: - - source: 1#out_file1 - steps: - random: - tool_id: random_lines1 - state: - num_lines: 1 - input: - $link: inner_input - seed_source: - seed_source_selector: set_seed - seed: asdf - in: - inner_input: first_cat/out_file1 - second_cat: - tool_id: cat1 - state: - input1: - $link: nested_workflow#1:out_file1 - queries: - - input2: - $link: nested_workflow#1:out_file1 -""" - NESTED_WORKFLOW_AUTO_LABELS_MODERN_SYNTAX = """ class: GalaxyWorkflow inputs: @@ -487,10 +446,10 @@ steps: - tool_id: cat1 state: input1: - $link: first_cat#out_file1 + $link: first_cat/out_file1 queries: input2: - $link: embed1#output1 + $link: embed1/output1 test_data: input1: "hello world" """) @@ -832,12 +791,12 @@ steps: inner_cond: cond_param_inner: true input1: - $link: 0#out_file1 + $link: 0/out_file1 cat: tool_id: cat1 in: - input1: identifier#output1 - queries_0|input2: identifier#output1 + input1: identifier/output1 + queries_0|input2: identifier/output1 """) with self.dataset_populator.test_history() as history_id: invocation_id = self.__invoke_workflow(history_id, workflow_id) @@ -873,11 +832,11 @@ steps: list_in_list_out: tool_id: collection_creates_list in: - input1: job_props#list_output + input1: job_props/list_output identifier: tool_id: identifier_collection in: - input1: list_in_list_out#list_output + input1: list_in_list_out/list_output """) with self.dataset_populator.test_history() as history_id: invocation_id = self.__invoke_workflow(history_id, workflow_id) @@ -1220,8 +1179,7 @@ test_data: "chrX\t152691446\t152691471\tCCDS14735.1_cds_0_0_chrX_152691447_f\t0\t+\nchrX\t152691446\t152691471\tCCDS14735.1_cds_0_0_chrX_152691447_f\t0\t+\n", content) - for workflow_text in [NESTED_WORKFLOW_AUTO_LABELS_LEGACY_SYNTAX, NESTED_WORKFLOW_AUTO_LABELS_MODERN_SYNTAX]: - run_test(workflow_text) + run_test(NESTED_WORKFLOW_AUTO_LABELS_MODERN_SYNTAX) @skip_without_tool("cat1") @skip_without_tool("collection_paired_test") @@ -1274,7 +1232,7 @@ steps: tool_id: '__FLATTEN__' state: input: - $link: nested#list_output + $link: nested/list_output join_identifier: '-' """, test_data={}, history_id=history_id) details = self.dataset_populator.get_history_collection_details(history_id, hid=14) @@ -1400,7 +1358,7 @@ steps: tool_id: "__FILTER_FAILED_DATASETS__" state: input: - $link: mixed_collection#out_file1 + $link: mixed_collection/out_file1 cat: tool_id: cat1 @@ -1870,7 +1828,7 @@ steps: state: num_lines: 2 input: - $link: empty_list#output + $link: empty_list/output seed_source: seed_source_selector: set_seed seed: asdf @@ -1953,7 +1911,7 @@ steps: state: num_lines: 2 input: - $link: empty_list#output + $link: empty_list/output seed_source: seed_source_selector: set_seed seed: asdf @@ -2388,7 +2346,7 @@ steps: state: floattest: 3.14 inttest: - $link: forty_two#out1 + $link: forty_two/out1 test_data: {} """, history_id=history_id) @@ -2417,7 +2375,7 @@ steps: - label: consume_expression_parameter tool_id: validation_default in: - input1: param_out#text_param + input1: param_out/text_param outputs: out_file1: rename: "replaced_param_collection" @@ -3272,7 +3230,7 @@ steps: create_pair: tool_id: collection_creates_pair in: - input1: first_cat#out_file1 + input1: first_cat/out_file1 outputs: paired_output: remove_tags: @@ -4103,10 +4061,10 @@ steps: - tool_id: cat1 state: input1: - $link: first_cat#out_file1 + $link: first_cat/out_file1 queries: - input2: - $link: embed1#output1 + $link: embed1/output1 test_data: input1: "hello world" """) diff --git a/lib/galaxy_test/api/test_workflows_from_yaml.py b/lib/galaxy_test/api/test_workflows_from_yaml.py index 73e0c101af6..c8dfe57c8b2 100644 --- a/lib/galaxy_test/api/test_workflows_from_yaml.py +++ b/lib/galaxy_test/api/test_workflows_from_yaml.py @@ -296,10 +296,10 @@ steps: - tool_id: cat1 state: input1: - $link: first_cat#out_file1 + $link: first_cat/out_file1 queries: - input2: - $link: embed1#output1 + $link: embed1/output1 test_data: input1: "hello world" """, history_id=history_id) diff --git a/lib/galaxy_test/base/workflow_fixtures.py b/lib/galaxy_test/base/workflow_fixtures.py index 33651b93ace..f51af2b98e7 100644 --- a/lib/galaxy_test/base/workflow_fixtures.py +++ b/lib/galaxy_test/base/workflow_fixtures.py @@ -143,11 +143,11 @@ steps: - tool_id: collection_paired_test state: f1: - $link: 1#paired_output + $link: 1/paired_output - tool_id: cat_list state: input1: - $link: 2#out1 + $link: 2/out1 """ @@ -175,7 +175,7 @@ steps: state: num_lines: 1 input: - $link: apply#output + $link: apply/output seed_source: seed_source_selector: set_seed seed: asdf @@ -279,7 +279,7 @@ steps: inner_input: data outputs: workflow_output: - outputSource: random_lines#out_file1 + outputSource: random_lines/out_file1 steps: - tool_id: random_lines1 label: random_lines diff --git a/lib/tool_shed/test/base/twilltestcase.py b/lib/tool_shed/test/base/twilltestcase.py index 569b70b7cdd..3786eb2dfa3 100644 --- a/lib/tool_shed/test/base/twilltestcase.py +++ b/lib/tool_shed/test/base/twilltestcase.py @@ -8,9 +8,6 @@ import tempfile import time from json import loads -# Be sure to use Galaxy's vanilla pyparsing instead of the older version -# imported by twill. -import pyparsing # noqa: F401 import requests import twill.commands as tc from mercurial import commands, hg, ui diff --git a/run_tests.sh b/run_tests.sh index 7e6bdb798f2..1a1336346a7 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -269,7 +269,7 @@ exists() { type "$1" >/dev/null 2>/dev/null } -DOCKER_DEFAULT_IMAGE='galaxy/testing-base:19.05.0' +DOCKER_DEFAULT_IMAGE='galaxy/testing-base:20.05.00' test_script="./scripts/functional_tests.py" report_file="run_functional_tests.html" @@ -312,7 +312,7 @@ then echo "Docker version:" docker --version echo "Launching docker container for testing with extra args ${DOCKER_RUN_EXTRA_ARGS}..." - name=$(python -c 'import re; import uuid; print re.sub("-", "", str(uuid.uuid4()))') + name=$(python -c 'import re; import uuid; print(re.sub("-", "", str(uuid.uuid4())))') # Create a cache dir for pip, so it has the right owner DOCKER_PIP_CACHE_DIR="$HOME"/.cache/docker_galaxy_pip mkdir -p "$DOCKER_PIP_CACHE_DIR" diff --git a/test/docker/base/Dockerfile b/test/docker/base/Dockerfile index af8c07fc030..758ccad7c9b 100644 --- a/test/docker/base/Dockerfile +++ b/test/docker/base/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 +FROM ubuntu:18.04 LABEL maintainer="John Chilton " ARG CHROME_VERSION="google-chrome-beta" @@ -11,8 +11,6 @@ ENV DEBIAN_FRONTEND=noninteractive \ POSTGRES_MAJOR=9.5 \ GALAXY_ROOT=/galaxy \ GALAXY_VIRTUAL_ENV=/galaxy_venv \ - GALAXY_VIRTUAL_ENV_2=/galaxy_venv \ - GALAXY_VIRTUAL_ENV_3=/galaxy_venv3 \ LC_ALL=C.UTF-8 # Pre-install a bunch of packages to speed up ansible steps. @@ -20,13 +18,10 @@ RUN apt-get update -y && apt-get install -y software-properties-common apt-trans apt-add-repository -y ppa:ansible/ansible && \ curl -s https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \ echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list && \ - curl -s http://neuro.debian.net/lists/xenial.us-ca.full > /etc/apt/sources.list.d/neurodebian.sources.list && \ - apt-key adv --recv-keys --keyserver hkp://pool.sks-keyservers.net:80 0xA5D32F012649A5A9 && \ apt-get update -y && \ apt-get install -y --no-install-recommends postgresql postgresql-client ansible wget \ python3-dev \ git-core \ - python-prettytable python-virtualenv python-pip \ rsync swig sysstat unzip \ openssl \ bzip2 \ @@ -37,9 +32,6 @@ RUN apt-get update -y && apt-get install -y software-properties-common apt-trans locales \ xvfb \ ${CHROME_VERSION:-google-chrome-stable} \ - ffmpeg \ - bcftools \ - singularity-container \ libnss3 libgconf-2-4 && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* @@ -69,24 +61,11 @@ RUN mkdir /etc/galaxy && cd /tmp/ansible && mkdir roles && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* RUN cd $GALAXY_ROOT && \ - virtualenv -p /usr/bin/python3 $GALAXY_VIRTUAL_ENV_3 && \ - for VENV in $GALAXY_VIRTUAL_ENV_2 $GALAXY_VIRTUAL_ENV_3; do \ - GALAXY_VIRTUAL_ENV=$VENV ./scripts/common_startup.sh --dev-wheels && \ - $VENV/bin/pip install psycopg2-binary; done && \ + virtualenv -p /usr/bin/python3 $GALAXY_VIRTUAL_ENV && \ + ./scripts/common_startup.sh --dev-wheels && \ + $GALAXY_VIRTUAL_ENV/bin/pip install psycopg2-binary && \ rm -rf ~/.cache/pip -RUN cd $GALAXY_ROOT && \ - echo "Prepopulating postgres database" && \ - su -c '/usr/lib/postgresql/${POSTGRES_MAJOR}/bin/pg_ctl -o "-F" start -D /opt/galaxy/db' postgres && \ - sleep 3 && \ - GALAXY_CONFIG_DATABASE_CONNECTION="postgresql://root@localhost:5930/galaxy" bash create_db.sh && \ - echo "Prepopulating sqlite database" && \ - GALAXY_CONFIG_DATABASE_CONNECTION="sqlite:////opt/galaxy/galaxy.sqlite" bash create_db.sh && \ - echo "Prepopulating toolshed postgres database" && \ - TOOL_SHED_CONFIG_DATABASE_CONNECTION="postgresql://root@localhost:5930/toolshed" bash create_db.sh tool_shed && \ - echo "Prepopulating toolshed sqlite database" && \ - TOOL_SHED_CONFIG_DATABASE_CONNECTION="sqlite:////opt/galaxy/toolshed.sqlite" bash create_db.sh tool_shed - #======================================== # Add Selenium user with passwordless sudo #======================================== diff --git a/test/docker/base/ansible_vars.yml b/test/docker/base/ansible_vars.yml index d92d87f6f41..027365d086e 100644 --- a/test/docker/base/ansible_vars.yml +++ b/test/docker/base/ansible_vars.yml @@ -17,7 +17,7 @@ galaxy_db_log: /tmp/pSQL.log psql_galaxyftp_password: ignoredthis galaxy_config_file: "{{ galaxy_server_dir }}/config/galaxy.ini" configure_docker: no -postgresql_bin_dir: /usr/lib/postgresql/9.5/bin +postgresql_bin_dir: /usr/lib/postgresql/10/bin galaxy_manage_database: no # avoid some packages unneeded packages install by ansible-galaxy-os install_maintainance_packages: false @@ -25,6 +25,7 @@ galaxy_user_name: "root" galaxy_extras_install_packages: true galaxy_job_conf_path: "/etc/galaxy/job_conf.xml" galaxy_build_client: no +galaxy_virtualenv_python: python3 galaxy_web_processes: 1 galaxy_handler_processes: 2 diff --git a/test/docker/base/run_test_wrapper.sh b/test/docker/base/run_test_wrapper.sh index 46295549ef0..4ac43222424 100644 --- a/test/docker/base/run_test_wrapper.sh +++ b/test/docker/base/run_test_wrapper.sh @@ -4,7 +4,7 @@ set -e if [ ! -z "$USE_SELENIUM" ]; then # Start Selenium. - sudo -H -E -u seluser /opt/bin/entry_point.sh & + sudo -E -H -u seluser /opt/bin/entry_point.sh & fi echo "Deleting galaxy user - it may not exist and this is fine." @@ -16,16 +16,15 @@ echo "Creating galaxy group with gid $GALAXY_TEST_UID - it may already exist and groupadd -r galaxy -g "$GALAXY_TEST_UID" | true echo "Creating galaxy user with uid $GALAXY_TEST_UID - it may already exist and this is fine." useradd -u $GALAXY_TEST_UID -r -g galaxy -d /home/galaxy -c "Galaxy User" galaxy -s /bin/bash | true -mkdir -p /home/galaxy/.singularity echo "Setting galaxy user password - the operation may fail." echo "galaxy:galaxy" | chpasswd | true -chown -R "$GALAXY_TEST_UID:$GALAXY_TEST_UID" "${GALAXY_VIRTUAL_ENV:-/galaxy_venv}" /home/galaxy/ +chown -R "$GALAXY_TEST_UID:$GALAXY_TEST_UID" "${GALAXY_VIRTUAL_ENV:-/galaxy_venv}" : ${GALAXY_TEST_DATABASE_TYPE:-"postgres"} if [ "$GALAXY_TEST_DATABASE_TYPE" = "postgres" ]; then echo "Starting postgres and then sleeping for 3 seconds" - su -c '/usr/lib/postgresql/9.5/bin/pg_ctl -o "-F" start -D /opt/galaxy/db' postgres + su -c '/usr/lib/postgresql/10/bin/pg_ctl -o "-F" start -D /opt/galaxy/db' postgres sleep 3 GALAXY_TEST_INSTALL_DB_MERGED="true" GALAXY_TEST_DBURI="postgresql://root@localhost:5930/galaxy?client_encoding=utf8" @@ -49,15 +48,8 @@ cd /galaxy : ${GALAXY_VIRTUAL_ENV:=.venv} -HOME=/home/galaxy - echo "Running common startup for updated dependencies (if any)" -sudo -E -u "#${GALAXY_TEST_UID}" ./scripts/common_startup.sh --dev-wheels || { echo "common_startup.sh failed"; exit 1; } - -echo "Upgrading test database..." -sudo -E -u "#${GALAXY_TEST_UID}" GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION="$GALAXY_TEST_DBURI" sh manage_db.sh upgrade -echo "Upgrading tool shed database... $TOOL_SHED_CONFIG_OVERRIDE_DATABASE_CONNECTION" -sudo -E -u "#${GALAXY_TEST_UID}" TOOL_SHED_CONFIG_OVERRIDE_DATABASE_CONNECTION="$TOOL_SHED_TEST_DBURI" sh manage_db.sh upgrade tool_shed +sudo -E -H -u "#${GALAXY_TEST_UID}" ./scripts/common_startup.sh --dev-wheels || { echo "common_startup.sh failed"; exit 1; } # Ensure Selenium is running before starting tests. if [ ! -z "$USE_SELENIUM" ]; @@ -78,7 +70,7 @@ fi if [ -z "$GALAXY_NO_TESTS" ]; then - sudo -E -u "#${GALAXY_TEST_UID}" sh run_tests.sh --skip-common-startup $@ + sudo -E -H -u "#${GALAXY_TEST_UID}" sh run_tests.sh --skip-common-startup $@ else GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION="$GALAXY_TEST_DBURI" TOOL_SHED_CONFIG_OVERRIDE_DATABASE_CONNECTION="$TOOL_SHED_TEST_DBURI" diff --git a/test/integration/test_fail_job_tool_unavailable.py b/test/integration/test_fail_job_tool_unavailable.py index 1b53845441a..da638369e25 100644 --- a/test/integration/test_fail_job_tool_unavailable.py +++ b/test/integration/test_fail_job_tool_unavailable.py @@ -27,20 +27,21 @@ class FailJobWhenToolUnavailableTestCase(integration_util.IntegrationTestCase): self.workflow_populator.run_workflow(""" class: GalaxyWorkflow steps: - - label: sleep + sleep: run: class: GalaxyTool command: sleep 20s && echo 'hello world 2' > '$output1' outputs: output1: format: txt - - tool_id: cat1 + cat: + tool_id: cat1 state: input1: - $link: sleep#output1 + $link: sleep/output1 queries: input2: - $link: sleep#output1 + $link: sleep/output1 """, history_id=self.history_id, assert_ok=False, wait=False) # Wait until workflow is fully scheduled, otherwise can't test effect of removing tool from queued job time.sleep(10)