From a3ff1297435721d3f687052382b75a2a2f2f9b25 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 19 Sep 2017 10:55:20 +0200 Subject: [PATCH 01/66] Explicitly write registry.xml when creating a job for the upload tool --- lib/galaxy/jobs/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 11858c09dad..e4ba3e55c0e 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -893,6 +893,11 @@ class JobWrapper( object, HasResourceParameters ): # The tool execution is given a working directory beneath the # "job" working directory. + if job.tool_id == 'upload1': + # We usually write the 'registry.xml' file together with the metadata tool, + # but if `embed_metadata_in_job` is set to false 'registry.xml' will not be available + # to the upload tool, which will then fail. + self.app.datatypes_registry.to_xml_file(os.path.join(self.working_directory, 'registry.xml')) self.tool_working_directory = os.path.join(self.working_directory, "working") safe_makedirs(self.tool_working_directory) log.debug( '(%s) Working directory for job is: %s', From d980dfa162b340423ffd8822ac9dcdf6f1997e80 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 19 Sep 2017 16:02:34 +0200 Subject: [PATCH 02/66] Write registry.xml for all tools that require galaxy's python env This should cover the upload and data_source tools. --- lib/galaxy/jobs/__init__.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index e4ba3e55c0e..9a2d60bb97d 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -866,6 +866,9 @@ class JobWrapper( object, HasResourceParameters ): self.galaxy_lib_dir # Shell fragment to inject dependencies self.dependency_shell_commands = self.tool.build_dependency_shell_commands(job_directory=self.working_directory) + if self.tool.requires_galaxy_python_environment: + # These tools (upload, metadata, data_source) may need access to the datatypes registry. + self.app.datatypes_registry.to_xml_file(os.path.join(self.working_directory, 'registry.xml')) # We need command_line persisted to the db in order for Galaxy to re-queue the job # if the server was stopped and restarted before the job finished job.command_line = unicodify(self.command_line) @@ -893,11 +896,6 @@ class JobWrapper( object, HasResourceParameters ): # The tool execution is given a working directory beneath the # "job" working directory. - if job.tool_id == 'upload1': - # We usually write the 'registry.xml' file together with the metadata tool, - # but if `embed_metadata_in_job` is set to false 'registry.xml' will not be available - # to the upload tool, which will then fail. - self.app.datatypes_registry.to_xml_file(os.path.join(self.working_directory, 'registry.xml')) self.tool_working_directory = os.path.join(self.working_directory, "working") safe_makedirs(self.tool_working_directory) log.debug( '(%s) Working directory for job is: %s', From b7b3ffb1813f31b7811bdbb35602743294356f66 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 19 Sep 2017 20:39:35 +0200 Subject: [PATCH 03/66] Fix unit test failure --- test/unit/jobs/test_job_wrapper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/jobs/test_job_wrapper.py b/test/unit/jobs/test_job_wrapper.py index 544546c9275..b981f0806a3 100644 --- a/test/unit/jobs/test_job_wrapper.py +++ b/test/unit/jobs/test_job_wrapper.py @@ -162,6 +162,7 @@ class MockTool(object): self.version_string_cmd = TEST_VERSION_COMMAND self.tool_dir = "/path/to/tools" self.dependencies = [] + self.requires_galaxy_python_environment = False def build_dependency_shell_commands(self, job_directory): return TEST_DEPENDENCIES_COMMANDS From 7fd6813dd7fcf373cc09a3da6fd6f4bf5c8b0863 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Tue, 19 Sep 2017 14:43:00 -0400 Subject: [PATCH 04/66] pin flake8 deps for release --- tox.ini | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tox.ini b/tox.ini index f9f44eaa08a..9a4cf6490b8 100644 --- a/tox.ini +++ b/tox.ini @@ -6,8 +6,8 @@ skipsdist = True commands = bash .ci/flake8_wrapper.sh whitelist_externals = bash deps = - flake8 - flake8-docstrings>=1.1.0 + flake8==3.4.1 + flake8-docstrings==1.1.0 [testenv:py33-lint] commands = bash .ci/flake8_py3_wrapper.sh @@ -17,7 +17,7 @@ deps = flake8 [testenv:py34-lint] commands = bash .ci/flake8_py3_wrapper.sh whitelist_externals = bash -deps = flake8 +deps = flake8==3.4.1 [testenv:py35-lint] commands = bash .ci/flake8_py3_wrapper.sh @@ -44,15 +44,15 @@ whitelist_externals = bash skip_install = True deps = flake8 - flake8-import-order>=0.9 + flake8-import-order==0.13 [testenv:py27-lint-imports-include-list] commands = bash .ci/flake8_wrapper_imports.sh whitelist_externals = bash skip_install = True deps = - flake8 - flake8-import-order>=0.9 + flake8==3.4.1 + flake8-import-order==0.13 [testenv:qunit] commands = bash run_tests.sh -q @@ -80,7 +80,7 @@ whitelist_externals = bash skip_install = True deps = flake8 - flake8-docstrings>=1.1.0 + flake8-docstrings==1.1.0 [testenv:py27-lint-docstring-include-list] @@ -89,4 +89,4 @@ whitelist_externals = bash skip_install = True deps = flake8 - flake8-docstrings>=1.1.0 + flake8-docstrings==1.1.0 From ee6b8b1af8c997d874a9ad7213231582b69cf82c Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Tue, 19 Sep 2017 20:15:44 -0400 Subject: [PATCH 05/66] pin the last flake8 thanks @nsoranzo :eagle: :eyes: --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 9a4cf6490b8..593185b8a7e 100644 --- a/tox.ini +++ b/tox.ini @@ -88,5 +88,5 @@ commands = bash .ci/flake8_wrapper_docstrings.sh --include whitelist_externals = bash skip_install = True deps = - flake8 + flake8==3.4.1 flake8-docstrings==1.1.0 From 5404f7cda7ca4f3b9a2c423bdd29898fb7d8c911 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 20 Sep 2017 11:11:56 -0400 Subject: [PATCH 06/66] Small (temporary, mako is going away) layer to support modern routes in old display_base pages (workflow, history, etc). --- templates/display_base.mako | 12 ++++++------ templates/display_common.mako | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/templates/display_base.mako b/templates/display_base.mako index f0780e2f31c..9015da34e3a 100644 --- a/templates/display_base.mako +++ b/templates/display_base.mako @@ -117,10 +117,10 @@ ## Get URL to other published items owned by user that owns this item. <% ##TODO: is there a better way to create this URL? Can't use 'f-username' as a key b/c it's not a valid identifier. - controller_name = get_controller_name( item ) + modern_route = modern_route_for_controller(get_controller_name(item)) item_plural = get_item_plural( item ) - href_to_all_items = h.url_for( controller='/' + controller_name, action='list_published') - href_to_user_items = h.url_for( controller='/' + controller_name, action='list_published', xxx=item.user.username) + href_to_all_items = h.url_for( controller='/' + modern_route, action='list_published') + href_to_user_items = h.url_for( controller='/' + modern_route, action='list_published', xxx=item.user.username) href_to_user_items = href_to_user_items.replace( 'xxx', 'f-username') %> @@ -162,10 +162,10 @@ <% ## FIXME: duplicated from above for now - controller_name = get_controller_name( item ) + modern_route = modern_route_for_controller(get_controller_name(item)) item_plural = get_item_plural( item ) - href_to_all_items = h.url_for( controller='/' + controller_name, action='list_published') - href_to_user_items = h.url_for( controller='/' + controller_name, action='list_published', xxx=item.user.username) + href_to_all_items = h.url_for( controller='/' + modern_route , action='list_published') + href_to_user_items = h.url_for( controller='/' + modern_route, action='list_published', xxx=item.user.username) href_to_user_items = href_to_user_items.replace( 'xxx', 'f-username') %> diff --git a/templates/display_common.mako b/templates/display_common.mako index d78633df939..2fa1f3e49b9 100644 --- a/templates/display_common.mako +++ b/templates/display_common.mako @@ -111,6 +111,21 @@ %> +<%def name="modern_route_for_controller( controller )"> + <% + if controller == "history": + return "histories" + elif controller == "workflow": + return "workflows" + elif controller == "dataset": + return "datasets" + elif controller == "page": + return "pages" + else: + return controller + %> + + ## Returns item user/owner. <%def name="get_item_user( item )"> <% From 69a87003f70e3925554fcc88fd2080edab619ed1 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 20 Sep 2017 20:26:21 -0400 Subject: [PATCH 07/66] Fix modules resolver (addition of the resolver_msg property on the base class references name and version attributes that were not defined in the ModuleDependecy). --- lib/galaxy/tools/deps/resolvers/modules.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/galaxy/tools/deps/resolvers/modules.py b/lib/galaxy/tools/deps/resolvers/modules.py index 53b5107eb3b..284f82d559d 100644 --- a/lib/galaxy/tools/deps/resolvers/modules.py +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -161,6 +161,14 @@ class ModuleDependency(Dependency): self.module_version = module_version self._exact = exact + @property + def name(self): + return self.module_name + + @property + def version(self): + return self.module_version + @property def exact(self): return self._exact From 2495ceae267da873813d2dc1aae145b686d18672 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 21 Sep 2017 14:02:53 -0400 Subject: [PATCH 08/66] Remove overly chatty debug statement in workflow output cleanup -- this happens pretty often and much of the time is not a problem at all. If we have to track down any new issues with workflow output cleanup, it's easy enough to temporarily add this logging. --- lib/galaxy/jobs/actions/post.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy/jobs/actions/post.py b/lib/galaxy/jobs/actions/post.py index 2f7402f399c..545328bbeb1 100644 --- a/lib/galaxy/jobs/actions/post.py +++ b/lib/galaxy/jobs/actions/post.py @@ -308,7 +308,6 @@ class DeleteIntermediatesAction(DefaultJobAction): safe_to_delete = True for job_to_check in [d_j.job for d_j in input_dataset.dependent_jobs]: if job_to_check != job and job_to_check.state not in [job.states.OK, job.states.DELETED]: - log.debug("Workflow Intermediates cleanup attempted, but non-terminal state '%s' detected for job %s" % (job_to_check.state, job_to_check.id)) safe_to_delete = False if safe_to_delete: # Support purging here too. From 4ee18019e0750a363ba34ae5acece4faa8aec850 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 21 Sep 2017 15:54:44 -0400 Subject: [PATCH 09/66] Fix Selenium tests outright broken in #4669. Way to go @jmchilton. --- test/selenium_tests/test_saved_histories.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/selenium_tests/test_saved_histories.py b/test/selenium_tests/test_saved_histories.py index 4272ec46b93..422b7c8d713 100644 --- a/test/selenium_tests/test_saved_histories.py +++ b/test/selenium_tests/test_saved_histories.py @@ -170,7 +170,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): search_input.send_keys(self.history4_name) self.send_enter(search_input) - self.assert_grid_histories_are(['No Items']) + self.assert_grid_histories_are([]) @selenium_test def test_advanced_search(self): @@ -187,7 +187,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self.unset_filter('name', self.history2_name) self.set_filter(name_filter_selector, self.history4_name) - self.assert_grid_histories_are(['No Items']) + self.assert_grid_histories_are([]) self.unset_filter('name', self.history4_name) # Search by tags @@ -196,7 +196,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self.unset_filter('tags', self.history3_tags[0]) self.set_filter(tags_filter_selector, self.history4_tags[0]) - self.assert_grid_histories_are(['No Items']) + self.assert_grid_histories_are([]) self.unset_filter('tags', self.history4_tags[0]) @selenium_test From 244fc834cd4dd3980cc3edb755d22ca5a9372421 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 11 Sep 2017 12:09:04 -0400 Subject: [PATCH 10/66] Explicitly include PyPI as an --extra-index-url to pip so we can install from PyPI even when a package exists (but the correct version or architecture does not exist) on wheels.galaxyproject.org. Previously, PyPI was only checked after wheels.galaxyproject.org because pypiserver performs an automatic redirect to PyPI for any packages that do not exist in its index. --- scripts/common_startup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index 679a88c7a2c..a1e64df09b8 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -139,9 +139,9 @@ if [ $REPLACE_PIP -eq 1 ]; then fi if [ $FETCH_WHEELS -eq 1 ]; then - pip install -r requirements.txt --index-url "${GALAXY_WHEELS_INDEX_URL}" + pip install -r requirements.txt --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url https://pypi.python.org/pypi GALAXY_CONDITIONAL_DEPENDENCIES=$(PYTHONPATH=lib python -c "import galaxy.dependencies; print '\n'.join(galaxy.dependencies.optional('$GALAXY_CONFIG_FILE'))") - [ -z "$GALAXY_CONDITIONAL_DEPENDENCIES" ] || echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | pip install -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}" + [ -z "$GALAXY_CONDITIONAL_DEPENDENCIES" ] || echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | pip install -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url https://pypi.python.org/pypi fi if [ $FETCH_WHEELS -eq 1 -a $DEV_WHEELS -eq 1 ]; then From 74ad0acb52031355c7efe28b00d1d9b0205174f2 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Thu, 14 Sep 2017 23:47:31 -0400 Subject: [PATCH 11/66] Remove repetition in Galaxy env setup in the job script and fix issues with remote metadata on Pulsar. --- .../dependencies/pinned-requirements.txt | 2 +- lib/galaxy/jobs/command_factory.py | 14 +------- lib/galaxy/jobs/runners/pulsar.py | 32 ++++++++++++----- .../job_script/DEFAULT_JOB_FILE_TEMPLATE.sh | 35 +++++++++++-------- 4 files changed, 46 insertions(+), 37 deletions(-) diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 4c4f7601c3b..1bd08e183cb 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -57,7 +57,7 @@ anyjson==0.3.3 # Pulsar requirements psutil==4.1.0 -pulsar-galaxy-lib==0.7.0.dev5 +pulsar-galaxy-lib==0.7.5 # sqlalchemy-migrate and dependencies sqlalchemy-migrate==0.10.0 diff --git a/lib/galaxy/jobs/command_factory.py b/lib/galaxy/jobs/command_factory.py index abf1a0e84de..fb2db1201ac 100644 --- a/lib/galaxy/jobs/command_factory.py +++ b/lib/galaxy/jobs/command_factory.py @@ -17,19 +17,7 @@ log = getLogger(__name__) CAPTURE_RETURN_CODE = "return_code=$?" YIELD_CAPTURED_CODE = 'sh -c "exit $return_code"' SETUP_GALAXY_FOR_METADATA = """ -if [ "$GALAXY_LIB" != "None" ]; then - if [ -n "$PYTHONPATH" ]; then - PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" - else - PYTHONPATH="$GALAXY_LIB" - fi - export PYTHONPATH -fi -if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" \ - -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then - . "$GALAXY_VIRTUAL_ENV/bin/activate" -fi -GALAXY_PYTHON=`command -v python` +_galaxy_setup_environment True """ diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 9722ea32736..956d8eb88da 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -51,7 +51,10 @@ __all__ = ( 'PulsarEmbeddedJobRunner', ) -MINIMUM_PULSAR_VERSION = LooseVersion("0.7.0.dev3") +MINIMUM_PULSAR_VERSIONS = { + '_default_': LooseVersion("0.7.0.dev3"), + 'remote_metadata': LooseVersion("0.7.5"), +} NO_REMOTE_GALAXY_FOR_METADATA_MESSAGE = "Pulsar misconfiguration - Pulsar client configured to set metadata remotely, but remote Pulsar isn't properly configured with a galaxy_home directory." NO_REMOTE_DATATYPES_CONFIG = "Pulsar client is configured to use remote datatypes configuration when setting metadata externally, but Pulsar is not configured with this information. Defaulting to datatypes_conf.xml." @@ -59,7 +62,7 @@ GENERIC_REMOTE_ERROR = "Failed to communicate with remote job server." FAILED_REMOTE_ERROR = "Remote job server indicated a problem running or monitoring this job." LOST_REMOTE_ERROR = "Remote job server could not determine this job's state." -UPGRADE_PULSAR_ERROR = "Galaxy is misconfigured, please contact administrator. The target Pulsar server is unsupported, this version of Galaxy requires Pulsar version %s or newer." % MINIMUM_PULSAR_VERSION +UPGRADE_PULSAR_ERROR = "Galaxy is misconfigured, please contact administrator. The target Pulsar server is unsupported, this version of Galaxy requires Pulsar version %s or newer." # Is there a good way to infer some default for this? Can only use # url_for from web threads. https://gist.github.com/jmchilton/9098762 @@ -310,6 +313,11 @@ class PulsarJobRunner(AsynchronousJobRunner): pulsar_job_state.job_destination = job_destination self.monitor_job(pulsar_job_state) + def __needed_features(self, client): + return { + 'remote_metadata': PulsarJobRunner.__remote_metadata(client), + } + def __prepare_job(self, job_wrapper, job_destination): """Build command-line and Pulsar client for this job.""" command_line = None @@ -319,8 +327,9 @@ class PulsarJobRunner(AsynchronousJobRunner): try: client = self.get_client_from_wrapper(job_wrapper) tool = job_wrapper.tool - remote_job_config = client.setup(tool.id, tool.version) - PulsarJobRunner.check_job_config(remote_job_config) + remote_job_config = client.setup(tool.id, tool.version, tool.requires_galaxy_python_environment) + needed_features = self.__needed_features(client) + PulsarJobRunner.check_job_config(remote_job_config, check_features=needed_features) rewrite_parameters = PulsarJobRunner.__rewrite_parameters(client) prepare_kwds = {} if rewrite_parameters: @@ -593,12 +602,17 @@ class PulsarJobRunner(AsynchronousJobRunner): return client_outputs @staticmethod - def check_job_config(remote_job_config): + def check_job_config(remote_job_config, check_features=None): + check_features = check_features or {} # 0.6.0 was newest Pulsar version that did not report it's version. pulsar_version = LooseVersion(remote_job_config.get('pulsar_version', "0.6.0")) + needed_version = LooseVersion("0.0.0") log.info("pulsar_version is %s" % pulsar_version) - if pulsar_version < MINIMUM_PULSAR_VERSION: - raise UnsupportedPulsarException() + for feature in list(check_features.keys()) + ['_default_']: + if pulsar_version < MINIMUM_PULSAR_VERSIONS[feature]: + needed_version = max(needed_version, MINIMUM_PULSAR_VERSIONS[feature]) + if pulsar_version < needed_version: + raise UnsupportedPulsarException(needed_version) @staticmethod def __dependencies_description(pulsar_client, job_wrapper): @@ -854,5 +868,5 @@ class PulsarComputeEnvironment(ComputeEnvironment): class UnsupportedPulsarException(Exception): - def __init__(self): - super(UnsupportedPulsarException, self).__init__(UPGRADE_PULSAR_ERROR) + def __init__(self, needed): + super(UnsupportedPulsarException, self).__init__(UPGRADE_PULSAR_ERROR % needed) diff --git a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh index 801df6acbc1..2a83d133b89 100644 --- a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh +++ b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh @@ -1,25 +1,32 @@ #!$shell +_galaxy_setup_environment() { + local _use_framework_galaxy="$1" + if [ "$GALAXY_LIB" != "None" -a "$_use_framework_galaxy" = "True" ]; then + if [ -n "$PYTHONPATH" ]; then + PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" + else + PYTHONPATH="$GALAXY_LIB" + fi + export PYTHONPATH + fi + $env_setup_commands + if [ "$GALAXY_VIRTUAL_ENV" != "None" -a "$_use_framework_galaxy" = "True" \ + -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" \ + -a "`command -v python`" != "$GALAXY_VIRTUAL_ENV/bin/python" ]; then + . "$GALAXY_VIRTUAL_ENV/bin/activate" + fi +} + $headers $integrity_injection $slots_statement export GALAXY_SLOTS +GALAXY_VIRTUAL_ENV="$galaxy_virtual_env" PRESERVE_GALAXY_ENVIRONMENT="$preserve_python_environment" GALAXY_LIB="$galaxy_lib" -if [ "$GALAXY_LIB" != "None" -a "$PRESERVE_GALAXY_ENVIRONMENT" = "True" ]; then - if [ -n "$PYTHONPATH" ]; then - PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" - else - PYTHONPATH="$GALAXY_LIB" - fi - export PYTHONPATH -fi -$env_setup_commands -GALAXY_VIRTUAL_ENV="$galaxy_virtual_env" -if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" \ - -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" -a "$PRESERVE_GALAXY_ENVIRONMENT" = "True" ]; then - . "$GALAXY_VIRTUAL_ENV/bin/activate" -fi +_galaxy_setup_environment "$PRESERVE_GALAXY_ENVIRONMENT" +GALAXY_PYTHON=`command -v python` $instrument_pre_commands cd $working_directory $command From 276af4db2ab5e81f4af8f35b7d09f48d9fed8dcd Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Fri, 15 Sep 2017 16:58:08 -0400 Subject: [PATCH 12/66] Job script enhancements based on feedback. --- lib/galaxy/jobs/command_factory.py | 2 +- .../runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/jobs/command_factory.py b/lib/galaxy/jobs/command_factory.py index fb2db1201ac..f8426aeefe6 100644 --- a/lib/galaxy/jobs/command_factory.py +++ b/lib/galaxy/jobs/command_factory.py @@ -17,7 +17,7 @@ log = getLogger(__name__) CAPTURE_RETURN_CODE = "return_code=$?" YIELD_CAPTURED_CODE = 'sh -c "exit $return_code"' SETUP_GALAXY_FOR_METADATA = """ -_galaxy_setup_environment True +[ "$GALAXY_VIRTUAL_ENV" = "None" ] && GALAXY_VIRTUAL_ENV="$_GALAXY_VIRTUAL_ENV"; _galaxy_setup_environment True """ diff --git a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh index 2a83d133b89..68da1b3e362 100644 --- a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh +++ b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh @@ -1,5 +1,7 @@ #!$shell +$headers + _galaxy_setup_environment() { local _use_framework_galaxy="$1" if [ "$GALAXY_LIB" != "None" -a "$_use_framework_galaxy" = "True" ]; then @@ -11,18 +13,17 @@ _galaxy_setup_environment() { export PYTHONPATH fi $env_setup_commands - if [ "$GALAXY_VIRTUAL_ENV" != "None" -a "$_use_framework_galaxy" = "True" \ - -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" \ + if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" \ -a "`command -v python`" != "$GALAXY_VIRTUAL_ENV/bin/python" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate" fi } -$headers $integrity_injection $slots_statement export GALAXY_SLOTS GALAXY_VIRTUAL_ENV="$galaxy_virtual_env" +_GALAXY_VIRTUAL_ENV="$galaxy_virtual_env" PRESERVE_GALAXY_ENVIRONMENT="$preserve_python_environment" GALAXY_LIB="$galaxy_lib" _galaxy_setup_environment "$PRESERVE_GALAXY_ENVIRONMENT" From 53095d688be81a36c31b9721caebf58b383cfe72 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 20 Sep 2017 19:33:02 -0400 Subject: [PATCH 13/66] Fix remote datatypes_conf.xml and registry.xml param rewriting with Pulsar --- lib/galaxy/jobs/__init__.py | 10 ++++++---- lib/galaxy/jobs/command_factory.py | 2 ++ lib/galaxy/jobs/runners/pulsar.py | 2 +- lib/galaxy/model/metadata.py | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 1e21b45c9db..f6bd6093947 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -1648,7 +1648,8 @@ class JobWrapper(object, HasResourceParameters): def setup_external_metadata(self, exec_dir=None, tmp_dir=None, dataset_files_path=None, config_root=None, - config_file=None, resolve_metadata_dependencies=False, + config_file=None, datatypes_config=None, + resolve_metadata_dependencies=False, set_extension=True, **kwds): # extension could still be 'auto' if this is the upload tool. job = self.get_job() @@ -1667,8 +1668,9 @@ class JobWrapper(object, HasResourceParameters): config_root = self.app.config.root if config_file is None: config_file = self.app.config.config_file - datatypes_config = os.path.join(self.working_directory, 'registry.xml') - self.app.datatypes_registry.to_xml_file(path=datatypes_config) + if datatypes_config is None: + datatypes_config = os.path.join(self.working_directory, 'registry.xml') + self.app.datatypes_registry.to_xml_file(path=datatypes_config) command = self.external_output_metadata.setup_external_metadata([output_dataset_assoc.dataset for output_dataset_assoc in job.output_datasets + job.output_library_datasets], @@ -1977,7 +1979,7 @@ class TaskWrapper(JobWrapper): pass def setup_external_metadata(self, exec_dir=None, tmp_dir=None, dataset_files_path=None, - config_root=None, config_file=None, + config_root=None, config_file=None, datatypes_config=None, set_extension=True, **kwds): # There is no metadata setting for tasks. This is handled after the merge, at the job level. return "" diff --git a/lib/galaxy/jobs/command_factory.py b/lib/galaxy/jobs/command_factory.py index f8426aeefe6..7922531ee8b 100644 --- a/lib/galaxy/jobs/command_factory.py +++ b/lib/galaxy/jobs/command_factory.py @@ -186,6 +186,7 @@ def __handle_metadata(commands_builder, job_wrapper, runner, remote_command_para output_fnames = metadata_kwds.get('output_fnames', job_wrapper.get_output_fnames()) config_root = metadata_kwds.get('config_root', None) config_file = metadata_kwds.get('config_file', None) + datatypes_config = metadata_kwds.get('datatypes_config', None) compute_tmp_dir = metadata_kwds.get('compute_tmp_dir', None) resolve_metadata_dependencies = job_wrapper.commands_in_new_shell metadata_command = job_wrapper.setup_external_metadata( @@ -196,6 +197,7 @@ def __handle_metadata(commands_builder, job_wrapper, runner, remote_command_para set_extension=False, config_root=config_root, config_file=config_file, + datatypes_config=datatypes_config, compute_tmp_dir=compute_tmp_dir, resolve_metadata_dependencies=resolve_metadata_dependencies, kwds={'overwrite': False} diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 956d8eb88da..aabb38b1d5e 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -701,7 +701,7 @@ class PulsarJobRunner(AsynchronousJobRunner): remote_datatypes_config = os.path.join(remote_galaxy_home, 'datatypes_conf.xml') metadata_kwds['datatypes_config'] = remote_datatypes_config else: - datatypes_config = os.path.join(configs_directory, 'registry.xml') + datatypes_config = os.path.join(job_wrapper.working_directory, 'registry.xml') self.app.datatypes_registry.to_xml_file(path=datatypes_config) # Ensure this file gets pushed out to the remote config dir. job_wrapper.extra_filenames.append(datatypes_config) diff --git a/lib/galaxy/model/metadata.py b/lib/galaxy/model/metadata.py index a1dd075ffd8..fa6e9b3537a 100644 --- a/lib/galaxy/model/metadata.py +++ b/lib/galaxy/model/metadata.py @@ -836,7 +836,7 @@ class JobExternalOutputMetadataWrapper(object): sa_session.add(metadata_files) sa_session.flush() metadata_files_list.append(metadata_files) - args = '"%s" "%s" %s %s' % (datatypes_config, + args = '"%s" "%s" %s %s' % (metadata_path_on_compute(datatypes_config), job_metadata, " ".join(map(__metadata_files_list_to_cmd_line, metadata_files_list)), max_metadata_value_size) From db5185cca0bb9b5f59d8fcf218eb4c5d81cb5911 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Thu, 21 Sep 2017 16:12:46 -0400 Subject: [PATCH 14/66] Pulsar release was 0.8.0 not 0.7.5 --- lib/galaxy/dependencies/pinned-requirements.txt | 2 +- lib/galaxy/jobs/runners/pulsar.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 1bd08e183cb..80b2bcffa29 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -57,7 +57,7 @@ anyjson==0.3.3 # Pulsar requirements psutil==4.1.0 -pulsar-galaxy-lib==0.7.5 +pulsar-galaxy-lib==0.8.0 # sqlalchemy-migrate and dependencies sqlalchemy-migrate==0.10.0 diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index aabb38b1d5e..459e41fd1fc 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -53,7 +53,7 @@ __all__ = ( MINIMUM_PULSAR_VERSIONS = { '_default_': LooseVersion("0.7.0.dev3"), - 'remote_metadata': LooseVersion("0.7.5"), + 'remote_metadata': LooseVersion("0.8.0"), } NO_REMOTE_GALAXY_FOR_METADATA_MESSAGE = "Pulsar misconfiguration - Pulsar client configured to set metadata remotely, but remote Pulsar isn't properly configured with a galaxy_home directory." From 2669ab022ed2f90faac54c510014f87783efeaa0 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 20 Sep 2017 14:52:24 -0400 Subject: [PATCH 15/66] Pass job output file unqualified names to Pulsar so that it can create them before running the job. --- lib/galaxy/jobs/__init__.py | 10 ++++++++++ lib/galaxy/jobs/runners/pulsar.py | 7 +++++++ 2 files changed, 17 insertions(+) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 1e21b45c9db..8d5f263d9d6 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -1571,6 +1571,9 @@ class JobWrapper(object, HasResourceParameters): paths.append(DatasetPath(da.id, real_path=real_path, false_path=false_path, mutable=False)) return paths + def get_output_basenames(self): + return map(os.path.basename, map(str, self.get_output_fnames())) + def get_output_fnames(self): if self.output_paths is None: self.compute_outputs() @@ -1997,6 +2000,10 @@ class ComputeEnvironment(object): compute server. """ + @abstractmethod + def output_names(self): + """ Output unqualified filenames defined by job. """ + @abstractmethod def output_paths(self): """ Output DatasetPaths defined by job. """ @@ -2062,6 +2069,9 @@ class SharedComputeEnvironment(SimpleComputeEnvironment): self.job_wrapper = job_wrapper self.job = job + def output_names(self): + return self.job_wrapper.get_output_basenames() + def output_paths(self): return self.job_wrapper.get_output_fnames() diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 9722ea32736..f1b34681e03 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -277,8 +277,10 @@ class PulsarJobRunner(AsynchronousJobRunner): dependencies_description = PulsarJobRunner.__dependencies_description(client, job_wrapper) rewrite_paths = not PulsarJobRunner.__rewrite_parameters(client) unstructured_path_rewrites = {} + output_names = [] if compute_environment: unstructured_path_rewrites = compute_environment.unstructured_path_rewrites + output_names = compute_environment.output_names() client_job_description = ClientJobDescription( command_line=command_line, @@ -292,6 +294,7 @@ class PulsarJobRunner(AsynchronousJobRunner): env=client.env, rewrite_paths=rewrite_paths, arbitrary_files=unstructured_path_rewrites, + touch_outputs=output_names, ) job_id = pulsar_submit_job(client, client_job_description, remote_job_config) log.info("Pulsar job submitted with job_id %s" % job_id) @@ -783,6 +786,10 @@ class PulsarComputeEnvironment(ComputeEnvironment): version_path = new_version_path self._version_path = version_path + def output_names(self): + # Maybe this should use the path mapper, but the path mapper just uses basenames + return self.job_wrapper.get_output_basenames() + def output_paths(self): local_output_paths = self._wrapper_output_paths From 3344c77a188cd92ffb2e7150a4f4a2f4f7facedb Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 20 Sep 2017 15:56:37 -0400 Subject: [PATCH 16/66] Bump pulsar lib version to 0.8.0 for output touch support. --- lib/galaxy/dependencies/pinned-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 4c4f7601c3b..80b2bcffa29 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -57,7 +57,7 @@ anyjson==0.3.3 # Pulsar requirements psutil==4.1.0 -pulsar-galaxy-lib==0.7.0.dev5 +pulsar-galaxy-lib==0.8.0 # sqlalchemy-migrate and dependencies sqlalchemy-migrate==0.10.0 From c7699cd3a0e16b9522af334cf2ea1a107fbe5f57 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Thu, 21 Sep 2017 18:23:31 -0400 Subject: [PATCH 17/66] Fix lint error --- lib/galaxy/jobs/runners/pulsar.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 459e41fd1fc..72efe1f0e1f 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -670,7 +670,6 @@ class PulsarJobRunner(AsynchronousJobRunner): raise Exception(NO_REMOTE_GALAXY_FOR_METADATA_MESSAGE) metadata_kwds['exec_dir'] = remote_galaxy_home outputs_directory = remote_job_config['outputs_directory'] - configs_directory = remote_job_config['configs_directory'] working_directory = remote_job_config['working_directory'] metadata_directory = remote_job_config['metadata_directory'] # For metadata calculation, we need to build a list of of output From 31e2721aabf1eaec6f91c71e4f78ee7543da8f7f Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 22 Sep 2017 07:16:52 -0400 Subject: [PATCH 18/66] Selenium - handle potentially stale elements when fetching tags. https://jenkins.galaxyproject.org/job/selenium/529/artifact/529-test-errors/test_tagging2017092204301506069010/stacktrace.txt --- test/galaxy_selenium/navigates_galaxy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/galaxy_selenium/navigates_galaxy.py b/test/galaxy_selenium/navigates_galaxy.py index 7bdcc96d987..5a87c5732c8 100644 --- a/test/galaxy_selenium/navigates_galaxy.py +++ b/test/galaxy_selenium/navigates_galaxy.py @@ -516,6 +516,7 @@ class NavigatesGalaxy(HasDriver): tag_display = workflow_row_element.find_element_by_css_selector(".tags-display") tag_display.click() + @retry_during_transitions def workflow_index_tags(self, workflow_index=0): workflow_row_element = self.workflow_index_table_row(workflow_index) tag_display = workflow_row_element.find_element_by_css_selector(".tags-display") From 9e6e340adc99884a88dc50a2d6d35cdbd3f74fc6 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 22 Sep 2017 07:18:02 -0400 Subject: [PATCH 19/66] Selenium - handle potentially stale search box on workflow index page. https://jenkins.galaxyproject.org/job/selenium/530/artifact/530-test-errors/test_index_search2017092205301506072611/stacktrace.txt --- .../test_workflow_management.py | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/test/selenium_tests/test_workflow_management.py b/test/selenium_tests/test_workflow_management.py index 776c4ed9ee7..3d76b0f1c9c 100644 --- a/test/selenium_tests/test_workflow_management.py +++ b/test/selenium_tests/test_workflow_management.py @@ -1,5 +1,6 @@ from .framework import ( retry_assertion_during_transitions, + retry_during_transitions, selenium_test, SeleniumTestCase, ) @@ -73,20 +74,13 @@ class WorkflowManagementTestCase(SeleniumTestCase): self.workflow_index_rename("searchforthis") self._assert_showing_n_workflows(1) - search_box = self.workflow_index_click_search() - search_box.send_keys("doesnotmatch") + self._click_and_search("doesnotmatch") self._assert_showing_n_workflows(0) - # Prevent stale element textbox by re-fetching, seems to be - # needed but I don't understand why exactly. -John - search_box = self.workflow_index_click_search() - search_box.clear() - self.send_enter(search_box) + self._click_and_search() self._assert_showing_n_workflows(1) - search_box = self.workflow_index_click_search() - search_box.send_keys("searchforthis") - self.send_enter(search_box) + self._click_and_search("searchforthis") self._assert_showing_n_workflows(1) @selenium_test @@ -109,6 +103,16 @@ class WorkflowManagementTestCase(SeleniumTestCase): self.workflow_index_open() assert_published_column_text_is("Yes") + @retry_during_transitions + def _click_and_search(self, search_term=None): + # Allow default search_term of None to just clear search + search_box = self.workflow_index_click_search() + search_box.clear() + if search_term is not None: + search_box.send_keys(search_term) + self.send_enter(search_box) + return search_box + @retry_assertion_during_transitions def _assert_showing_n_workflows(self, n): self.assertEqual(len(self.workflow_index_table_elements()), n) From 3ffdfc67e75a608c05b35086233d40d7c09440cd Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 22 Sep 2017 07:58:18 -0400 Subject: [PATCH 20/66] Selenium - catch popups fading out as transitions to retry various actions on. Previously we only detected transitions for retrying actions based on stale element exceptions, this expands that to include popups that may be fading out for instance. I think this is what is happening with the transiently failing test here https://jenkins.galaxyproject.org/job/selenium/528/artifact/528-test-errors/test_save_as2017092203291506065342/stacktrace.txt. The exception indicates that a click was not clickable because a modal element that was fading out - though it had been previously clickable. I think this should fix that. --- test/galaxy_selenium/has_driver.py | 12 ++++++++++ test/galaxy_selenium/navigates_galaxy.py | 28 +++++++++++++++++++++--- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/test/galaxy_selenium/has_driver.py b/test/galaxy_selenium/has_driver.py index 021de2eac0a..4dca4fbe5d3 100644 --- a/test/galaxy_selenium/has_driver.py +++ b/test/galaxy_selenium/has_driver.py @@ -140,5 +140,17 @@ class HasDriver: ) +def execption_indicates_not_clickable(exception): + return "not clickable" in str(exception) + + def exception_indicates_stale_element(exception): return "stale" in str(exception) + + +__all__ = ( + "execption_indicates_not_clickable", + "exception_indicates_stale_element", + "HasDriver", + "TimeoutException", +) diff --git a/test/galaxy_selenium/navigates_galaxy.py b/test/galaxy_selenium/navigates_galaxy.py index 5a87c5732c8..ed3773cb87c 100644 --- a/test/galaxy_selenium/navigates_galaxy.py +++ b/test/galaxy_selenium/navigates_galaxy.py @@ -15,7 +15,12 @@ import requests import yaml from .data import NAVIGATION_DATA -from .has_driver import exception_indicates_stale_element, HasDriver, TimeoutException +from .has_driver import ( + execption_indicates_not_clickable, + exception_indicates_stale_element, + HasDriver, + TimeoutException, +) from . import sizzle # Test case data @@ -28,7 +33,24 @@ class NullTourCallback(object): pass -def retry_call_during_transitions(f, attempts=5, sleep=.1, exception_check=exception_indicates_stale_element): +def excepion_seems_to_indicate_transition(e): + """True if exception seems to indicate the page state is transitioning. + + Galaxy features many different transition effects that change the page state over time. + These transitions make it slightly more difficult to test Galaxy because atomic input + actions take an indeterminate amount of time to be reflected on the screen. This method + takes a Selenium assertion and tries to infer if such a transition could be the root + cause of the exception. The methods that follow use it to allow retrying actions during + transitions. + + Currently the two kinds of exceptions that we say may indicate a transition are + StaleElement exceptions (a DOM element grabbed at one step is no longer available) + and "not clickable" exceptions (so perhaps a popup modal is blocking a click). + """ + return exception_indicates_stale_element(e) or execption_indicates_not_clickable(e) + + +def retry_call_during_transitions(f, attempts=5, sleep=.1, exception_check=excepion_seems_to_indicate_transition): previous_attempts = 0 while True: try: @@ -44,7 +66,7 @@ def retry_call_during_transitions(f, attempts=5, sleep=.1, exception_check=excep previous_attempts += 1 -def retry_during_transitions(f, attempts=5, sleep=.1, exception_check=exception_indicates_stale_element): +def retry_during_transitions(f, attempts=5, sleep=.1, exception_check=excepion_seems_to_indicate_transition): @wraps(f) def _retry(*args, **kwds): From 77b61ba82d3eec55c5acd2ae4ad3ec9fde0cc9db Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 22 Sep 2017 08:16:53 -0400 Subject: [PATCH 21/66] Selenium - attempted fix for transiently failing history options test. Comments inline about this - but the target failure I'm hoping this fixes is: https://jenkins.galaxyproject.org/job/selenium/528/artifact/528-test-errors/test_options2017092203081506064101/stacktrace.txt --- test/galaxy_selenium/navigates_galaxy.py | 12 ++++++++++++ test/selenium_tests/test_history_options.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/test/galaxy_selenium/navigates_galaxy.py b/test/galaxy_selenium/navigates_galaxy.py index ed3773cb87c..e500422446a 100644 --- a/test/galaxy_selenium/navigates_galaxy.py +++ b/test/galaxy_selenium/navigates_galaxy.py @@ -819,6 +819,18 @@ class NavigatesGalaxy(HasDriver): self.click_center() return text + @retry_during_transitions + def assert_selector_absent_or_hidden_after_transitions(self, selector): + """Variant of assert_selector_absent_or_hidden that retries during transitions. + + In the parent method - the element is found and then it is checked to see + if it is visible. It may disappear from the page in the middle there + and cause a StaleElement error. For checks where we care about the final + resting state after transitions - this method can be used to retry + during those transitions. + """ + return self.assert_selector_absent_or_hidden(selector) + def assert_tooltip_text(self, element, expected, sleep=0, click_away=True): text = self.get_tooltip_text(element, sleep=sleep, click_away=click_away) assert text == expected, "Tooltip text [%s] was not expected text [%s]." % (text, expected) diff --git a/test/selenium_tests/test_history_options.py b/test/selenium_tests/test_history_options.py index 8e8f975801c..f701f65d357 100644 --- a/test/selenium_tests/test_history_options.py +++ b/test/selenium_tests/test_history_options.py @@ -27,4 +27,4 @@ class HistoryOptionsTestCase(SeleniumTestCase): self.click_hda_title(hda_id, wait=True) - self.assert_selector_absent_or_hidden(hda_body_selector) + self.assert_selector_absent_or_hidden_after_transitions(hda_body_selector) From 17348a62fe92716ed80cac6f0925e5d5c3203deb Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 22 Sep 2017 08:41:25 -0400 Subject: [PATCH 22/66] Selenium - more robust inline searching for published histories grid. Refactor recent search improvement for workflow management into a method that can be shared with published histories grid. Should address the transient failure test here: https://jenkins.galaxyproject.org/job/selenium/521/artifact/521-test-errors/test_history_grid_search_standard2017092120151506039303/stacktrace.txt --- test/galaxy_selenium/navigates_galaxy.py | 21 +++++++++++++++++++ .../test_published_histories_grid.py | 10 ++------- .../test_workflow_management.py | 16 +++----------- 3 files changed, 26 insertions(+), 21 deletions(-) diff --git a/test/galaxy_selenium/navigates_galaxy.py b/test/galaxy_selenium/navigates_galaxy.py index e500422446a..ecb594869ee 100644 --- a/test/galaxy_selenium/navigates_galaxy.py +++ b/test/galaxy_selenium/navigates_galaxy.py @@ -223,12 +223,27 @@ class NavigatesGalaxy(HasDriver): raise self.prepend_timeout_message(e, message) return history_item_selector_state + def published_grid_search_for(self, search_term=None): + return self._inline_search_for( + '#input-free-text-search-filter', + search_term, + ) + def get_logged_in_user(self): return self.api_get("users/current") def is_logged_in(self): return "email" in self.get_logged_in_user() + @retry_during_transitions + def _inline_search_for(self, selector, search_term=None): + search_box = self.wait_for_and_click_selector(selector) + search_box.clear() + if search_term is not None: + search_box.send_keys(search_term) + self.send_enter(search_box) + return search_box + def _get_random_name(self, prefix=None, suffix=None, len=10): return '%s%s%s' % ( prefix or '', @@ -502,6 +517,12 @@ class NavigatesGalaxy(HasDriver): def workflow_index_click_search(self): return self.wait_for_and_click_selector("input.search-wf") + def workflow_index_search_for(self, search_term=None): + return self._inline_search_for( + "input.search-wf", + search_term, + ) + def workflow_index_click_import(self): self.wait_for_and_click_selector(self.test_data["selectors"]["workflows"]["import_button"]) diff --git a/test/selenium_tests/test_published_histories_grid.py b/test/selenium_tests/test_published_histories_grid.py index 9937a19e602..d1c0aa8df5e 100644 --- a/test/selenium_tests/test_published_histories_grid.py +++ b/test/selenium_tests/test_published_histories_grid.py @@ -22,17 +22,11 @@ class HistoryGridTestCase(SharedStateSeleniumTestCase): def test_history_grid_search_standard(self): self.navigate_to_published_histories_page() - input_selector = '#input-free-text-search-filter' - search_input = self.wait_for_selector(input_selector) - search_input.send_keys(self.history1_name) - self.send_enter(search_input) - + self.published_grid_search_for(self.history1_name) self.assert_grid_histories_are([self.history1_name]) self.unset_filter('free-text-search', self.history1_name) - search_input = self.wait_for_selector(input_selector) - search_input.send_keys(self.history4_name) - self.send_enter(search_input) + self.published_grid_search_for(self.history4_name) self.assert_grid_histories_are(['No Items']) diff --git a/test/selenium_tests/test_workflow_management.py b/test/selenium_tests/test_workflow_management.py index 3d76b0f1c9c..beea3684851 100644 --- a/test/selenium_tests/test_workflow_management.py +++ b/test/selenium_tests/test_workflow_management.py @@ -74,13 +74,13 @@ class WorkflowManagementTestCase(SeleniumTestCase): self.workflow_index_rename("searchforthis") self._assert_showing_n_workflows(1) - self._click_and_search("doesnotmatch") + self.workflow_index_search_for("doesnotmatch") self._assert_showing_n_workflows(0) - self._click_and_search() + self.workflow_index_search_for() self._assert_showing_n_workflows(1) - self._click_and_search("searchforthis") + self.workflow_index_search_for("searchforthis") self._assert_showing_n_workflows(1) @selenium_test @@ -103,16 +103,6 @@ class WorkflowManagementTestCase(SeleniumTestCase): self.workflow_index_open() assert_published_column_text_is("Yes") - @retry_during_transitions - def _click_and_search(self, search_term=None): - # Allow default search_term of None to just clear search - search_box = self.workflow_index_click_search() - search_box.clear() - if search_term is not None: - search_box.send_keys(search_term) - self.send_enter(search_box) - return search_box - @retry_assertion_during_transitions def _assert_showing_n_workflows(self, n): self.assertEqual(len(self.workflow_index_table_elements()), n) From cf037fcc0c0e1f8954361812ec36d14515c8f3ab Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 22 Sep 2017 08:53:24 -0400 Subject: [PATCH 23/66] Selenium - more robust modal checking in test_workflow_editor. During this transient failure: https://jenkins.galaxyproject.org/job/selenium/519/artifact/519-test-errors/test_missing_tools2017092118271506032874/stacktrace.txt the workflow was still loading when an assertion about the resulting modal text was issued. Avoiding the arbitrary sleep and instead just retrying the assertion until it holds true or times out is more robust and should prevent that. --- test/galaxy_selenium/navigates_galaxy.py | 7 ++++-- test/selenium_tests/test_workflow_editor.py | 24 ++++++++++----------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/test/galaxy_selenium/navigates_galaxy.py b/test/galaxy_selenium/navigates_galaxy.py index ecb594869ee..cff486af9a7 100644 --- a/test/galaxy_selenium/navigates_galaxy.py +++ b/test/galaxy_selenium/navigates_galaxy.py @@ -26,6 +26,9 @@ from . import sizzle # Test case data DEFAULT_PASSWORD = '123456' +RETRY_DURING_TRANSITIONS_SLEEP_DEFAULT = .1 +RETRY_DURING_TRANSITIONS_ATTEMPTS_DEFAULT = 10 + class NullTourCallback(object): @@ -50,7 +53,7 @@ def excepion_seems_to_indicate_transition(e): return exception_indicates_stale_element(e) or execption_indicates_not_clickable(e) -def retry_call_during_transitions(f, attempts=5, sleep=.1, exception_check=excepion_seems_to_indicate_transition): +def retry_call_during_transitions(f, attempts=RETRY_DURING_TRANSITIONS_ATTEMPTS_DEFAULT, sleep=RETRY_DURING_TRANSITIONS_SLEEP_DEFAULT, exception_check=excepion_seems_to_indicate_transition): previous_attempts = 0 while True: try: @@ -66,7 +69,7 @@ def retry_call_during_transitions(f, attempts=5, sleep=.1, exception_check=excep previous_attempts += 1 -def retry_during_transitions(f, attempts=5, sleep=.1, exception_check=excepion_seems_to_indicate_transition): +def retry_during_transitions(f, attempts=RETRY_DURING_TRANSITIONS_ATTEMPTS_DEFAULT, sleep=RETRY_DURING_TRANSITIONS_SLEEP_DEFAULT, exception_check=excepion_seems_to_indicate_transition): @wraps(f) def _retry(*args, **kwds): diff --git a/test/selenium_tests/test_workflow_editor.py b/test/selenium_tests/test_workflow_editor.py index 6dd7044b500..458cb3c032a 100644 --- a/test/selenium_tests/test_workflow_editor.py +++ b/test/selenium_tests/test_workflow_editor.py @@ -1,6 +1,7 @@ import time from .framework import ( + retry_assertion_during_transitions, selenium_test, SeleniumTestCase ) @@ -52,10 +53,7 @@ class WorkflowEditorTestCase(SeleniumTestCase): workflow_populator.upload_yaml_workflow(WORKFLOW_WITH_OLD_TOOL_VERSION, exact_tools=True) self.workflow_index_open() self.workflow_index_click_option("Edit") - time.sleep(.5) - modal_element = self.wait_for_selector_visible(self.modal_body_selector()) - text = modal_element.text - assert "Using version '0.2' instead of version '0.0.1'" in text, text + self.assert_modal_has_text("Using version '0.2' instead of version '0.0.1'") @selenium_test def test_editor_invalid_tool_state(self): @@ -63,11 +61,8 @@ class WorkflowEditorTestCase(SeleniumTestCase): workflow_populator.upload_yaml_workflow(WORKFLOW_WITH_INVALID_STATE, exact_tools=True) self.workflow_index_open() self.workflow_index_click_option("Edit") - time.sleep(.5) - modal_element = self.wait_for_selector_visible(self.modal_body_selector()) - text = modal_element.text - assert "Using version '0.2' instead of version '0.0.1'" in text, text - assert "Using default: '1'" in text, text + self.assert_modal_has_text("Using version '0.2' instead of version '0.0.1'") + self.assert_modal_has_text("Using default: '1'") @selenium_test def test_missing_tools(self): @@ -84,10 +79,7 @@ steps: """) self.workflow_index_open() self.workflow_index_click_option("Edit") - time.sleep(.5) - modal_element = self.wait_for_selector_visible(self.modal_body_selector()) - text = modal_element.text - assert "Tool is not installed" in text, text + self.assert_modal_has_text("Tool is not installed") def workflow_create_new(self, name=None, annotation=None): self.workflow_index_open() @@ -104,3 +96,9 @@ steps: 'workflow_annotation': annotation, }) self.click_submit(form_element) + + @retry_assertion_during_transitions + def assert_modal_has_text(self, expected_text): + modal_element = self.wait_for_selector_visible(self.modal_body_selector()) + text = modal_element.text + assert expected_text in text, "Failed to find expected text [%s] in modal text [%s]" % (expected_text, text) From 341667437ae718258a8d543c4e09d187f7f85326 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 22 Sep 2017 09:22:09 -0400 Subject: [PATCH 24/66] Fix selenium test retrying of tests in test_saved_histories. The automatic retrying of tests only runs for methods - not for the full test case (if that makes sense) - so setUp() isn't rerun. By refactoring the login action out of the setUp method and into the test function for these cases - when the test is retried the user should be logged back in again as well. --- .../test_published_histories_grid.py | 4 ---- test/selenium_tests/test_saved_histories.py | 21 ++++++++++++++----- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/test/selenium_tests/test_published_histories_grid.py b/test/selenium_tests/test_published_histories_grid.py index d1c0aa8df5e..7cf86e06884 100644 --- a/test/selenium_tests/test_published_histories_grid.py +++ b/test/selenium_tests/test_published_histories_grid.py @@ -9,10 +9,6 @@ from .framework import ( class HistoryGridTestCase(SharedStateSeleniumTestCase): - def setUp(self): - super(HistoryGridTestCase, self).setUp() - self.home() - @selenium_test def test_history_grid_histories(self): self.navigate_to_published_histories_page() diff --git a/test/selenium_tests/test_saved_histories.py b/test/selenium_tests/test_saved_histories.py index 422b7c8d713..8bd65ec0348 100644 --- a/test/selenium_tests/test_saved_histories.py +++ b/test/selenium_tests/test_saved_histories.py @@ -9,18 +9,15 @@ from .framework import ( class SavedHistoriesTestCase(SharedStateSeleniumTestCase): - def setUp(self): - super(SavedHistoriesTestCase, self).setUp() - self.home() - self.submit_login(self.user_email, retries=3) - @selenium_test def test_saved_histories_list(self): + self._login() self.navigate_to_saved_histories_page() self.assert_histories_in_grid([self.history2_name, self.history3_name]) @selenium_test def test_history_switch(self): + self._login() self.navigate_to_saved_histories_page() self.click_popup_option(self.history2_name, 'Switch') time.sleep(1) @@ -29,6 +26,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): @selenium_test def test_history_view(self): + self._login() self.navigate_to_saved_histories_page() self.click_popup_option(self.history2_name, 'View') history_name = self.wait_for_selector('.name.editable-text') @@ -36,6 +34,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): @selenium_test def test_history_publish(self): + self._login() self.navigate_to_saved_histories_page() # Publish the history @@ -52,6 +51,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): @selenium_test def test_rename_history(self): + self._login() self.navigate_to_saved_histories_page() self.click_popup_option('Unnamed history', 'Rename') @@ -72,6 +72,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): @selenium_test def test_delete_and_undelete_history(self): + self._login() self.navigate_to_saved_histories_page() # Delete the history @@ -93,6 +94,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): @selenium_test def test_permanently_delete_history(self): + self._login() self.create_history(self.history4_name) self.navigate_to_saved_histories_page() @@ -111,6 +113,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): @selenium_test def test_delete_and_undelete_multiple_histories(self): + self._login() self.navigate_to_saved_histories_page() delete_button_selector = 'input[type="button"][value="Delete"]' @@ -139,6 +142,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): @selenium_test def test_sort_by_name(self): + self._login() self.navigate_to_saved_histories_page() self.wait_for_and_click_selector('.sort-link[sort_key="name"]') @@ -156,6 +160,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): @selenium_test def test_standard_search(self): + self._login() self.navigate_to_saved_histories_page() input_selector = '#input-free-text-search-filter' @@ -174,6 +179,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): @selenium_test def test_advanced_search(self): + self._login() self.navigate_to_saved_histories_page() self.show_advanced_search() @@ -201,6 +207,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): @selenium_test def test_tags(self): + self._login() self.navigate_to_saved_histories_page() # Click the add tag button @@ -221,6 +228,10 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self.assert_grid_histories_are([self.history2_name], False) + def _login(self): + self.home() + self.submit_login(self.user_email, retries=3) + @retry_assertion_during_transitions def assert_grid_histories_are(self, expected_histories, sort_matters=True): actual_histories = self.get_histories() From 3014b1068bc4b7dc42d3f4c0754f6eed881b453f Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Fri, 22 Sep 2017 14:44:59 -0400 Subject: [PATCH 25/66] remove unused import --- test/selenium_tests/test_workflow_management.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/selenium_tests/test_workflow_management.py b/test/selenium_tests/test_workflow_management.py index beea3684851..760d1c85108 100644 --- a/test/selenium_tests/test_workflow_management.py +++ b/test/selenium_tests/test_workflow_management.py @@ -1,6 +1,5 @@ from .framework import ( retry_assertion_during_transitions, - retry_during_transitions, selenium_test, SeleniumTestCase, ) From 8fa1f9a2fea0e15213bfc4c666feaff472076225 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 22 Sep 2017 14:47:15 -0400 Subject: [PATCH 26/66] Small tweaks to fixup #2866. --- lib/galaxy/webapps/reports/config.py | 2 +- scripts/common_startup_functions.sh | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/reports/config.py b/lib/galaxy/webapps/reports/config.py index b69c3f2640f..80b20d3b46e 100644 --- a/lib/galaxy/webapps/reports/config.py +++ b/lib/galaxy/webapps/reports/config.py @@ -64,7 +64,7 @@ class Configuration(object): def check(self): # Check that required directories exist - for path in self.root, self.file_path, self.template_path: + for path in self.root, self.template_path: if not os.path.isdir(path): raise ConfigurationError("Directory does not exist: %s" % path) diff --git a/scripts/common_startup_functions.sh b/scripts/common_startup_functions.sh index 8deed854db5..6b2e0cc3d6e 100644 --- a/scripts/common_startup_functions.sh +++ b/scripts/common_startup_functions.sh @@ -1,6 +1,8 @@ #!/bin/sh -uwsgi_args="--master --pythonpath=lib" +: ${GALAXY_STATIC_DIRECTORY:=static} +: ${GALAXY_STYLE_DIRECTORY:=static/style/blue} +uwsgi_args="--master --pythonpath=lib --static-map /static=$GALAXY_STATIC_DIRECTORY --static-map /static/style=$GALAXY_STYLE_DIRECTORY" parse_common_args() { INITIALIZE_TOOL_DEPENDENCIES=1 From 41013563e5e15745fd2bf796340578f6d758fbe8 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 24 Sep 2017 21:18:50 +0200 Subject: [PATCH 27/66] Update CITATION file with 2016 paper --- CITATION | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CITATION b/CITATION index 2151602534b..49be43c17d1 100644 --- a/CITATION +++ b/CITATION @@ -1,6 +1,13 @@ If you use or extend Galaxy in your published work, please cite each of the following publications: +- Afgan E, Baker D, van den Beek M, Blankenberg D, Bouvier D, Čech M, Chilton + J, Clements D, Coraor N, Eberhard C, Grüning B, Guerler A, Hillman-Jackson + J,Von Kuster G, Rasche E, Soranzo N, Turaga N, Taylor J, Nekrutenko A, Goecks + J. "The Galaxy platform for accessible, reproducible and collaborative + biomedical analyses: 2016 update." + Nucleic acids research, 44(W1), W3-W10. + - Goecks, J, Nekrutenko, A, Taylor, J and The Galaxy Team. "Galaxy: a comprehensive approach for supporting accessible, reproducible, and transparent computational research in the life sciences." @@ -22,6 +29,17 @@ See also: https://galaxyproject.org/citing-galaxy BibTeX format: +@article{afgan2016galaxy, + title={The Galaxy platform for accessible, reproducible and collaborative biomedical analyses: 2016 update}, + author={Afgan, Enis and Baker, Dannon and Van den Beek, Marius and Blankenberg, Daniel and Bouvier, Dave and Čech, Martin and Chilton, John and Clements, Dave and Coraor, Nate and Eberhard, Carl and others}, + journal={Nucleic acids research}, + volume={44}, + number={W1}, + pages={W3--W10}, + year={2016}, + publisher={Oxford University Press} +} + @article{goecks2010galaxy, title={Galaxy: a comprehensive approach for supporting accessible, reproducible, and transparent computational research in the life sciences}, author={Goecks, Jeremy and Nekrutenko, Anton and Taylor, James and The Galaxy Team}, From 0b1dd637a98c9b878972a3b7b0faf783a045f2b3 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 23 Sep 2017 18:09:20 -0400 Subject: [PATCH 28/66] Selenium - bug fix for history test options. Wait on something useful instead of picking an arbitrary sleep time and praying. --- test/galaxy_selenium/navigates_galaxy.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/galaxy_selenium/navigates_galaxy.py b/test/galaxy_selenium/navigates_galaxy.py index cff486af9a7..5c5ae4a2d86 100644 --- a/test/galaxy_selenium/navigates_galaxy.py +++ b/test/galaxy_selenium/navigates_galaxy.py @@ -755,10 +755,14 @@ class NavigatesGalaxy(HasDriver): else: item_selector = self.history_panel_item_selector(kwds["hid"]) title_selector = "%s .title" % item_selector + details_selector = "%s .details" % item_selector + details_displayed = self.selector_is_displayed(details_selector) self.wait_for_and_click_selector(title_selector) if kwds.get("wait", False): - # Find a better way to wait for transition - time.sleep(.5) + if details_displayed: + self.wait_for_selector_absent_or_hidden(details_selector) + else: + self.wait_for_selector_visible(details_selector) def click_hda_title(self, hda_id, wait=False): # TODO: Replace with calls to history_panel_click_item_title. From aa825d22aa8ec934fc4ec2a30f20fc821c8e3c0b Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 23 Sep 2017 18:34:19 -0400 Subject: [PATCH 29/66] Selenium - fix transient issue with saved histories test. --- test/galaxy_selenium/navigates_galaxy.py | 5 +++++ test/selenium_tests/test_saved_histories.py | 14 +++++--------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/test/galaxy_selenium/navigates_galaxy.py b/test/galaxy_selenium/navigates_galaxy.py index 5c5ae4a2d86..c238e23ab8a 100644 --- a/test/galaxy_selenium/navigates_galaxy.py +++ b/test/galaxy_selenium/navigates_galaxy.py @@ -659,6 +659,11 @@ class NavigatesGalaxy(HasDriver): menu_selector = self.test_data["historyOptions"]["selectors"]["menu"] return menu_selector + @retry_during_transitions + def histories_click_advanced_search(self): + search_selector = '#standard-search .advanced-search-toggle' + self.wait_for_and_click_selector(search_selector) + def history_panel_add_tags(self, tags): tag_icon_selector = self.test_data['historyPanel']['selectors']['history']['tagIcon'] tag_area_selector = self.test_data['historyPanel']['selectors']['history']['tagArea'] diff --git a/test/selenium_tests/test_saved_histories.py b/test/selenium_tests/test_saved_histories.py index 8bd65ec0348..eeed1d20b23 100644 --- a/test/selenium_tests/test_saved_histories.py +++ b/test/selenium_tests/test_saved_histories.py @@ -43,7 +43,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self.navigate_to_saved_histories_page() - self.show_advanced_search() + self.histories_click_advanced_search() self.select_filter('sharing', 'published') time.sleep(1) @@ -80,7 +80,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self.assert_histories_in_grid([self.history2_name], False) - self.show_advanced_search() + self.histories_click_advanced_search() self.select_filter('deleted', 'True') time.sleep(1) @@ -106,7 +106,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self.assert_histories_in_grid([self.history4_name], False) - self.show_advanced_search() + self.histories_click_advanced_search() self.select_filter('deleted', 'True') self.assert_histories_in_grid([self.history4_name]) @@ -125,7 +125,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self.assert_histories_in_grid([self.history2_name, self.history3_name], False) - self.show_advanced_search() + self.histories_click_advanced_search() self.select_filter('deleted', 'True') time.sleep(1) @@ -182,7 +182,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self._login() self.navigate_to_saved_histories_page() - self.show_advanced_search() + self.histories_click_advanced_search() name_filter_selector = '#input-name-filter' tags_filter_selector = '#input-tags-filter' @@ -299,10 +299,6 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self.click_history_option('Create New') self.history_panel_rename(name) - def show_advanced_search(self): - search_selector = '#standard-search .advanced-search-toggle' - self.wait_for_and_click_selector(search_selector) - def select_filter(self, filter_key, filter_value): filter_selector = 'a[filter_key="%s"][filter_val="%s"]' % \ (filter_key, filter_value) From 76fe60180a4b9059d0fb110573e93bf6274be893 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 23 Sep 2017 20:30:44 -0400 Subject: [PATCH 30/66] Selenium - fix bug where snapshots might be used in tests before it is setup. --- test/selenium_tests/framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/selenium_tests/framework.py b/test/selenium_tests/framework.py index f775ade1679..01532fc8fe5 100644 --- a/test/selenium_tests/framework.py +++ b/test/selenium_tests/framework.py @@ -149,8 +149,8 @@ class SeleniumTestCase(FunctionalTestCase, NavigatesGalaxy): self.target_url_from_selenium = GALAXY_TEST_EXTERNAL_FROM_SELENIUM else: self.target_url_from_selenium = self.url - self.setup_driver_and_session() self.snapshots = [] + self.setup_driver_and_session() def tearDown(self): exception = None From 4a4d0498d6a371dc8ddaa226d33ff5cf0a897e69 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 24 Sep 2017 15:10:13 -0400 Subject: [PATCH 31/66] Selenium - better fix for upload build click before callback registered. Should fix this transient failure. https://jenkins.galaxyproject.org/job/selenium/568/artifact/568-test-errors/test_upload_paired_list2017092319301506209412/stacktrace.txt --- test/galaxy_selenium/navigates_galaxy.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/test/galaxy_selenium/navigates_galaxy.py b/test/galaxy_selenium/navigates_galaxy.py index c238e23ab8a..3900ebfebe8 100644 --- a/test/galaxy_selenium/navigates_galaxy.py +++ b/test/galaxy_selenium/navigates_galaxy.py @@ -460,13 +460,19 @@ class NavigatesGalaxy(HasDriver): @retry_during_transitions def upload_build(self): - build_button = self.wait_for_selector_clickable("div#collection button#btn-build") - # TODO: Eliminate the need for this hack. This hack is in here because the test - # occasionally fails at the next step because the UI has not transitioned to the - # new content. I assume that means this click is sent before the callback is - # registered. + build_selector = "div#collection button#btn-build" + # Pause a bit to let the callback on the build button be registered. time.sleep(.5) - build_button.click() + # Click the Build button and make sure it disappears. + self.wait_for_and_click_selector(build_selector) + try: + self.wait_for_selector_absent_or_hidden(build_selector) + except TimeoutException: + # Sometimes the callback in the JS hasn't be registered by the + # time that the build button is clicked. By the time the timeout + # has been registered - it should have been. + self.wait_for_and_click_selector(build_selector) + self.wait_for_selector_absent_or_hidden(build_selector) def upload_queue_local_file(self, test_path, tab_id="regular"): self.wait_for_and_click_selector("div#%s button#btn-local" % tab_id) From 6168e243b28cd85b8b8b719acc55d3e2108af43a Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 24 Sep 2017 15:12:17 -0400 Subject: [PATCH 32/66] Selenium - fix rerunning test errors for test_custom_builds. Fixed similar problem for saved histories tests with 341667437ae718258a8d543c4e09d187f7f85326. Should fix this problem https://jenkins.galaxyproject.org/job/selenium/568/artifact/568-test-errors/test_build_delete2017092319091506208182/stacktrace.txt. --- test/selenium_tests/test_custom_builds.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/test/selenium_tests/test_custom_builds.py b/test/selenium_tests/test_custom_builds.py index f29d04e189b..f15cef7b4df 100644 --- a/test/selenium_tests/test_custom_builds.py +++ b/test/selenium_tests/test_custom_builds.py @@ -9,19 +9,19 @@ from .framework import ( class CustomBuildsTestcase(SharedStateSeleniumTestCase): - def setUp(self): - super(CustomBuildsTestcase, self).setUp() - self.home() # ensure Galaxy is loaded - self.submit_login(self.user_email, retries=2) - @selenium_test def test_build_add(self): + self._login() + self.navigate_to_custom_builds_page() + self.add_custom_build(self.build_name1, self.build_key1) self.assert_custom_builds_in_grid([self.build_name1]) @selenium_test def test_build_delete(self): + self._login() + self.navigate_to_custom_builds_page() self.add_custom_build(self.build_name2, self.build_key2) @@ -39,6 +39,10 @@ class CustomBuildsTestcase(SharedStateSeleniumTestCase): else: self.assertEqual(intersection, set()) + def _login(self): + self.home() # ensure Galaxy is loaded + self.submit_login(self.user_email, retries=2) + def add_custom_build(self, build_name, build_key): name_div = self.wait_for_selector('div[tour_id="name"') name_input = name_div.find_element_by_css_selector('.ui-form-field input') From c90a84e1c211daf680b7e0a8c7aca246bc3152cf Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 24 Sep 2017 15:29:30 -0400 Subject: [PATCH 33/66] Selenium - fix transient error in history switching test. History sometimes takes longer than that second to switch in the GUI, so start a retry cycle on that assertion after a half a second. xref https://jenkins.galaxyproject.org/job/selenium/586/artifact/586-test-errors/test_history_switch2017092413201506273602/stacktrace.txt --- test/selenium_tests/test_saved_histories.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test/selenium_tests/test_saved_histories.py b/test/selenium_tests/test_saved_histories.py index eeed1d20b23..ffb3f4e6dac 100644 --- a/test/selenium_tests/test_saved_histories.py +++ b/test/selenium_tests/test_saved_histories.py @@ -20,9 +20,14 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self._login() self.navigate_to_saved_histories_page() self.click_popup_option(self.history2_name, 'Switch') - time.sleep(1) - history_name = self.history_panel_name_element() - self.assertEqual(history_name.text, self.history2_name) + time.sleep(.5) + + @retry_assertion_during_transitions + def assert_history_name_switched(): + history_name = self.history_panel_name_element() + self.assertEqual(history_name.text, self.history2_name) + + assert_history_name_switched() @selenium_test def test_history_view(self): From 97cf3b94d1571401f3d57a83dd1a644ff356e772 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 24 Sep 2017 15:41:31 -0400 Subject: [PATCH 34/66] Selenium - fix transiently failing test caused by not waiting on anonymous history loading. After logging out - may need to a wait a second to give the UI and backend a chance to create a new history for the now anoymous user. Pause and let that happen. xref https://jenkins.galaxyproject.org/job/selenium/585/artifact/585-test-errors/test_clean_anon_history_after_logout2017092412041506269041/ --- test/galaxy_selenium/navigates_galaxy.py | 6 ++++++ test/selenium_tests/test_anon_history.py | 17 ++++++++++------- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/test/galaxy_selenium/navigates_galaxy.py b/test/galaxy_selenium/navigates_galaxy.py index 3900ebfebe8..bb5f93d8274 100644 --- a/test/galaxy_selenium/navigates_galaxy.py +++ b/test/galaxy_selenium/navigates_galaxy.py @@ -201,6 +201,12 @@ class NavigatesGalaxy(HasDriver): self.history_panel_refresh_click() return rval + def history_panel_wait_for_history_loaded(self): + # Use the search box showing up as a proxy that the history display + # has left the "loading" state and is showing a valid set of history contents + # (even if empty). + self.wait_for_selector_visible("#current-history-panel input.search-query") + def history_panel_wait_for_hid_hidden(self, hid, timeout=60): current_history_id = self.current_history_id() contents = self.api_get("histories/%s/contents" % current_history_id) diff --git a/test/selenium_tests/test_anon_history.py b/test/selenium_tests/test_anon_history.py index a037bfefa54..f26b8655f44 100644 --- a/test/selenium_tests/test_anon_history.py +++ b/test/selenium_tests/test_anon_history.py @@ -38,10 +38,7 @@ class AnonymousHistoriesTestCase(SeleniumTestCase): @selenium_test def test_anon_history_after_registration(self): - self.home() - self.perform_upload(self.get_filename("1.txt")) - self.wait_for_history() - self.register() + self._upload_file_anonymous_then_register_user() hda = self.latest_history_item() self.home() element = self.wait_for_selector(self.hda_div_selector(hda["id"])) @@ -49,10 +46,16 @@ class AnonymousHistoriesTestCase(SeleniumTestCase): @selenium_test def test_clean_anon_history_after_logout(self): + self._upload_file_anonymous_then_register_user() + self.logout_if_needed() + # Give Galaxy a chance to load the new empty history for that now + # anonymous user. Make sure this new history is empty. + self.history_panel_wait_for_history_loaded() + history_contents = self.current_history_contents() + assert len(history_contents) == 0 + + def _upload_file_anonymous_then_register_user(self): self.home() self.perform_upload(self.get_filename("1.txt")) self.wait_for_history() self.register() - self.logout_if_needed() - history_contents = self.current_history_contents() - assert len(history_contents) == 0 From 9d3ed321be4d0179616af7478eb96bb358b29607 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 24 Sep 2017 15:50:27 -0400 Subject: [PATCH 35/66] Selenium - fix transient history in test_saved_histories related to fetching history names. xref https://jenkins.galaxyproject.org/job/selenium/588/artifact/588-test-errors/test_delete_and_undelete_history2017092415231506281014/stacktrace.txt --- test/selenium_tests/test_saved_histories.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/selenium_tests/test_saved_histories.py b/test/selenium_tests/test_saved_histories.py index ffb3f4e6dac..196cb086a41 100644 --- a/test/selenium_tests/test_saved_histories.py +++ b/test/selenium_tests/test_saved_histories.py @@ -1,6 +1,7 @@ import time from .framework import ( + retry_during_transitions, retry_assertion_during_transitions, selenium_test, SharedStateSeleniumTestCase, @@ -254,8 +255,9 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): else: self.assertEqual(intersection, set()) + @retry_during_transitions def get_histories(self): - time.sleep(1.5) + time.sleep(.5) names = [] grid = self.wait_for_selector('#grid-table-body') for row in grid.find_elements_by_tag_name('tr'): From 9f5309688eb17b92d4733a7f9752f97e5a4139db Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 24 Sep 2017 17:13:36 -0400 Subject: [PATCH 36/66] Import order fix. --- test/selenium_tests/test_saved_histories.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/selenium_tests/test_saved_histories.py b/test/selenium_tests/test_saved_histories.py index 196cb086a41..64a8c5ddd86 100644 --- a/test/selenium_tests/test_saved_histories.py +++ b/test/selenium_tests/test_saved_histories.py @@ -1,8 +1,8 @@ import time from .framework import ( - retry_during_transitions, retry_assertion_during_transitions, + retry_during_transitions, selenium_test, SharedStateSeleniumTestCase, ) From 32b85ecb591d01acbbbb9ed1bedd1a48804a0837 Mon Sep 17 00:00:00 2001 From: E Rasche Date: Wed, 13 Sep 2017 06:03:55 +0000 Subject: [PATCH 37/66] Only permit yaml.safe_loading of data Event trusted data, belt + suspenders method. --- lib/galaxy/containers/__init__.py | 2 +- lib/galaxy/datatypes/registry.py | 2 +- lib/galaxy/jobs/runners/pulsar.py | 2 +- lib/galaxy/tools/deps/conda_compat.py | 2 +- lib/galaxy/tools/deps/resolvers/__init__.py | 2 +- lib/galaxy/tools/locations/dockstore.py | 2 +- lib/galaxy/tools/parser/factory.py | 2 +- lib/galaxy/tools/toolbox/parser.py | 2 +- lib/galaxy/tours/__init__.py | 2 +- lib/galaxy/util/plugin_config.py | 2 +- lib/galaxy/util/properties.py | 2 +- .../visualization/plugins/interactive_environments.py | 2 +- lib/galaxy/webapps/config_manage.py | 4 ++-- lib/galaxy/webapps/galaxy/api/users.py | 2 +- lib/galaxy/webapps/galaxy/controllers/visualization.py | 2 +- lib/galaxy/webhooks/__init__.py | 2 +- scripts/grt/export.py | 4 ++-- scripts/grt/upload.py | 4 ++-- test/api/test_workflows.py | 2 +- test/api/test_workflows_from_yaml.py | 6 +++--- test/base/workflows_format_2/converter.py | 4 ++-- test/base/workflows_format_2/main.py | 2 +- test/galaxy_selenium/data.py | 2 +- test/galaxy_selenium/navigates_galaxy.py | 2 +- test/unit/workflows/workflow_support.py | 2 +- 25 files changed, 31 insertions(+), 31 deletions(-) diff --git a/lib/galaxy/containers/__init__.py b/lib/galaxy/containers/__init__.py index 4b33dbe8d5a..6cebdd82e14 100644 --- a/lib/galaxy/containers/__init__.py +++ b/lib/galaxy/containers/__init__.py @@ -306,7 +306,7 @@ def parse_containers_config(containers_config_file): conf = DEFAULT_CONF.copy() try: with open(containers_config_file) as fh: - c = yaml.load(fh) + c = yaml.safe_load(fh) conf.update(c.get('containers', {})) except (OSError, IOError) as exc: if exc.errno == errno.ENOENT: diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 31902b14513..f11b8498719 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -359,7 +359,7 @@ class Registry(object): build_sites_config_file = getattr(self.config, "build_sites_config_file", None) if build_sites_config_file and os.path.exists(build_sites_config_file): with open(build_sites_config_file, "r") as f: - build_sites_config = yaml.load(f) + build_sites_config = yaml.safe_load(f) if not isinstance(build_sites_config, list): self.log.exception("Build sites configuration YAML file does not declare list of sites.") return diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 9722ea32736..9fee6ef7f81 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -217,7 +217,7 @@ class PulsarJobRunner(AsynchronousJobRunner): else: log.info("Loading Pulsar app configuration from %s" % pulsar_conf_path) with open(pulsar_conf_path, "r") as f: - conf.update(yaml.load(f) or {}) + conf.update(yaml.safe_load(f) or {}) if "job_metrics_config_file" not in conf: conf["job_metrics"] = self.app.job_metrics if "staging_directory" not in conf: diff --git a/lib/galaxy/tools/deps/conda_compat.py b/lib/galaxy/tools/deps/conda_compat.py index c45573ef49d..10527bf890d 100644 --- a/lib/galaxy/tools/deps/conda_compat.py +++ b/lib/galaxy/tools/deps/conda_compat.py @@ -61,7 +61,7 @@ def _render_jinja2(recipe_dir): @_Memoized def yamlize(data): - res = yaml.load(data) + res = yaml.safe_load(data) # ensure the result is a dict if res is None: res = {} diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py index e8188e0964a..5c3811024e0 100644 --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -97,7 +97,7 @@ class MappableDependencyResolver: @staticmethod def _mapping_file_to_list(mapping_file): with open(mapping_file, "r") as f: - raw_mapping = yaml.load(f) or [] + raw_mapping = yaml.safe_load(f) or [] return map(RequirementMapping.from_dict, raw_mapping) def _expand_mappings(self, requirement): diff --git a/lib/galaxy/tools/locations/dockstore.py b/lib/galaxy/tools/locations/dockstore.py index b379c23c38f..3eb96e55e24 100644 --- a/lib/galaxy/tools/locations/dockstore.py +++ b/lib/galaxy/tools/locations/dockstore.py @@ -57,7 +57,7 @@ class _Ga4ghToolClient(object): if as_string: return descriptor_str else: - return yaml.load(descriptor_str) + return yaml.safe_load(descriptor_str) @property def _requests(self): diff --git a/lib/galaxy/tools/parser/factory.py b/lib/galaxy/tools/parser/factory.py index 52a11ad7b57..b6655be63d5 100644 --- a/lib/galaxy/tools/parser/factory.py +++ b/lib/galaxy/tools/parser/factory.py @@ -62,7 +62,7 @@ def ordered_load(stream): yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping) - return yaml.load(stream, OrderedLoader) + return yaml.safe_load(stream, OrderedLoader) def get_input_source(content): diff --git a/lib/galaxy/tools/toolbox/parser.py b/lib/galaxy/tools/toolbox/parser.py index 3c79a8b4d7b..db1b582404c 100644 --- a/lib/galaxy/tools/toolbox/parser.py +++ b/lib/galaxy/tools/toolbox/parser.py @@ -59,7 +59,7 @@ class YamlToolConfSource(ToolConfSource): def __init__(self, config_filename): with open(config_filename, "r") as f: - as_dict = yaml.load(f) + as_dict = yaml.safe_load(f) self.as_dict = as_dict def parse_tool_path(self): diff --git a/lib/galaxy/tours/__init__.py b/lib/galaxy/tours/__init__.py index 0511aa29e83..c35d147759c 100644 --- a/lib/galaxy/tours/__init__.py +++ b/lib/galaxy/tours/__init__.py @@ -68,7 +68,7 @@ class ToursRegistry(object): tour_id = os.path.splitext(filename)[0] try: with open(tour_path) as handle: - conf = yaml.load(handle) + conf = yaml.safe_load(handle) tour = tour_loader(conf) self.tours[tour_id] = tour_loader(conf) log.info("Loaded tour '%s'" % tour_id) diff --git a/lib/galaxy/util/plugin_config.py b/lib/galaxy/util/plugin_config.py index 133998f210f..7f6866d6f2d 100644 --- a/lib/galaxy/util/plugin_config.py +++ b/lib/galaxy/util/plugin_config.py @@ -83,4 +83,4 @@ def __read_yaml(path): raise ImportError("Attempting to read YAML configuration file - but PyYAML dependency unavailable.") with open(path, "rb") as f: - return yaml.load(f) + return yaml.safe_load(f) diff --git a/lib/galaxy/util/properties.py b/lib/galaxy/util/properties.py index 2ca160c27f7..8c22e0aaf59 100644 --- a/lib/galaxy/util/properties.py +++ b/lib/galaxy/util/properties.py @@ -67,7 +67,7 @@ def load_app_properties( config_section = "galaxy" with open(config_file, "r") as f: - raw_properties = yaml.load(f) + raw_properties = yaml.safe_load(f) properties = raw_properties[config_section] or {} override_prefix = "%sOVERRIDE_" % config_prefix diff --git a/lib/galaxy/visualization/plugins/interactive_environments.py b/lib/galaxy/visualization/plugins/interactive_environments.py index 161882e9409..602da1257e1 100644 --- a/lib/galaxy/visualization/plugins/interactive_environments.py +++ b/lib/galaxy/visualization/plugins/interactive_environments.py @@ -111,7 +111,7 @@ class InteractiveEnvironmentRequest(object): raise Exception("[{0}] Could not find allowed_images.yml, or image tag in {0}.ini file for ".format(self.attr.viz_id)) with open(fn, 'r') as handle: - self.allowed_images = [x['image'] for x in yaml.load(handle)] + self.allowed_images = [x['image'] for x in yaml.safe_load(handle)] if len(self.allowed_images) == 0: raise Exception("No allowed images specified for " + self.attr.viz_id) diff --git a/lib/galaxy/webapps/config_manage.py b/lib/galaxy/webapps/config_manage.py index 07d0bc03658..4cad182179e 100644 --- a/lib/galaxy/webapps/config_manage.py +++ b/lib/galaxy/webapps/config_manage.py @@ -670,7 +670,7 @@ def _ordered_load(stream): def include(self, node): filename = os.path.join(self._root, self.construct_scalar(node)) with open(filename, 'r') as f: - return yaml.load(f, OrderedLoader) + return yaml.safe_load(f, OrderedLoader) def construct_mapping(loader, node): loader.flatten_mapping(node) @@ -681,7 +681,7 @@ def _ordered_load(stream): construct_mapping) OrderedLoader.add_constructor('!include', OrderedLoader.include) - return yaml.load(stream, OrderedLoader) + return yaml.safe_load(stream, OrderedLoader) def _ordered_dump(data, stream=None, Dumper=yaml.Dumper, **kwds): diff --git a/lib/galaxy/webapps/galaxy/api/users.py b/lib/galaxy/webapps/galaxy/api/users.py index 67ec64351f3..63c9e3a78a0 100644 --- a/lib/galaxy/webapps/galaxy/api/users.py +++ b/lib/galaxy/webapps/galaxy/api/users.py @@ -282,7 +282,7 @@ class UserAPIController(BaseAPIController, UsesTagsMixin, CreatesUsersMixin, Cre path = trans.app.config.user_preferences_extra_config_file try: with open(path, 'r') as stream: - config = yaml.load(stream) + config = yaml.safe_load(stream) except: log.warning('Config file (%s) could not be found or is malformed.' % path) return {} diff --git a/lib/galaxy/webapps/galaxy/controllers/visualization.py b/lib/galaxy/webapps/galaxy/controllers/visualization.py index f134ebe48b4..9c7c1e5424e 100644 --- a/lib/galaxy/webapps/galaxy/controllers/visualization.py +++ b/lib/galaxy/webapps/galaxy/controllers/visualization.py @@ -961,7 +961,7 @@ class VisualizationController(BaseUIController, SharableMixin, UsesVisualization continue with open(image_file, 'r') as handle: - self.gie_image_map[gie] = yaml.load(handle) + self.gie_image_map[gie] = yaml.safe_load(handle) return trans.fill_template_mako( "visualization/gie.mako", diff --git a/lib/galaxy/webhooks/__init__.py b/lib/galaxy/webhooks/__init__.py index 7e9c9482433..6cc2c34d9d9 100644 --- a/lib/galaxy/webhooks/__init__.py +++ b/lib/galaxy/webhooks/__init__.py @@ -64,7 +64,7 @@ class WebhooksRegistry(object): def load_webhook_from_config(self, config_dir, config_file): try: with open(os.path.join(config_dir, config_file)) as file: - config = yaml.load(file) + config = yaml.safe_load(file) path = os.path.normpath(os.path.join(config_dir, '..')) webhook = Webhook( config['name'], diff --git a/scripts/grt/export.py b/scripts/grt/export.py index 4619c6929c6..da8048522f2 100644 --- a/scripts/grt/export.py +++ b/scripts/grt/export.py @@ -191,11 +191,11 @@ def main(argv): annotate('init_start', 'Loading GRT configuration...') try: with open(args.config) as handle: - config = yaml.load(handle) + config = yaml.safe_load(handle) except Exception: logging.info('Using default GRT configuration') with open(sample_config) as handle: - config = yaml.load(handle) + config = yaml.safe_load(handle) annotate('init_end') REPORT_DIR = args.report_directory diff --git a/scripts/grt/upload.py b/scripts/grt/upload.py index 332cbe89e6d..a93efb86a62 100644 --- a/scripts/grt/upload.py +++ b/scripts/grt/upload.py @@ -28,11 +28,11 @@ def main(argv): logging.info('Loading GRT configuration...') try: with open(args.config) as handle: - config = yaml.load(handle) + config = yaml.safe_load(handle) except Exception: logging.info('Using default GRT configuration') with open(sample_config) as handle: - config = yaml.load(handle) + config = yaml.safe_load(handle) REPORT_DIR = args.report_directory GRT_URL = config['grt']['url'].rstrip('/') + '/' diff --git a/test/api/test_workflows.py b/test/api/test_workflows.py index 79a0c1c2218..f3c45c03368 100644 --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -172,7 +172,7 @@ class BaseWorkflowsApiTestCase(api.ApiTestCase): ) if jobs_descriptions is None: assert source_type != "path" - jobs_descriptions = yaml.load(has_workflow) + jobs_descriptions = yaml.safe_load(has_workflow) test_data = jobs_descriptions.get("test_data", {}) diff --git a/test/api/test_workflows_from_yaml.py b/test/api/test_workflows_from_yaml.py index 9beddf7aa2e..2b39628c27d 100644 --- a/test/api/test_workflows_from_yaml.py +++ b/test/api/test_workflows_from_yaml.py @@ -55,11 +55,11 @@ steps: assert tool_count['random_lines1'] == 1 assert tool_count['cat1'] == 2 -# FIXME: This test fails on some machines due to (we're guessing) yaml loading +# FIXME: This test fails on some machines due to (we're guessing) yaml.safe_loading # order being not guaranteed and inconsistent across platforms. The workflow -# yaml loader probably needs to enforce order using something like the +# yaml.safe_loader probably needs to enforce order using something like the # approach described here: -# https://stackoverflow.com/questions/13297744/pyyaml-control-ordering-of-items-called-by-yaml-load +# https://stackoverflow.com/questions/13297744/pyyaml-control-ordering-of-items-called-by-yaml.safe_load # def test_multiple_input( self ): # history_id = self.dataset_populator.new_history() # self._run_jobs(""" diff --git a/test/base/workflows_format_2/converter.py b/test/base/workflows_format_2/converter.py index 05a9a319c49..c35ec2797c0 100644 --- a/test/base/workflows_format_2/converter.py +++ b/test/base/workflows_format_2/converter.py @@ -32,7 +32,7 @@ RUN_ACTIONS_TO_STEPS = { def yaml_to_workflow(has_yaml, galaxy_interface, workflow_directory): """Convert a Format 2 workflow into standard Galaxy format from supplied stream.""" - as_python = yaml.load(has_yaml) + as_python = yaml.safe_load(has_yaml) return python_to_workflow(as_python, galaxy_interface, workflow_directory) @@ -109,7 +109,7 @@ def _python_to_workflow(as_python, conversion_context): run_action_path = run_action["@import"] runnable_path = os.path.join(conversion_context.workflow_directory, run_action_path) with open(runnable_path, "r") as f: - runnable_description = yaml.load(f) + runnable_description = yaml.safe_load(f) run_action = runnable_description run_class = run_action["class"] diff --git a/test/base/workflows_format_2/main.py b/test/base/workflows_format_2/main.py index 70d59a8db8d..a4af30e8da6 100644 --- a/test/base/workflows_format_2/main.py +++ b/test/base/workflows_format_2/main.py @@ -20,7 +20,7 @@ def convert_and_import_workflow(has_workflow, **kwds): if workflow_directory is None: workflow_directory = os.path.dirname(has_workflow) with open(workflow_path, "r") as f: - has_workflow = yaml.load(f) + has_workflow = yaml.safe_load(f) if workflow_directory is not None: workflow_directory = os.path.abspath(workflow_directory) diff --git a/test/galaxy_selenium/data.py b/test/galaxy_selenium/data.py index a3d969f8a4f..ef15e27197b 100644 --- a/test/galaxy_selenium/data.py +++ b/test/galaxy_selenium/data.py @@ -3,4 +3,4 @@ from pkg_resources import resource_string import yaml data_yaml = resource_string(__name__, 'navigation-data.yml').decode("UTF-8") -NAVIGATION_DATA = yaml.load(data_yaml) +NAVIGATION_DATA = yaml.safe_load(data_yaml) diff --git a/test/galaxy_selenium/navigates_galaxy.py b/test/galaxy_selenium/navigates_galaxy.py index cff486af9a7..1a60e7ada62 100644 --- a/test/galaxy_selenium/navigates_galaxy.py +++ b/test/galaxy_selenium/navigates_galaxy.py @@ -798,7 +798,7 @@ class NavigatesGalaxy(HasDriver): self.home() with open(path, "r") as f: - tour_dict = yaml.load(f) + tour_dict = yaml.safe_load(f) steps = tour_dict["steps"] for i, step in enumerate(steps): title = step.get("title", None) diff --git a/test/unit/workflows/workflow_support.py b/test/unit/workflows/workflow_support.py index ee04ded4fc9..6f4f567cb66 100644 --- a/test/unit/workflows/workflow_support.py +++ b/test/unit/workflows/workflow_support.py @@ -75,7 +75,7 @@ class TestToolbox(object): def yaml_to_model(has_dict, id_offset=100): if isinstance(has_dict, str): - has_dict = yaml.load(has_dict) + has_dict = yaml.safe_load(has_dict) workflow = model.Workflow() workflow.steps = [] From df41d4f4087f9b133750a2c350ea207753f30f54 Mon Sep 17 00:00:00 2001 From: E Rasche Date: Wed, 13 Sep 2017 06:17:27 +0000 Subject: [PATCH 38/66] Remove some shell=True and fix commands --- lib/galaxy/datatypes/converters/interval_to_coverage.py | 5 +++-- lib/galaxy/datatypes/converters/lped_to_pbed_converter.py | 6 +++--- .../datatypes/converters/pbed_ldreduced_converter.py | 3 +-- lib/galaxy/datatypes/converters/pbed_to_lped_converter.py | 4 +--- lib/galaxy/datatypes/text.py | 8 ++++---- lib/galaxy/jobs/runners/util/job_script/__init__.py | 2 +- lib/galaxy/jobs/runners/util/kill.py | 6 +++--- lib/galaxy/jobs/transfer_manager.py | 8 ++++---- 8 files changed, 20 insertions(+), 22 deletions(-) diff --git a/lib/galaxy/datatypes/converters/interval_to_coverage.py b/lib/galaxy/datatypes/converters/interval_to_coverage.py index b0b8bb4695c..d4ddd98927b 100644 --- a/lib/galaxy/datatypes/converters/interval_to_coverage.py +++ b/lib/galaxy/datatypes/converters/interval_to_coverage.py @@ -133,8 +133,9 @@ if __name__ == "__main__": # Sort through a tempfile first temp_file = tempfile.NamedTemporaryFile(mode="r") environ['LC_ALL'] = 'POSIX' - commandline = "sort -f -n -k %d -k %d -k %d -o %s %s" % (chr_col_1 + 1, start_col_1 + 1, end_col_1 + 1, temp_file.name, in_fname) - subprocess.check_call(commandline, shell=True) + subprocess.check_call([ + 'sort', '-f', '-n', '-k', chr_col_1 + 1, '-k', start_col_1 + 1, '-k', end_col_1 + 1, '-o', temp_file.name, in_fname + ]) coverage = CoverageWriter(out_stream=open(out_fname, "a"), chromCol=chr_col_2, positionCol=position_col_2, diff --git a/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py b/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py index 777259c3e0e..83b8050e46a 100644 --- a/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py +++ b/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py @@ -72,9 +72,9 @@ def rgConv(inpedfilepath, outhtmlname, outfilepath, plink): if not missval: print('### lped_to_pbed_converter.py cannot identify missing value in %s' % pedf) missval = '0' - cl = '%s --noweb --file %s --make-bed --out %s --missing-genotype %s' % (plink, inpedfilepath, outroot, missval) - p = subprocess.Popen(cl, shell=True, cwd=outfilepath) - p.wait() # run plink + subprocess.check_call([plink, '--noweb', '--file', inpedfilepath, + '--make-bed', '--out', outroot, + '--missing-genotype', missval], cwd=outfilepath) def main(): diff --git a/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py b/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py index 86c4a4d051f..fbe1834b8b1 100644 --- a/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py +++ b/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py @@ -41,8 +41,7 @@ def pruneLD(plinktasks=[], cd='./', vclbase=[]): for task in plinktasks: # each is a list vcl = vclbase + task with open(plog, 'w') as sto: - x = subprocess.Popen(' '.join(vcl), shell=True, stdout=sto, stderr=sto, cwd=cd) - x.wait() + subprocess.check_call(vcl, stdout=sto, stderr=sto, cwd=cd) try: lplog = open(plog, 'r').readlines() lplog = [elem for elem in lplog if elem.find('Pruning SNP') == -1] diff --git a/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py b/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py index 023c6ee5f30..24a052384a3 100644 --- a/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py +++ b/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py @@ -40,9 +40,7 @@ def rgConv(inpedfilepath, outhtmlname, outfilepath, plink): """ basename = os.path.split(inpedfilepath)[-1] # get basename outroot = os.path.join(outfilepath, basename) - cl = '%s --noweb --bfile %s --recode --out %s ' % (plink, inpedfilepath, outroot) - p = subprocess.Popen(cl, shell=True, cwd=outfilepath) - p.wait() # run plink + subprocess.check_call([plink, '--noweb', '--bfile', inpedfilepath, '--recode', '--out', outroot], cwd=outfilepath) def main(): diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index 4d71302b3f0..c201b872b1f 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -148,11 +148,11 @@ class Ipynb(Json): ofilename = ofile_handle.name ofile_handle.close() try: - cmd = 'jupyter nbconvert --to html --template full %s --output %s' % (dataset.file_name, ofilename) - log.info("Calling command %s" % cmd) - subprocess.call(cmd, shell=True) + cmd = ['jupyter', 'nbconvert', '--to', 'html', '--template', 'full', dataset.file_name, '--output', ofilename] + log.info("Calling command %s", ' '.join(cmd)) + subprocess.check_call(cmd) ofilename = '%s.html' % ofilename - except: + except subprocess.CalledProcessError: ofilename = dataset.file_name log.exception('Command "%s" failed. Could not convert the Jupyter Notebook to HTML, defaulting to plain text.', cmd) return open(ofilename) diff --git a/lib/galaxy/jobs/runners/util/job_script/__init__.py b/lib/galaxy/jobs/runners/util/job_script/__init__.py index 77af249e5c4..4c4c6b86e45 100644 --- a/lib/galaxy/jobs/runners/util/job_script/__init__.py +++ b/lib/galaxy/jobs/runners/util/job_script/__init__.py @@ -119,7 +119,7 @@ def _handle_script_integrity(path, config): sleep_amt = getattr(config, "check_job_script_integrity_sleep", DEFAULT_INTEGRITY_SLEEP) for i in range(count): try: - proc = subprocess.Popen([path], shell=True, env={"ABC_TEST_JOB_SCRIPT_INTEGRITY_XYZ": "1"}) + proc = subprocess.Popen([path], env={"ABC_TEST_JOB_SCRIPT_INTEGRITY_XYZ": "1"}) proc.wait() if proc.returncode == 42: script_integrity_verified = True diff --git a/lib/galaxy/jobs/runners/util/kill.py b/lib/galaxy/jobs/runners/util/kill.py index c0dbd913b13..1a458c63913 100644 --- a/lib/galaxy/jobs/runners/util/kill.py +++ b/lib/galaxy/jobs/runners/util/kill.py @@ -1,6 +1,6 @@ import os +import subprocess from platform import system -from subprocess import Popen from time import sleep try: @@ -41,8 +41,8 @@ def _stock_kill_pid(pid): def __kill_windows(pid): try: - Popen("taskkill /F /T /PID %i" % pid, shell=True) - except Exception: + subprocess.check_call(['taskkill', '/F', '/T', '/PID', pid]) + except subprocess.CalledProcessError: pass diff --git a/lib/galaxy/jobs/transfer_manager.py b/lib/galaxy/jobs/transfer_manager.py index 01acd6aae8f..8f3cd4539ee 100644 --- a/lib/galaxy/jobs/transfer_manager.py +++ b/lib/galaxy/jobs/transfer_manager.py @@ -23,7 +23,7 @@ class TransferManager(object): def __init__(self, app): self.app = app self.sa_session = app.model.context.current - self.command = 'python %s' % os.path.abspath(os.path.join(os.getcwd(), 'scripts', 'transfer.py')) + self.command = ['python', os.path.abspath(os.path.join(os.getcwd(), 'scripts', 'transfer.py'))] if app.config.get_bool('enable_job_recovery', True): # Only one Galaxy server process should be able to recover jobs! (otherwise you'll have nasty race conditions) self.running = True @@ -68,9 +68,9 @@ class TransferManager(object): # The transfer script should daemonize fairly quickly - if this is # not the case, this process will need to be moved to a # non-blocking method. - cmd = '%s %s' % (self.command, tj.id) - log.debug('Transfer command is: %s' % cmd) - p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + cmd = self.command + [tj.id] + log.debug('Transfer command is: %s', ' '.join(cmd)) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) p.wait() output = p.stdout.read(32768) if p.returncode != 0: From cd6a1afaa9980d958ad324fa987a07e7608bf841 Mon Sep 17 00:00:00 2001 From: E Rasche Date: Wed, 13 Sep 2017 06:26:33 +0000 Subject: [PATCH 39/66] Replace some os.system calls with subprocess --- lib/galaxy/datatypes/sequence.py | 4 ++-- lib/galaxy/datatypes/tabular.py | 13 +++++-------- lib/galaxy/jobs/runners/pulsar.py | 4 ++-- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index 1267ff3da4a..ee47ad70bbd 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -7,6 +7,7 @@ import logging import os import re import string +import subprocess import sys from cgi import escape from itertools import islice @@ -693,8 +694,7 @@ class BaseFastq (Sequence): else: commands = Sequence.get_split_commands_sequential(is_gzip(input_name), input_name, output_name, start_sequence, sequence_count) for cmd in commands: - if 0 != os.system(cmd): - raise Exception("Executing '%s' failed" % cmd) + subprocess.check_call(cmd, shell=True) return True process_split_file = staticmethod(process_split_file) diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 0100448e65a..9a6f0eb2cb5 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -521,15 +521,12 @@ class Sam(Tabular): Multiple SAM files may each have headers. Since the headers should all be the same, remove the headers from files 1-n, keeping them in the first file only """ - cmd = 'mv %s %s' % (split_files[0], output_file) - result = os.system(cmd) - if result != 0: - raise Exception('Result %s from %s' % (result, cmd)) + shutil.move(split_files[0], output_file) + if len(split_files) > 1: - cmd = 'egrep -v -h "^@" %s >> %s' % (' '.join(split_files[1:]), output_file) - result = os.system(cmd) - if result != 0: - raise Exception('Result %s from %s' % (result, cmd)) + cmd = ['egrep', '-v', '-h' '^@'] + split_files[1:] + ['>>', output_file] + subprocess.check_call(cmd, shell=True) + merge = staticmethod(merge) # Dataproviders diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 9fee6ef7f81..8c15475ee7c 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -7,6 +7,7 @@ from __future__ import absolute_import # Need to import pulsar_client absolutel import errno import logging import os +import subprocess from distutils.version import LooseVersion from time import sleep @@ -382,8 +383,7 @@ class PulsarJobRunner(AsynchronousJobRunner): prepare_input_files_cmds = getattr(job_wrapper, 'prepare_input_files_cmds', None) if prepare_input_files_cmds is not None: for cmd in prepare_input_files_cmds: # run the commands to stage the input files - if 0 != os.system(cmd): - raise Exception('Error running file staging command: %s' % cmd) + subprocess.check_call(cmd, shell=True) job_wrapper.prepare_input_files_cmds = None # prevent them from being used in-line def _populate_parameter_defaults(self, job_destination): From 73337725bdd654904563dc54b04083a23e4a252d Mon Sep 17 00:00:00 2001 From: E Rasche Date: Wed, 13 Sep 2017 06:29:30 +0000 Subject: [PATCH 40/66] Two more commands to list --- lib/galaxy/objectstore/s3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/objectstore/s3.py b/lib/galaxy/objectstore/s3.py index f832688741e..0d8297316e8 100644 --- a/lib/galaxy/objectstore/s3.py +++ b/lib/galaxy/objectstore/s3.py @@ -68,7 +68,7 @@ class S3ObjectStore(ObjectStore): log.info("Cache cleaner manager started") # Test if 'axel' is available for parallel download and pull the key into cache try: - subprocess.call('axel') + subprocess.check_call(['axel']) self.use_axel = True except OSError: self.use_axel = False @@ -341,7 +341,7 @@ class S3ObjectStore(ObjectStore): log.debug("Parallel pulled key '%s' into cache to %s", rel_path, self._get_cache_path(rel_path)) ncores = multiprocessing.cpu_count() url = key.generate_url(7200) - ret_code = subprocess.call("axel -a -n %s '%s'" % (ncores, url)) + ret_code = subprocess.call(['axel', '-a', '-n', ncores, url]) if ret_code == 0: return True else: From c85db3fd68590d3d2f61b0b3f88026773d9563a0 Mon Sep 17 00:00:00 2001 From: E Rasche Date: Wed, 13 Sep 2017 06:57:24 +0000 Subject: [PATCH 41/66] Some more command refactoring --- lib/galaxy/datatypes/binary.py | 37 ++++++++----------- lib/galaxy/tools/deps/mulled/mulled_build.py | 15 +++----- .../tools/deps/mulled/mulled_build_channel.py | 3 +- 3 files changed, 21 insertions(+), 34 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index b4ab524b778..0f8184cd1d1 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -253,18 +253,17 @@ class Bam(Binary): raise Exception(message) # Get the version of samtools via --version-only, if available - p = subprocess.Popen(['samtools', '--version-only'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - output, error = p.communicate() - - # --version-only is available - # Format is +htslib- - if p.returncode == 0: + try: + output = subprocess.check_output(['samtools', '--version-only']) + # --version-only is available + # Format is +htslib- version = output.split('+')[0] return version + except subprocess.CalledProcessError: + # --version-only not available + pass - output = subprocess.Popen(['samtools'], stderr=subprocess.PIPE, stdout=subprocess.PIPE).communicate()[1] + output = subprocess.check_output(['samtools']) lines = output.split('\n') for line in lines: if line.lower().startswith('version'): @@ -294,10 +293,8 @@ class Bam(Binary): def _is_coordinate_sorted(self, file_name): """See if the input BAM file is sorted from the header information.""" - params = ["samtools", "view", "-H", file_name] - output = subprocess.Popen(params, stderr=subprocess.PIPE, stdout=subprocess.PIPE).communicate()[0] - # find returns -1 if string is not found - return output.find("SO:coordinate") != -1 or output.find("SO:sorted") != -1 + output = subprocess.check_output(["samtools", "view", "-H", file_name]) + return 'SO:coordinate' in output or 'SO:sorted' in output def dataset_content_needs_grooming(self, file_name): """See if file_name is a sorted BAM file""" @@ -322,8 +319,7 @@ class Bam(Binary): return False index_name = tempfile.NamedTemporaryFile(prefix="bam_index").name stderr_name = tempfile.NamedTemporaryFile(prefix="bam_index_stderr").name - command = 'samtools index %s %s' % (file_name, index_name) - proc = subprocess.Popen(args=command, shell=True, stderr=open(stderr_name, 'wb')) + proc = subprocess.Popen(['samtools', 'index', file_name, index_name], stderr=open(stderr_name, 'wb')) proc.wait() stderr = open(stderr_name).read().strip() if stderr: @@ -366,8 +362,8 @@ class Bam(Binary): tmp_sorted_dataset_file_name_prefix = os.path.join(tmp_dir, 'sorted') stderr_name = tempfile.NamedTemporaryFile(dir=tmp_dir, prefix="bam_sort_stderr").name samtools_created_sorted_file_name = "%s.bam" % tmp_sorted_dataset_file_name_prefix # samtools accepts a prefix, not a filename, it always adds .bam to the prefix - command = "samtools sort %s %s" % (file_name, tmp_sorted_dataset_file_name_prefix) - proc = subprocess.Popen(args=command, shell=True, cwd=tmp_dir, stderr=open(stderr_name, 'wb')) + proc = subprocess.Popen(['samtools', 'sort', file_name, tmp_sorted_dataset_file_name_prefix], + cwd=tmp_dir, stderr=open(stderr_name, 'wb')) exit_code = proc.wait() # Did sort succeed? stderr = open(stderr_name).read().strip() @@ -1309,11 +1305,8 @@ class ExcelXls(Binary): edam_format = "format_3468" def sniff(self, filename): - mime_type = subprocess.check_output("file --mime-type '{}'".format(filename), shell=True).rstrip() - if mime_type.find("application/vnd.ms-excel") != -1: - return True - else: - return False + mime_type = subprocess.check_output(['file', '--mime-type', filename]).strip() + return "application/vnd.ms-excel" in mime_type def get_mime(self): """Returns the mime type of the datatype""" diff --git a/lib/galaxy/tools/deps/mulled/mulled_build.py b/lib/galaxy/tools/deps/mulled/mulled_build.py index 5d362f12cd3..381afbe2718 100644 --- a/lib/galaxy/tools/deps/mulled/mulled_build.py +++ b/lib/galaxy/tools/deps/mulled/mulled_build.py @@ -116,17 +116,12 @@ def get_affected_packages(args): """ recipes_dir = args.recipes_dir hours = args.diff_hours - cmd = """cd '%s' && git log --diff-filter=ACMRTUXB --name-only --pretty="" --since="%s hours ago" | grep -E '^recipes/.*/meta.yaml' | sort | uniq""" % (recipes_dir, hours) - pkg_list = check_output(cmd, shell=True) - ret = list() - for pkg in pkg_list.strip().split('\n'): + cmd = ['git', 'log', '--diff-filter=ACMRTUXB', '--name-only', '--pretty=""', '--since="%s hours ago"' % hours] + changed_files = subprocess.check_output(cmd, cwd=recipes_dir).strip().split('\n') + pkg_list = set([x for x in changed_files if x.startswith('recipes/') and x.endswith('meta.yaml')]) + for pkg in pkg_list: if pkg and os.path.exists(os.path.join(recipes_dir, pkg)): - ret.append((get_pkg_name(args, pkg), get_tests(args, pkg))) - return ret - - -def check_output(cmd, shell=True): - return subprocess.check_output(cmd, shell=shell) + yield (get_pkg_name(args, pkg), get_tests(args, pkg)) def conda_versions(pkg_name, file_name): diff --git a/lib/galaxy/tools/deps/mulled/mulled_build_channel.py b/lib/galaxy/tools/deps/mulled/mulled_build_channel.py index 23e3021187b..ff87189b65b 100644 --- a/lib/galaxy/tools/deps/mulled/mulled_build_channel.py +++ b/lib/galaxy/tools/deps/mulled/mulled_build_channel.py @@ -55,8 +55,7 @@ def _new_versions(quay, conda): def run_channel(args, build_last_n_versions=1): """Build list of involucro commands (as shell snippet) to run.""" - pkgs = get_affected_packages(args) - for pkg_name, pkg_tests in pkgs: + for pkg_name, pkg_tests in get_affected_packages(args): repo_data = _fetch_repo_data(args) c = conda_versions(pkg_name, repo_data) # only package the most recent N versions From 670897ec5ca2284c7fad453c3d595defe3cca280 Mon Sep 17 00:00:00 2001 From: E Rasche Date: Wed, 13 Sep 2017 07:03:48 +0000 Subject: [PATCH 42/66] One more os command --- lib/galaxy/webapps/reports/controllers/system.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/webapps/reports/controllers/system.py b/lib/galaxy/webapps/reports/controllers/system.py index 6359c1defd8..5c325c2239e 100644 --- a/lib/galaxy/webapps/reports/controllers/system.py +++ b/lib/galaxy/webapps/reports/controllers/system.py @@ -1,5 +1,6 @@ import logging import os +import subprocess from datetime import datetime, timedelta from decimal import Decimal @@ -148,12 +149,11 @@ class System(BaseUIController): message=message) def get_disk_usage(self, file_path): - df_cmd = 'df -h ' + file_path is_sym_link = os.path.islink(file_path) file_system = disk_size = disk_used = disk_avail = disk_cap_pct = mount = None - df_file = os.popen(df_cmd) - while True: - df_line = df_file.readline() + df_output = subprocess.check_output(['df', '-h', file_path]) + + for df_line in df_output: df_line = df_line.strip() if df_line: df_line = df_line.lower() @@ -176,7 +176,6 @@ class System(BaseUIController): pass else: break # EOF - df_file.close() return (file_system, disk_size, disk_used, disk_avail, disk_cap_pct, mount) @web.expose From 9b8e76f9d9df0ae0ba6238d44a8f07b97fe34d6f Mon Sep 17 00:00:00 2001 From: E Rasche Date: Wed, 13 Sep 2017 07:32:01 +0000 Subject: [PATCH 43/66] Convert urllib to requests Which has built-in protection against urls like "file:///tmp/a" --- cron/build_chrom_db.py | 6 ++--- cron/parse_builds.py | 5 ++--- cron/parse_builds_3_sites.py | 7 +++--- lib/galaxy/external_services/actions.py | 4 ++-- .../pacific_biosciences_smrt_portal.py | 7 +++--- lib/galaxy/managers/citations.py | 8 +++---- lib/galaxy/tools/data/__init__.py | 5 ++--- .../tools/imp_exp/unpack_tar_gz_archive.py | 19 +++++++--------- lib/galaxy/web/proxy/__init__.py | 14 +++++------- lib/galaxy/webapps/config_manage.py | 7 +++--- .../webapps/galaxy/controllers/async.py | 4 ++-- .../galaxy/controllers/library_common.py | 14 ++++++------ lib/galaxy/webapps/galaxy/controllers/root.py | 6 ++--- .../webapps/galaxy/controllers/workflow.py | 4 ++-- .../webapps/tool_shed/controllers/upload.py | 12 +++++----- lib/tool_shed/capsule/capsule_manager.py | 22 ++++++++----------- scripts/data_libraries/build_lucene_index.py | 6 ++--- scripts/edam_mapping.py | 4 ++-- scripts/microbes/harvest_bacteria.py | 6 ++--- scripts/tool_shed/api/export.py | 15 +++++-------- scripts/transfer.py | 1 + 21 files changed, 76 insertions(+), 100 deletions(-) diff --git a/cron/build_chrom_db.py b/cron/build_chrom_db.py index f6b2b8efc2f..d3cba98b8dc 100644 --- a/cron/build_chrom_db.py +++ b/cron/build_chrom_db.py @@ -15,10 +15,10 @@ from __future__ import print_function import fileinput import os +import requests import sys from six.moves.urllib.parse import urlencode -from six.moves.urllib.request import urlopen import parse_builds @@ -36,8 +36,8 @@ def getchrominfo(url, db): "hgta_regionType": "", "position": "", "hgta_doTopSubmit": "get info"}) - page = urlopen(URL) - for line in page: + page = requests.get(URL).text + for line in page.split('\n'): line = line.rstrip("\r\n") if line.startswith("#"): continue diff --git a/cron/parse_builds.py b/cron/parse_builds.py index c30197184f8..f61214186b7 100644 --- a/cron/parse_builds.py +++ b/cron/parse_builds.py @@ -6,21 +6,20 @@ build description """ from __future__ import print_function +import requests import sys import xml.etree.ElementTree as ElementTree -from six.moves.urllib.request import urlopen def getbuilds(url): try: - page = urlopen(url) + text = requests.get(url).text except: print("#Unable to open " + url) print("?\tunspecified (?)") sys.exit(1) - text = page.read() try: tree = ElementTree.fromstring(text) except: diff --git a/cron/parse_builds_3_sites.py b/cron/parse_builds_3_sites.py index b22cd449bf9..464064f4736 100644 --- a/cron/parse_builds_3_sites.py +++ b/cron/parse_builds_3_sites.py @@ -4,10 +4,9 @@ Connects to sites and determines which builds are available at each. """ from __future__ import print_function +import requests import xml.etree.ElementTree as ElementTree -from six.moves.urllib.request import urlopen - sites = ['http://genome.ucsc.edu/cgi-bin/', 'http://archaea.ucsc.edu/cgi-bin/', 'http://genome-test.cse.ucsc.edu/cgi-bin/'] @@ -20,11 +19,11 @@ def main(): trackurl = sites[i] + "hgTracks?" builds = [] try: - page = urlopen(site) + page = requests.get(site) except: print("#Unable to connect to " + site) continue - text = page.read() + text = page.text try: tree = ElementTree.fromstring(text) except: diff --git a/lib/galaxy/external_services/actions.py b/lib/galaxy/external_services/actions.py index 69fd20ca57b..71520265c49 100644 --- a/lib/galaxy/external_services/actions.py +++ b/lib/galaxy/external_services/actions.py @@ -1,6 +1,6 @@ # Contains actions that are used in External Services import logging -from urllib import urlopen +import requests from galaxy.web import url_for from galaxy.util.template import fill_template from result_handlers.basic import ExternalServiceActionResultHandler @@ -104,7 +104,7 @@ class ExternalServiceWebAPIActionResult(ExternalServiceResult): @property def content(self): if self._content is None: - self._content = urlopen(self.url).read() + self._content = requests.get(self.url).read() return self._content diff --git a/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py b/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py index a0e2707c951..267e0878655 100644 --- a/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py +++ b/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py @@ -4,10 +4,9 @@ produced by SMRT Portal. """ import json import logging +import requests from string import Template -from six.moves.urllib.request import urlopen - from .data_transfer import DataTransfer log = logging.getLogger(__name__) @@ -88,8 +87,8 @@ class SMRTPortalPlugin(DataTransfer): if self._missing_params(job.params, ['smrt_host', 'smrt_job_id']): return self.job_states.INVALID url = 'http://' + job.params['smrt_host'] + self.api_path + '/Jobs/' + job.params['smrt_job_id'] + '/Status' - r = urlopen(url) - status = json.loads(r.read()) + r = requests.get(url) + status = r.json() # TODO: error handling: unexpected json or bad response, bad url, etc. if status['Code'] == 'Completed': log.debug("SMRT Portal job '%s' is Completed. Initiating transfer." % job.params['smrt_job_id']) diff --git a/lib/galaxy/managers/citations.py b/lib/galaxy/managers/citations.py index 7b88b2a3728..5ff70e7c666 100644 --- a/lib/galaxy/managers/citations.py +++ b/lib/galaxy/managers/citations.py @@ -1,6 +1,6 @@ import functools import os -import urllib2 +import requests from beaker.cache import CacheManager from beaker.util import parse_cache_config_options @@ -47,10 +47,8 @@ class DoiCache(object): def _raw_get_bibtex(self, doi): dx_url = "http://dx.doi.org/" + doi headers = {'Accept': 'text/bibliography; style=bibtex, application/x-bibtex'} - req = urllib2.Request(dx_url, data="", headers=headers) - response = urllib2.urlopen(req) - bibtex = response.read() - return bibtex + req = requests.get(dx_url, headers=headers) + return req.text def get_bibtex(self, doi): createfunc = functools.partial(self._raw_get_bibtex, doi) diff --git a/lib/galaxy/tools/data/__init__.py b/lib/galaxy/tools/data/__init__.py index 92aa6c995ed..98013c5201e 100644 --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -11,13 +11,12 @@ import logging import os import os.path import re +import requests import string import time from glob import glob from tempfile import NamedTemporaryFile -from six.moves.urllib.request import urlopen - from galaxy import util from galaxy.util.dictifiable import Dictifiable from galaxy.util.odict import odict @@ -340,7 +339,7 @@ class TabularToolDataTable(ToolDataTable, Dictifiable): if filename: tmp_file = NamedTemporaryFile(prefix='TTDT_URL_%s-' % self.name) try: - tmp_file.write(urlopen(filename, timeout=url_timeout).read()) + tmp_file.write(requests.get(filename, timeout=url_timeout).text) except Exception as e: log.error('Error loading Data Table URL "%s": %s', filename, e) continue diff --git a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py index c00a636d5a7..31e5c922a6d 100644 --- a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py +++ b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py @@ -10,13 +10,12 @@ from __future__ import print_function import math import optparse import os +import requests import sys import tarfile import tempfile from base64 import b64decode -from six.moves.urllib.request import urlopen - # Set max size of archive/file that will be handled to be 100 GB. This is # arbitrary and should be adjusted as needed. MAX_SIZE = 100 * math.pow(2, 30) @@ -27,18 +26,16 @@ def url_to_file(url, dest_file): Transfer a file from a remote URL to a temporary file. """ try: - url_reader = urlopen(url) + url_reader = requests.get(url, stream=True) CHUNK = 10 * 1024 # 10k total = 0 fp = open(dest_file, 'wb') - while True: - chunk = url_reader.read(CHUNK) - if not chunk: - break - fp.write(chunk) - total += CHUNK - if total > MAX_SIZE: - break + for chunk in url_reader.iter_content(chunk_size=CHUNK): + if chunk: + fp.write(chunk) + total += CHUNK + if total > MAX_SIZE: + break fp.close() return dest_file except Exception as e: diff --git a/lib/galaxy/web/proxy/__init__.py b/lib/galaxy/web/proxy/__init__.py index 6ff3ee3399f..fcb23ba56c0 100644 --- a/lib/galaxy/web/proxy/__init__.py +++ b/lib/galaxy/web/proxy/__init__.py @@ -8,7 +8,7 @@ from galaxy.util import sockets from galaxy.util.lazy_process import LazyProcess, NoOpLazyProcess from galaxy.util import sqlite from galaxy.util import unique_id -import urllib2 +import requests import time log = logging.getLogger(__name__) @@ -301,21 +301,17 @@ class RestGolangProxyIpc(object): 'ContainerIds': container_ids, } - req = urllib2.Request(self.api_url) - req.add_header('Content-Type', 'application/json') - # Sometimes it takes our poor little proxy a second or two to get # going, so if this fails, re-call ourselves with an increased timeout. try: - urllib2.urlopen(req, json.dumps(values)) - except urllib2.URLError as err: - log.debug(err) + requests.get(self.api_url, headers={'Content-Type': 'application/json'}, data=json.dumps(values)) + except requests.exceptions.ConnectionError as err: + log.exception(err) if sleep > 5: excp = "Could not contact proxy after %s seconds" % sum(range(sleep + 1)) raise Exception(excp) time.sleep(sleep) - self.handle_requests(authentication, proxy_requests, route_name, container_ids, sleep=sleep + 1) - pass + self.handle_requests(authentication, proxy_requests, route_name, container_ids, container_interface, sleep=sleep + 1) ProxyMapping = namedtuple('ProxyMapping', ['host', 'port', 'container_ids', 'container_interface']) diff --git a/lib/galaxy/webapps/config_manage.py b/lib/galaxy/webapps/config_manage.py index 4cad182179e..306cd7605f2 100644 --- a/lib/galaxy/webapps/config_manage.py +++ b/lib/galaxy/webapps/config_manage.py @@ -11,7 +11,7 @@ import string import sys import tempfile from textwrap import TextWrapper -import urllib2 +import requests import six from six import StringIO @@ -307,9 +307,8 @@ def _write_option_rst(args, rst, key, heading_level, option_value): def _build_uwsgi_schema(args, app_desc): - req = urllib2.Request('https://raw.githubusercontent.com/unbit/uwsgi-docs/master/Options.rst') - response = urllib2.urlopen(req) - rst_options = response.read() + req = requests.get('https://raw.githubusercontent.com/unbit/uwsgi-docs/master/Options.rst') + rst_options = req.text last_line = None current_opt = None diff --git a/lib/galaxy/webapps/galaxy/controllers/async.py b/lib/galaxy/webapps/galaxy/controllers/async.py index 7db4af7667f..001e832aea5 100644 --- a/lib/galaxy/webapps/galaxy/controllers/async.py +++ b/lib/galaxy/webapps/galaxy/controllers/async.py @@ -4,6 +4,7 @@ Upload class import logging import urllib +import requests from galaxy import jobs, web from galaxy.util import Params @@ -162,8 +163,7 @@ class ASync(BaseUIController): url = "%s%s%s" % (url, url_join_char, urllib.urlencode(params.flatten())) log.debug("connecting to -> %s" % url) trans.log_event("Async connecting to -> %s" % url) - text = urllib.urlopen(url).read(-1) - text = text.strip() + text = requests.get(url).text.strip() if not text.endswith('OK'): raise Exception(text) data.state = data.blurb = data.states.RUNNING diff --git a/lib/galaxy/webapps/galaxy/controllers/library_common.py b/lib/galaxy/webapps/galaxy/controllers/library_common.py index 365dd48c6b9..a35d6a261ef 100644 --- a/lib/galaxy/webapps/galaxy/controllers/library_common.py +++ b/lib/galaxy/webapps/galaxy/controllers/library_common.py @@ -1,16 +1,16 @@ import glob +import json import logging import operator import os import os.path +import requests import string import sys import tarfile import tempfile import urllib -import urllib2 import zipfile -from json import dumps, loads from markupsafe import escape from sqlalchemy import and_, false @@ -555,7 +555,7 @@ class LibraryCommon(BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMeta if len(em_string): payload = None try: - payload = loads(em_string) + payload = json.loads(em_string) except Exception: message = 'Invalid JSON input' status = 'error' @@ -1122,8 +1122,8 @@ class LibraryCommon(BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMeta json_file_path = upload_common.create_paramfile(trans, uploaded_datasets) data_list = [ud.data for ud in uploaded_datasets] job_params = {} - job_params['link_data_only'] = dumps(kwd.get('link_data_only', 'copy_files')) - job_params['uuid'] = dumps(kwd.get('uuid', None)) + job_params['link_data_only'] = json.dumps(kwd.get('link_data_only', 'copy_files')) + job_params['uuid'] = json.dumps(kwd.get('uuid', None)) job, output = upload_common.create_job(trans, tool_params, tool, json_file_path, data_list, folder=library_bunch.folder, job_params=job_params) trans.sa_session.add(job) trans.sa_session.flush() @@ -2762,8 +2762,8 @@ def lucene_search(trans, cntrller, search_term, search_url, **kwd): message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') full_url = "%s/find?%s" % (search_url, urllib.urlencode({"kwd" : search_term})) - response = urllib2.urlopen(full_url) - ldda_ids = loads(response.read())["ids"] + response = requests.get(full_url).text + ldda_ids = json.loads(response)["ids"] response.close() lddas = [trans.sa_session.query(trans.app.model.LibraryDatasetDatasetAssociation).get(ldda_id) for ldda_id in ldda_ids] return status, message, get_sorted_accessible_library_items(trans, cntrller, lddas, 'name') diff --git a/lib/galaxy/webapps/galaxy/controllers/root.py b/lib/galaxy/webapps/galaxy/controllers/root.py index 4817aee5b84..6913f705080 100644 --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -3,7 +3,7 @@ Contains the main interface in the Universe class """ import cgi import os -import urllib +import requests from paste.httpexceptions import HTTPNotFound, HTTPBadGateway @@ -470,8 +470,8 @@ class RootController(controller.JSAppLauncher, UsesAnnotations): def bucket_proxy(self, trans, bucket=None, **kwd): if bucket: trans.response.set_content_type('text/xml') - b_list_xml = urllib.urlopen('http://s3.amazonaws.com/%s/' % bucket) - return b_list_xml.read() + b_list_xml = requests.get('http://s3.amazonaws.com/%s/' % bucket) + return b_list_xml.text raise Exception("You must specify a bucket") # ---- Debug methods ---------------------------------------------------- diff --git a/lib/galaxy/webapps/galaxy/controllers/workflow.py b/lib/galaxy/webapps/galaxy/controllers/workflow.py index d5216798bed..55706557e22 100644 --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -4,7 +4,7 @@ import json import logging import os import sgmllib -import urllib2 +import requests from sqlalchemy import and_ from sqlalchemy.orm import joinedload @@ -763,7 +763,7 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi # Load workflow from external URL # NOTE: blocks the web thread. try: - workflow_data = urllib2.urlopen(url).read() + workflow_data = requests.get(url).text except Exception as e: message = "Failed to open URL: %s. Exception: %s" % (escape(url), escape(str(e))) status = 'error' diff --git a/lib/galaxy/webapps/tool_shed/controllers/upload.py b/lib/galaxy/webapps/tool_shed/controllers/upload.py index 1695cf69507..b4c4dccd8d5 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -1,9 +1,9 @@ import logging import os +import requests import shutil import tarfile import tempfile -import urllib from galaxy import util from galaxy import web @@ -74,7 +74,7 @@ class UploadController(BaseUIController): elif url: valid_url = True try: - stream = urllib.urlopen(url) + stream = requests.get(url, stream=True) except Exception as e: valid_url = False message = 'Error uploading file via http: %s' % str(e) @@ -83,11 +83,9 @@ class UploadController(BaseUIController): if valid_url: fd, uploaded_file_name = tempfile.mkstemp() uploaded_file = open(uploaded_file_name, 'wb') - while 1: - chunk = stream.read(util.CHUNK_SIZE) - if not chunk: - break - uploaded_file.write(chunk) + for chunk in stream.iter_content(chunk_size=util.CHUNK_SIZE): + if chunk: + uploaded_file.write(chunk) uploaded_file.flush() uploaded_file_filename = url.split('/')[-1] isempty = os.path.getsize(os.path.abspath(uploaded_file_name)) == 0 diff --git a/lib/tool_shed/capsule/capsule_manager.py b/lib/tool_shed/capsule/capsule_manager.py index 422d663c4a9..dcd192ce2f4 100644 --- a/lib/tool_shed/capsule/capsule_manager.py +++ b/lib/tool_shed/capsule/capsule_manager.py @@ -1,13 +1,13 @@ import contextlib import logging import os +import requests import shutil import tarfile import tempfile import threading from time import gmtime, strftime -from six.moves.urllib.request import urlopen from sqlalchemy import and_, false import tool_shed.repository_types.util as rt_util @@ -809,24 +809,20 @@ class ImportRepositoryManager(object): uploaded_file=None, capsule_file_name=None) if url: - valid_url = True try: - stream = urlopen(url) + stream = requests.get(url, stream=True) except Exception as e: - valid_url = False return_dict['error_message'] = 'Error importing file via http: %s' % str(e) return_dict['status'] = 'error' return return_dict - if valid_url: - fd, uploaded_file_name = tempfile.mkstemp() - uploaded_file = open(uploaded_file_name, 'wb') - while 1: - chunk = stream.read(CHUNK_SIZE) - if not chunk: - break + + fd, uploaded_file_name = tempfile.mkstemp() + uploaded_file = open(uploaded_file_name, 'wb') + for chunk in stream.iter_content(chunk_size=CHUNK_SIZE): + if chunk: uploaded_file.write(chunk) - uploaded_file.flush() - uploaded_file_filename = url.split('/')[-1] + uploaded_file.flush() + uploaded_file_filename = url.split('/')[-1] elif file_data not in ('', None): uploaded_file = file_data.file uploaded_file_name = uploaded_file.name diff --git a/scripts/data_libraries/build_lucene_index.py b/scripts/data_libraries/build_lucene_index.py index a53ec522163..710747068be 100644 --- a/scripts/data_libraries/build_lucene_index.py +++ b/scripts/data_libraries/build_lucene_index.py @@ -14,7 +14,7 @@ import csv import os import sys import urllib -import urllib2 +import requests sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib'))) @@ -39,9 +39,7 @@ def main(ini_file): def build_index(search_url, dataset_file): url = "%s/index?%s" % (search_url, urllib.urlencode({"docfile": dataset_file})) - request = urllib2.Request(url) - request.get_method = lambda: "PUT" - urllib2.urlopen(request) + requests.put(url) def create_dataset_file(dataset_iter): diff --git a/scripts/edam_mapping.py b/scripts/edam_mapping.py index 7858d9f5abb..7e70c5467cd 100644 --- a/scripts/edam_mapping.py +++ b/scripts/edam_mapping.py @@ -16,7 +16,7 @@ from __future__ import print_function import os import sys -import urllib2 +import requests from xml import etree sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib'))) @@ -35,7 +35,7 @@ EDAM_OWL_URL = "http://data.bioontology.org/ontologies/EDAM/submissions/25/downl if not os.path.exists("/tmp/edam.owl"): - open("/tmp/edam.owl", "w").write(urllib2.urlopen(EDAM_OWL_URL).read()) + open("/tmp/edam.owl", "w").write(requests.get(EDAM_OWL_URL).text) owl_xml_tree = etree.ElementTree.parse("/tmp/edam.owl") diff --git a/scripts/microbes/harvest_bacteria.py b/scripts/microbes/harvest_bacteria.py index 7161669b5c9..175ce70cb0d 100644 --- a/scripts/microbes/harvest_bacteria.py +++ b/scripts/microbes/harvest_bacteria.py @@ -8,7 +8,7 @@ import os import sys import time from ftplib import FTP -from urllib2 import urlopen +import requests from urllib import urlretrieve from BeautifulSoup import BeautifulSoup @@ -26,7 +26,7 @@ desired_ftp_files = {'GeneMark': {'ext': 'GeneMark-2.5f', 'parser': 'process_Gen # number, name, chroms, kingdom, group, genbank, refseq, info_url, ftp_url def iter_genome_projects(url="http://www.ncbi.nlm.nih.gov/genomes/lproks.cgi?view=1", info_url_base="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=genomeprj&cmd=Retrieve&dopt=Overview&list_uids="): - for row in BeautifulSoup(urlopen(url)).findAll(name='tr', bgcolor=["#EEFFDD", "#E8E8DD"]): + for row in BeautifulSoup(requests.get(url).text).findAll(name='tr', bgcolor=["#EEFFDD", "#E8E8DD"]): row = str(row).replace("\n", "").replace("\r", "") fields = row.split("") @@ -65,7 +65,7 @@ def get_chroms_by_project_id(org_num, base_url="http://www.ncbi.nlm.nih.gov/entr html_count += 1 url = "%s%s" % (base_url, org_num) try: - html = urlopen(url) + html = requests.get(url).text except: print "GENOME PROJECT FAILED:", html_count, "org:", org_num, url html = None diff --git a/scripts/tool_shed/api/export.py b/scripts/tool_shed/api/export.py index a198fffd966..68005de0746 100644 --- a/scripts/tool_shed/api/export.py +++ b/scripts/tool_shed/api/export.py @@ -11,7 +11,7 @@ import argparse import os import sys import tempfile -import urllib2 +import requests sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, 'lib')) from tool_shed.util import basic_util @@ -101,14 +101,11 @@ def main(options): src = None dst = None try: - src = urllib2.urlopen(download_url) - dst = open(file_path, 'wb') - while True: - chunk = src.read(CHUNK_SIZE) - if chunk: - dst.write(chunk) - else: - break + src = requests.get(download_url, stream=True) + with open(file_path, 'wb') as handle: + for chunk in src.iter_content(chunk_size=CHUNK_SIZE): + if chunk: + handle.write(chunk) except: raise finally: diff --git a/scripts/transfer.py b/scripts/transfer.py index 8bcb23f432e..3938ca48846 100644 --- a/scripts/transfer.py +++ b/scripts/transfer.py @@ -215,6 +215,7 @@ def transfer(app, transfer_job_id): def http_transfer(transfer_job): """Plugin" for handling http(s) transfers.""" url = transfer_job.params['url'] + assert url.startswith('http://') or url.startswith('https://') try: f = urllib2.urlopen(url) except urllib2.URLError as e: From db0400298651f6d4c9d909406c3fe632d057bbf7 Mon Sep 17 00:00:00 2001 From: E Rasche Date: Wed, 13 Sep 2017 07:47:54 +0000 Subject: [PATCH 44/66] remove unused imports --- cron/parse_builds.py | 1 - lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py | 1 - 2 files changed, 2 deletions(-) diff --git a/cron/parse_builds.py b/cron/parse_builds.py index f61214186b7..e728315be13 100644 --- a/cron/parse_builds.py +++ b/cron/parse_builds.py @@ -11,7 +11,6 @@ import sys import xml.etree.ElementTree as ElementTree - def getbuilds(url): try: text = requests.get(url).text diff --git a/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py b/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py index 267e0878655..0adf41e5e02 100644 --- a/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py +++ b/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py @@ -2,7 +2,6 @@ Module for managing jobs in Pacific Bioscience's SMRT Portal and automatically transferring files produced by SMRT Portal. """ -import json import logging import requests from string import Template From a68d6be3a1c6e6fd85c5ae94232467f193156d53 Mon Sep 17 00:00:00 2001 From: E Rasche Date: Wed, 13 Sep 2017 07:52:48 +0000 Subject: [PATCH 45/66] reorder imports --- test/functional/webhooks/phdcomics/helper/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/functional/webhooks/phdcomics/helper/__init__.py b/test/functional/webhooks/phdcomics/helper/__init__.py index b6828d2c21a..d3ec4d7fbe5 100644 --- a/test/functional/webhooks/phdcomics/helper/__init__.py +++ b/test/functional/webhooks/phdcomics/helper/__init__.py @@ -1,7 +1,7 @@ -import urllib -import re -import random import logging +import random +import re +import urllib log = logging.getLogger(__name__) From 93ffc4055b145811b86191dae8b865ea45b55fcb Mon Sep 17 00:00:00 2001 From: E Rasche Date: Wed, 13 Sep 2017 08:05:06 +0000 Subject: [PATCH 46/66] replace check_output with python processing and shell=False --- lib/galaxy/tools/deps/container_resolvers/mulled.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/tools/deps/container_resolvers/mulled.py b/lib/galaxy/tools/deps/container_resolvers/mulled.py index fc551cb883a..cf215a8f742 100644 --- a/lib/galaxy/tools/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tools/deps/container_resolvers/mulled.py @@ -3,6 +3,7 @@ import collections import logging import os +import subprocess import six @@ -11,7 +12,6 @@ from ..container_resolvers import ( ) from ..docker_util import build_docker_images_command from ..mulled.mulled_build import ( - check_output, DEFAULT_CHANNELS, ensure_installed, InvolucroContext, @@ -54,8 +54,8 @@ CachedV2MulledImageMultiTarget.package_hash = _package_hash def list_docker_cached_mulled_images(namespace=None, hash_func="v2"): command = build_docker_images_command(truncate=True, sudo=False) - command = "%s | tail -n +2 | tr -s ' ' | cut -d' ' -f1,2" % command - images_and_versions = check_output(command) + images_and_versions = subprocess.check_output(command).strip().split('\n') + images_and_versions = [line.split()[0:2] for line in images_and_versions[1:]] name_filter = get_filter(namespace) def output_line_to_image(line): From d79a2459a95dfda7856b9898d40d062f00456006 Mon Sep 17 00:00:00 2001 From: E Rasche Date: Wed, 13 Sep 2017 11:28:58 +0200 Subject: [PATCH 47/66] These we will revert to their original yaml.load It would be nice to fix them eventually, but we'll settle for an incremental improvement. --- lib/galaxy/tools/parser/factory.py | 2 +- lib/galaxy/webapps/config_manage.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/tools/parser/factory.py b/lib/galaxy/tools/parser/factory.py index b6655be63d5..52a11ad7b57 100644 --- a/lib/galaxy/tools/parser/factory.py +++ b/lib/galaxy/tools/parser/factory.py @@ -62,7 +62,7 @@ def ordered_load(stream): yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping) - return yaml.safe_load(stream, OrderedLoader) + return yaml.load(stream, OrderedLoader) def get_input_source(content): diff --git a/lib/galaxy/webapps/config_manage.py b/lib/galaxy/webapps/config_manage.py index 306cd7605f2..ee7b6b38069 100644 --- a/lib/galaxy/webapps/config_manage.py +++ b/lib/galaxy/webapps/config_manage.py @@ -669,7 +669,7 @@ def _ordered_load(stream): def include(self, node): filename = os.path.join(self._root, self.construct_scalar(node)) with open(filename, 'r') as f: - return yaml.safe_load(f, OrderedLoader) + return yaml.load(f, OrderedLoader) def construct_mapping(loader, node): loader.flatten_mapping(node) @@ -680,7 +680,7 @@ def _ordered_load(stream): construct_mapping) OrderedLoader.add_constructor('!include', OrderedLoader.include) - return yaml.safe_load(stream, OrderedLoader) + return yaml.load(stream, OrderedLoader) def _ordered_dump(data, stream=None, Dumper=yaml.Dumper, **kwds): From b3261309843f8cfec1a5a589b4628e44e6bf76df Mon Sep 17 00:00:00 2001 From: E Rasche Date: Wed, 13 Sep 2017 11:32:53 +0200 Subject: [PATCH 48/66] import ordering --- cron/parse_builds.py | 2 +- cron/parse_builds_3_sites.py | 3 ++- lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py | 3 ++- lib/galaxy/tools/data/__init__.py | 3 ++- lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py | 3 ++- lib/tool_shed/capsule/capsule_manager.py | 3 ++- 6 files changed, 11 insertions(+), 6 deletions(-) diff --git a/cron/parse_builds.py b/cron/parse_builds.py index e728315be13..2e4149fce57 100644 --- a/cron/parse_builds.py +++ b/cron/parse_builds.py @@ -6,10 +6,10 @@ build description """ from __future__ import print_function -import requests import sys import xml.etree.ElementTree as ElementTree +import requests def getbuilds(url): try: diff --git a/cron/parse_builds_3_sites.py b/cron/parse_builds_3_sites.py index 464064f4736..7d3af104420 100644 --- a/cron/parse_builds_3_sites.py +++ b/cron/parse_builds_3_sites.py @@ -4,9 +4,10 @@ Connects to sites and determines which builds are available at each. """ from __future__ import print_function -import requests import xml.etree.ElementTree as ElementTree +import requests + sites = ['http://genome.ucsc.edu/cgi-bin/', 'http://archaea.ucsc.edu/cgi-bin/', 'http://genome-test.cse.ucsc.edu/cgi-bin/'] diff --git a/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py b/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py index 0adf41e5e02..297b56e90b9 100644 --- a/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py +++ b/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py @@ -3,11 +3,12 @@ Module for managing jobs in Pacific Bioscience's SMRT Portal and automatically t produced by SMRT Portal. """ import logging -import requests from string import Template from .data_transfer import DataTransfer +import requests + log = logging.getLogger(__name__) __all__ = ('SMRTPortalPlugin', ) diff --git a/lib/galaxy/tools/data/__init__.py b/lib/galaxy/tools/data/__init__.py index 98013c5201e..f1fde0b7f40 100644 --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -11,7 +11,6 @@ import logging import os import os.path import re -import requests import string import time from glob import glob @@ -21,6 +20,8 @@ from galaxy import util from galaxy.util.dictifiable import Dictifiable from galaxy.util.odict import odict +import requests + log = logging.getLogger(__name__) DEFAULT_TABLE_TYPE = 'tabular' diff --git a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py index 31e5c922a6d..c8562f959d4 100644 --- a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py +++ b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py @@ -10,12 +10,13 @@ from __future__ import print_function import math import optparse import os -import requests import sys import tarfile import tempfile from base64 import b64decode +import requests + # Set max size of archive/file that will be handled to be 100 GB. This is # arbitrary and should be adjusted as needed. MAX_SIZE = 100 * math.pow(2, 30) diff --git a/lib/tool_shed/capsule/capsule_manager.py b/lib/tool_shed/capsule/capsule_manager.py index dcd192ce2f4..c07d1d5de95 100644 --- a/lib/tool_shed/capsule/capsule_manager.py +++ b/lib/tool_shed/capsule/capsule_manager.py @@ -1,7 +1,6 @@ import contextlib import logging import os -import requests import shutil import tarfile import tempfile @@ -21,6 +20,8 @@ from tool_shed.metadata import repository_metadata_manager from tool_shed.util import (basic_util, commit_util, common_util, encoding_util, hg_util, metadata_util, repository_util, shed_util_common as suc, xml_util) +import requests + log = logging.getLogger(__name__) From 8f4491bfffce5d13e2199df5d60e06e3672a08c1 Mon Sep 17 00:00:00 2001 From: E Rasche Date: Wed, 13 Sep 2017 11:57:17 +0200 Subject: [PATCH 49/66] Consume stderr --- lib/galaxy/datatypes/binary.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 0f8184cd1d1..924699a13a6 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -254,7 +254,7 @@ class Bam(Binary): # Get the version of samtools via --version-only, if available try: - output = subprocess.check_output(['samtools', '--version-only']) + output = subprocess.check_output(['samtools', '--version-only'], stderr=subprocess.PIPE) # --version-only is available # Format is +htslib- version = output.split('+')[0] @@ -263,7 +263,7 @@ class Bam(Binary): # --version-only not available pass - output = subprocess.check_output(['samtools']) + output = subprocess.check_output(['samtools'], stderr=subprocess.PIPE) lines = output.split('\n') for line in lines: if line.lower().startswith('version'): From 0f7dd52988269dc4567bdbbd71be0fd273741efd Mon Sep 17 00:00:00 2001 From: E Rasche Date: Wed, 13 Sep 2017 11:57:50 +0200 Subject: [PATCH 50/66] Copy changes over --- lib/galaxy/datatypes/converters/sam_to_bam.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/datatypes/converters/sam_to_bam.py b/lib/galaxy/datatypes/converters/sam_to_bam.py index dcc84b6c518..011f660b922 100644 --- a/lib/galaxy/datatypes/converters/sam_to_bam.py +++ b/lib/galaxy/datatypes/converters/sam_to_bam.py @@ -34,18 +34,17 @@ def _get_samtools_version(): if not cmd_exists('samtools'): raise Exception('This tool needs samtools, but it is not on PATH.') # Get the version of samtools via --version-only, if available - p = subprocess.Popen(['samtools', '--version-only'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - output, error = p.communicate() - - # --version-only is available - # Format is +htslib- - if p.returncode == 0: + try: + output = subprocess.check_output(['samtools', '--version-only'], stderr=subprocess.PIPE) + # --version-only is available + # Format is +htslib- version = output.split('+')[0] return version + except subprocess.CalledProcessError: + # --version-only not available + pass - output = subprocess.Popen(['samtools'], stderr=subprocess.PIPE, stdout=subprocess.PIPE).communicate()[1] + output = subprocess.check_output(['samtools'], stderr=subprocess.PIPE) lines = output.split('\n') for line in lines: if line.lower().startswith('version'): From 75df58ae86ff1796f0bb0ea975fc1d0d518c155a Mon Sep 17 00:00:00 2001 From: E Rasche Date: Thu, 14 Sep 2017 17:23:33 +0200 Subject: [PATCH 51/66] fix --- cron/parse_builds_3_sites.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cron/parse_builds_3_sites.py b/cron/parse_builds_3_sites.py index 7d3af104420..4ae43e52765 100644 --- a/cron/parse_builds_3_sites.py +++ b/cron/parse_builds_3_sites.py @@ -20,11 +20,11 @@ def main(): trackurl = sites[i] + "hgTracks?" builds = [] try: - page = requests.get(site) + text = requests.get(site).text except: print("#Unable to connect to " + site) continue - text = page.text + try: tree = ElementTree.fromstring(text) except: From fd92ac3eedd145635a74530ba62bb7fea7c540f4 Mon Sep 17 00:00:00 2001 From: E Rasche Date: Thu, 14 Sep 2017 17:23:40 +0200 Subject: [PATCH 52/66] revert this change --- lib/galaxy/datatypes/binary.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 924699a13a6..1cb5eb73686 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -252,18 +252,15 @@ class Bam(Binary): message = 'Attempting to use functionality requiring samtools, but it cannot be located on Galaxy\'s PATH.' raise Exception(message) - # Get the version of samtools via --version-only, if available - try: - output = subprocess.check_output(['samtools', '--version-only'], stderr=subprocess.PIPE) - # --version-only is available - # Format is +htslib- + p = subprocess.Popen(['samtools', '--version-only'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, error = p.communicate() + # --version-only is available + # Format is +htslib- + if p.returncode == 0: version = output.split('+')[0] return version - except subprocess.CalledProcessError: - # --version-only not available - pass - output = subprocess.check_output(['samtools'], stderr=subprocess.PIPE) + output = subprocess.Popen(['samtools'], stderr=subprocess.PIPE, stdout=subprocess.PIPE).communicate()[1] lines = output.split('\n') for line in lines: if line.lower().startswith('version'): From 27beb7030d99a511a428ad59211a3137a2761559 Mon Sep 17 00:00:00 2001 From: E Rasche Date: Thu, 14 Sep 2017 17:25:40 +0200 Subject: [PATCH 53/66] revert/sync --- lib/galaxy/datatypes/binary.py | 2 +- lib/galaxy/datatypes/converters/sam_to_bam.py | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 1cb5eb73686..87cf9aeee92 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -1302,7 +1302,7 @@ class ExcelXls(Binary): edam_format = "format_3468" def sniff(self, filename): - mime_type = subprocess.check_output(['file', '--mime-type', filename]).strip() + mime_type = subprocess.check_output(['file', '--mime-type', filename]) return "application/vnd.ms-excel" in mime_type def get_mime(self): diff --git a/lib/galaxy/datatypes/converters/sam_to_bam.py b/lib/galaxy/datatypes/converters/sam_to_bam.py index 011f660b922..8c54258eb9a 100644 --- a/lib/galaxy/datatypes/converters/sam_to_bam.py +++ b/lib/galaxy/datatypes/converters/sam_to_bam.py @@ -34,17 +34,15 @@ def _get_samtools_version(): if not cmd_exists('samtools'): raise Exception('This tool needs samtools, but it is not on PATH.') # Get the version of samtools via --version-only, if available - try: - output = subprocess.check_output(['samtools', '--version-only'], stderr=subprocess.PIPE) - # --version-only is available - # Format is +htslib- + p = subprocess.Popen(['samtools', '--version-only'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, error = p.communicate() + # --version-only is available + # Format is +htslib- + if p.returncode == 0: version = output.split('+')[0] return version - except subprocess.CalledProcessError: - # --version-only not available - pass - output = subprocess.check_output(['samtools'], stderr=subprocess.PIPE) + output = subprocess.Popen(['samtools'], stderr=subprocess.PIPE, stdout=subprocess.PIPE).communicate()[1] lines = output.split('\n') for line in lines: if line.lower().startswith('version'): From 1a57c06d6a81dc6f92eb4843c89a7333f874bc32 Mon Sep 17 00:00:00 2001 From: E Rasche Date: Thu, 14 Sep 2017 17:54:32 +0200 Subject: [PATCH 54/66] Fix import ordering Thanks @nsoranzo, I appreciate all the comments here. --- lib/galaxy/datatypes/tabular.py | 2 +- lib/galaxy/datatypes/text.py | 5 +++-- lib/galaxy/external_services/actions.py | 2 +- lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py | 4 ++-- lib/galaxy/jobs/transfer_manager.py | 6 ++++-- lib/galaxy/objectstore/s3.py | 6 +++--- lib/galaxy/tools/data/__init__.py | 4 ++-- lib/galaxy/web/proxy/__init__.py | 3 ++- lib/galaxy/webapps/config_manage.py | 2 +- lib/galaxy/webapps/galaxy/controllers/async.py | 1 + lib/galaxy/webapps/galaxy/controllers/library_common.py | 6 ++---- lib/galaxy/webapps/galaxy/controllers/root.py | 2 +- lib/galaxy/webapps/tool_shed/controllers/upload.py | 3 ++- lib/tool_shed/capsule/capsule_manager.py | 3 +-- 14 files changed, 26 insertions(+), 23 deletions(-) diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 9a6f0eb2cb5..234a436a648 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -524,7 +524,7 @@ class Sam(Tabular): shutil.move(split_files[0], output_file) if len(split_files) > 1: - cmd = ['egrep', '-v', '-h' '^@'] + split_files[1:] + ['>>', output_file] + cmd = ['egrep', '-v', '-h', '^@'] + split_files[1:] + ['>>', output_file] subprocess.check_call(cmd, shell=True) merge = staticmethod(merge) diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index c201b872b1f..3b72e3d48b0 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -10,6 +10,8 @@ import re import subprocess import tempfile +from six.moves import shlex_quote + from galaxy.datatypes.data import get_file_peek, Text from galaxy.datatypes.metadata import MetadataElement, MetadataParameter from galaxy.datatypes.sniff import iter_headers @@ -149,12 +151,11 @@ class Ipynb(Json): ofile_handle.close() try: cmd = ['jupyter', 'nbconvert', '--to', 'html', '--template', 'full', dataset.file_name, '--output', ofilename] - log.info("Calling command %s", ' '.join(cmd)) subprocess.check_call(cmd) ofilename = '%s.html' % ofilename except subprocess.CalledProcessError: ofilename = dataset.file_name - log.exception('Command "%s" failed. Could not convert the Jupyter Notebook to HTML, defaulting to plain text.', cmd) + log.exception('Command "%s" failed. Could not convert the Jupyter Notebook to HTML, defaulting to plain text.', map(shlex_quote, cmd)) return open(ofilename) def set_meta(self, dataset, **kwd): diff --git a/lib/galaxy/external_services/actions.py b/lib/galaxy/external_services/actions.py index 71520265c49..13dc83703d3 100644 --- a/lib/galaxy/external_services/actions.py +++ b/lib/galaxy/external_services/actions.py @@ -104,7 +104,7 @@ class ExternalServiceWebAPIActionResult(ExternalServiceResult): @property def content(self): if self._content is None: - self._content = requests.get(self.url).read() + self._content = requests.get(self.url).text return self._content diff --git a/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py b/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py index 297b56e90b9..b90f9940bcd 100644 --- a/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py +++ b/lib/galaxy/jobs/deferred/pacific_biosciences_smrt_portal.py @@ -5,10 +5,10 @@ produced by SMRT Portal. import logging from string import Template -from .data_transfer import DataTransfer - import requests +from .data_transfer import DataTransfer + log = logging.getLogger(__name__) __all__ = ('SMRTPortalPlugin', ) diff --git a/lib/galaxy/jobs/transfer_manager.py b/lib/galaxy/jobs/transfer_manager.py index 8f3cd4539ee..6bca709af3b 100644 --- a/lib/galaxy/jobs/transfer_manager.py +++ b/lib/galaxy/jobs/transfer_manager.py @@ -9,6 +9,8 @@ import socket import subprocess import threading +from six.moves import shlex_quote + from galaxy.util import listify, sleeper from galaxy.util.json import jsonrpc_request, validate_jsonrpc_response @@ -68,8 +70,8 @@ class TransferManager(object): # The transfer script should daemonize fairly quickly - if this is # not the case, this process will need to be moved to a # non-blocking method. - cmd = self.command + [tj.id] - log.debug('Transfer command is: %s', ' '.join(cmd)) + cmd = self.command.append(tj.id) + log.debug('Transfer command is: %s', ' '.join(map(shlex_quote, cmd))) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) p.wait() output = p.stdout.read(32768) diff --git a/lib/galaxy/objectstore/s3.py b/lib/galaxy/objectstore/s3.py index 0d8297316e8..34a2fc39d74 100644 --- a/lib/galaxy/objectstore/s3.py +++ b/lib/galaxy/objectstore/s3.py @@ -18,6 +18,7 @@ from galaxy.util import ( safe_relpath, string_as_bool, umask_fix_perms, + which, ) from galaxy.util.sleeper import Sleeper @@ -67,10 +68,9 @@ class S3ObjectStore(ObjectStore): self.cache_monitor_thread.start() log.info("Cache cleaner manager started") # Test if 'axel' is available for parallel download and pull the key into cache - try: - subprocess.check_call(['axel']) + if which('axel'): self.use_axel = True - except OSError: + else: self.use_axel = False def _configure_connection(self): diff --git a/lib/galaxy/tools/data/__init__.py b/lib/galaxy/tools/data/__init__.py index f1fde0b7f40..d3ec66072b7 100644 --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -16,12 +16,12 @@ import time from glob import glob from tempfile import NamedTemporaryFile +import requests + from galaxy import util from galaxy.util.dictifiable import Dictifiable from galaxy.util.odict import odict -import requests - log = logging.getLogger(__name__) DEFAULT_TABLE_TYPE = 'tabular' diff --git a/lib/galaxy/web/proxy/__init__.py b/lib/galaxy/web/proxy/__init__.py index fcb23ba56c0..581050a3d5e 100644 --- a/lib/galaxy/web/proxy/__init__.py +++ b/lib/galaxy/web/proxy/__init__.py @@ -8,9 +8,10 @@ from galaxy.util import sockets from galaxy.util.lazy_process import LazyProcess, NoOpLazyProcess from galaxy.util import sqlite from galaxy.util import unique_id -import requests import time +import requests + log = logging.getLogger(__name__) diff --git a/lib/galaxy/webapps/config_manage.py b/lib/galaxy/webapps/config_manage.py index ee7b6b38069..685b5abcf56 100644 --- a/lib/galaxy/webapps/config_manage.py +++ b/lib/galaxy/webapps/config_manage.py @@ -11,8 +11,8 @@ import string import sys import tempfile from textwrap import TextWrapper -import requests +import requests import six from six import StringIO diff --git a/lib/galaxy/webapps/galaxy/controllers/async.py b/lib/galaxy/webapps/galaxy/controllers/async.py index 001e832aea5..aabb551bdfc 100644 --- a/lib/galaxy/webapps/galaxy/controllers/async.py +++ b/lib/galaxy/webapps/galaxy/controllers/async.py @@ -4,6 +4,7 @@ Upload class import logging import urllib + import requests from galaxy import jobs, web diff --git a/lib/galaxy/webapps/galaxy/controllers/library_common.py b/lib/galaxy/webapps/galaxy/controllers/library_common.py index a35d6a261ef..2a1ac0a09dc 100644 --- a/lib/galaxy/webapps/galaxy/controllers/library_common.py +++ b/lib/galaxy/webapps/galaxy/controllers/library_common.py @@ -4,7 +4,6 @@ import logging import operator import os import os.path -import requests import string import sys import tarfile @@ -12,6 +11,7 @@ import tempfile import urllib import zipfile +import requests from markupsafe import escape from sqlalchemy import and_, false from sqlalchemy.orm import eagerload_all @@ -2762,9 +2762,7 @@ def lucene_search(trans, cntrller, search_term, search_url, **kwd): message = escape(kwd.get('message', '')) status = kwd.get('status', 'done') full_url = "%s/find?%s" % (search_url, urllib.urlencode({"kwd" : search_term})) - response = requests.get(full_url).text - ldda_ids = json.loads(response)["ids"] - response.close() + ldda_ids = requests.get(full_url).json()['ids'] lddas = [trans.sa_session.query(trans.app.model.LibraryDatasetDatasetAssociation).get(ldda_id) for ldda_id in ldda_ids] return status, message, get_sorted_accessible_library_items(trans, cntrller, lddas, 'name') diff --git a/lib/galaxy/webapps/galaxy/controllers/root.py b/lib/galaxy/webapps/galaxy/controllers/root.py index 6913f705080..a8d2df5f953 100644 --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -3,8 +3,8 @@ Contains the main interface in the Universe class """ import cgi import os -import requests +import requests from paste.httpexceptions import HTTPNotFound, HTTPBadGateway from galaxy import web diff --git a/lib/galaxy/webapps/tool_shed/controllers/upload.py b/lib/galaxy/webapps/tool_shed/controllers/upload.py index b4c4dccd8d5..2fa3d3be406 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -1,10 +1,11 @@ import logging import os -import requests import shutil import tarfile import tempfile +import requests + from galaxy import util from galaxy import web from galaxy.util import checkers diff --git a/lib/tool_shed/capsule/capsule_manager.py b/lib/tool_shed/capsule/capsule_manager.py index c07d1d5de95..5daa0b21c04 100644 --- a/lib/tool_shed/capsule/capsule_manager.py +++ b/lib/tool_shed/capsule/capsule_manager.py @@ -7,6 +7,7 @@ import tempfile import threading from time import gmtime, strftime +import requests from sqlalchemy import and_, false import tool_shed.repository_types.util as rt_util @@ -20,8 +21,6 @@ from tool_shed.metadata import repository_metadata_manager from tool_shed.util import (basic_util, commit_util, common_util, encoding_util, hg_util, metadata_util, repository_util, shed_util_common as suc, xml_util) -import requests - log = logging.getLogger(__name__) From 290971ca7a28eb579dc2d40c9653b9e160253634 Mon Sep 17 00:00:00 2001 From: E Rasche Date: Mon, 18 Sep 2017 10:32:49 +0200 Subject: [PATCH 55/66] more review comments --- cron/build_chrom_db.py | 2 +- cron/parse_builds.py | 1 + lib/galaxy/datatypes/text.py | 2 +- lib/galaxy/jobs/transfer_manager.py | 2 +- scripts/data_libraries/build_lucene_index.py | 1 + scripts/edam_mapping.py | 1 + scripts/microbes/harvest_bacteria.py | 2 +- scripts/tool_shed/api/export.py | 21 ++++++-------------- 8 files changed, 13 insertions(+), 19 deletions(-) diff --git a/cron/build_chrom_db.py b/cron/build_chrom_db.py index d3cba98b8dc..4775897548f 100644 --- a/cron/build_chrom_db.py +++ b/cron/build_chrom_db.py @@ -15,9 +15,9 @@ from __future__ import print_function import fileinput import os -import requests import sys +import requests from six.moves.urllib.parse import urlencode import parse_builds diff --git a/cron/parse_builds.py b/cron/parse_builds.py index 2e4149fce57..243440a331b 100644 --- a/cron/parse_builds.py +++ b/cron/parse_builds.py @@ -11,6 +11,7 @@ import xml.etree.ElementTree as ElementTree import requests + def getbuilds(url): try: text = requests.get(url).text diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index 3b72e3d48b0..12235e566c4 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -155,7 +155,7 @@ class Ipynb(Json): ofilename = '%s.html' % ofilename except subprocess.CalledProcessError: ofilename = dataset.file_name - log.exception('Command "%s" failed. Could not convert the Jupyter Notebook to HTML, defaulting to plain text.', map(shlex_quote, cmd)) + log.exception('Command "%s" failed. Could not convert the Jupyter Notebook to HTML, defaulting to plain text.', ' '.join(map(shlex_quote, cmd))) return open(ofilename) def set_meta(self, dataset, **kwd): diff --git a/lib/galaxy/jobs/transfer_manager.py b/lib/galaxy/jobs/transfer_manager.py index 6bca709af3b..8e06024832c 100644 --- a/lib/galaxy/jobs/transfer_manager.py +++ b/lib/galaxy/jobs/transfer_manager.py @@ -70,7 +70,7 @@ class TransferManager(object): # The transfer script should daemonize fairly quickly - if this is # not the case, this process will need to be moved to a # non-blocking method. - cmd = self.command.append(tj.id) + cmd = self.command + [tj.id] log.debug('Transfer command is: %s', ' '.join(map(shlex_quote, cmd))) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) p.wait() diff --git a/scripts/data_libraries/build_lucene_index.py b/scripts/data_libraries/build_lucene_index.py index 710747068be..b3f787e8b02 100644 --- a/scripts/data_libraries/build_lucene_index.py +++ b/scripts/data_libraries/build_lucene_index.py @@ -14,6 +14,7 @@ import csv import os import sys import urllib + import requests sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib'))) diff --git a/scripts/edam_mapping.py b/scripts/edam_mapping.py index 7e70c5467cd..bb5e495aa46 100644 --- a/scripts/edam_mapping.py +++ b/scripts/edam_mapping.py @@ -17,6 +17,7 @@ from __future__ import print_function import os import sys import requests + from xml import etree sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib'))) diff --git a/scripts/microbes/harvest_bacteria.py b/scripts/microbes/harvest_bacteria.py index 175ce70cb0d..3226dec78ce 100644 --- a/scripts/microbes/harvest_bacteria.py +++ b/scripts/microbes/harvest_bacteria.py @@ -8,9 +8,9 @@ import os import sys import time from ftplib import FTP -import requests from urllib import urlretrieve +import requests from BeautifulSoup import BeautifulSoup from util import get_bed_from_genbank, get_bed_from_glimmer3, get_bed_from_GeneMarkHMM, get_bed_from_GeneMark diff --git a/scripts/tool_shed/api/export.py b/scripts/tool_shed/api/export.py index 68005de0746..6a5d030c6a0 100644 --- a/scripts/tool_shed/api/export.py +++ b/scripts/tool_shed/api/export.py @@ -11,6 +11,7 @@ import argparse import os import sys import tempfile + import requests sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, 'lib')) @@ -98,21 +99,11 @@ def main(options): download_url = export_dict['download_url'] download_dir = os.path.abspath(options.download_dir) file_path = os.path.join(download_dir, repositories_archive_filename) - src = None - dst = None - try: - src = requests.get(download_url, stream=True) - with open(file_path, 'wb') as handle: - for chunk in src.iter_content(chunk_size=CHUNK_SIZE): - if chunk: - handle.write(chunk) - except: - raise - finally: - if src: - src.close() - if dst: - dst.close() + src = requests.get(download_url, stream=True) + with open(file_path, 'wb') as dst: + for chunk in src.iter_content(chunk_size=CHUNK_SIZE): + if chunk: + dst.write(chunk) print "Successfully exported revision ", options.changeset_revision, " of repository ", options.name, " owned by ", options.owner print "to location ", file_path else: From 15c73233c52b29ed94066c06ebb1e10c935ae28d Mon Sep 17 00:00:00 2001 From: E Rasche Date: Mon, 18 Sep 2017 11:32:02 +0200 Subject: [PATCH 56/66] fix lint --- scripts/edam_mapping.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/edam_mapping.py b/scripts/edam_mapping.py index bb5e495aa46..f483db8c1f5 100644 --- a/scripts/edam_mapping.py +++ b/scripts/edam_mapping.py @@ -16,10 +16,10 @@ from __future__ import print_function import os import sys -import requests - from xml import etree +import requests + sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib'))) import galaxy.model From 19d03edc7ff8384a5360d71365076a2594fb0b8b Mon Sep 17 00:00:00 2001 From: E Rasche Date: Mon, 18 Sep 2017 11:33:53 +0200 Subject: [PATCH 57/66] refactor missed call --- lib/galaxy/tools/deps/mulled/mulled_build_channel.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tools/deps/mulled/mulled_build_channel.py b/lib/galaxy/tools/deps/mulled/mulled_build_channel.py index ff87189b65b..8338fb51053 100644 --- a/lib/galaxy/tools/deps/mulled/mulled_build_channel.py +++ b/lib/galaxy/tools/deps/mulled/mulled_build_channel.py @@ -21,13 +21,13 @@ from __future__ import print_function import os import sys import time +import subprocess from ._cli import arg_parser from .mulled_build import ( add_build_arguments, args_to_mull_targets_kwds, build_target, - check_output, conda_versions, get_affected_packages, mull_targets, @@ -42,7 +42,13 @@ def _fetch_repo_data(args): repo_data = "%s-repodata.json" % channel if not os.path.exists(repo_data): platform_tag = 'osx-64' if sys.platform == 'darwin' else 'linux-64' - check_output("wget --quiet https://conda.anaconda.org/%s/%s/repodata.json.bz2 -O '%s.bz2' && bzip2 -d '%s.bz2'" % (channel, platform_tag, repo_data, repo_data)) + subprocess.check_call([ + 'wget', '--quiet', 'https://conda.anaconda.org/%s/%s/repodata.json.bz2' % (channel, platform_tag), + '-O', '%s.bz2' % repo_data + ]) + subprocess.check_call([ + 'bzip2', '-d', '%s.bz2' % repo_data + ]) return repo_data From 626a3dc2995f0d797232aa666f1bdf3e6a5d22dd Mon Sep 17 00:00:00 2001 From: E Rasche Date: Mon, 18 Sep 2017 13:16:06 +0200 Subject: [PATCH 58/66] fix ordering --- lib/galaxy/tools/deps/mulled/mulled_build_channel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/deps/mulled/mulled_build_channel.py b/lib/galaxy/tools/deps/mulled/mulled_build_channel.py index 8338fb51053..06e0c7afe6f 100644 --- a/lib/galaxy/tools/deps/mulled/mulled_build_channel.py +++ b/lib/galaxy/tools/deps/mulled/mulled_build_channel.py @@ -19,9 +19,9 @@ See recent changes that would be built with: from __future__ import print_function import os +import subprocess import sys import time -import subprocess from ._cli import arg_parser from .mulled_build import ( From 9f4c72a345ff629ea4b1430026d15decddf38617 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 25 Sep 2017 11:33:20 +0100 Subject: [PATCH 59/66] Un-pin flake8 deps for dev branch --- tox.ini | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tox.ini b/tox.ini index 593185b8a7e..f9f44eaa08a 100644 --- a/tox.ini +++ b/tox.ini @@ -6,8 +6,8 @@ skipsdist = True commands = bash .ci/flake8_wrapper.sh whitelist_externals = bash deps = - flake8==3.4.1 - flake8-docstrings==1.1.0 + flake8 + flake8-docstrings>=1.1.0 [testenv:py33-lint] commands = bash .ci/flake8_py3_wrapper.sh @@ -17,7 +17,7 @@ deps = flake8 [testenv:py34-lint] commands = bash .ci/flake8_py3_wrapper.sh whitelist_externals = bash -deps = flake8==3.4.1 +deps = flake8 [testenv:py35-lint] commands = bash .ci/flake8_py3_wrapper.sh @@ -44,15 +44,15 @@ whitelist_externals = bash skip_install = True deps = flake8 - flake8-import-order==0.13 + flake8-import-order>=0.9 [testenv:py27-lint-imports-include-list] commands = bash .ci/flake8_wrapper_imports.sh whitelist_externals = bash skip_install = True deps = - flake8==3.4.1 - flake8-import-order==0.13 + flake8 + flake8-import-order>=0.9 [testenv:qunit] commands = bash run_tests.sh -q @@ -80,7 +80,7 @@ whitelist_externals = bash skip_install = True deps = flake8 - flake8-docstrings==1.1.0 + flake8-docstrings>=1.1.0 [testenv:py27-lint-docstring-include-list] @@ -88,5 +88,5 @@ commands = bash .ci/flake8_wrapper_docstrings.sh --include whitelist_externals = bash skip_install = True deps = - flake8==3.4.1 - flake8-docstrings==1.1.0 + flake8 + flake8-docstrings>=1.1.0 From caed7b60aa98153049fa2e26f5855833e7cdfce0 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 25 Sep 2017 12:18:24 +0100 Subject: [PATCH 60/66] Use only latest publication --- CITATION | 69 ++++++++++++-------------------------------------------- 1 file changed, 14 insertions(+), 55 deletions(-) diff --git a/CITATION b/CITATION index 49be43c17d1..2142df9d514 100644 --- a/CITATION +++ b/CITATION @@ -1,72 +1,31 @@ -If you use or extend Galaxy in your published work, please cite each of the -following publications: +If you use or extend Galaxy in your published work, please cite this +publication: - Afgan E, Baker D, van den Beek M, Blankenberg D, Bouvier D, Čech M, Chilton J, Clements D, Coraor N, Eberhard C, Grüning B, Guerler A, Hillman-Jackson - J,Von Kuster G, Rasche E, Soranzo N, Turaga N, Taylor J, Nekrutenko A, Goecks + J, Von Kuster G, Rasche E, Soranzo N, Turaga N, Taylor J, Nekrutenko A, Goecks J. "The Galaxy platform for accessible, reproducible and collaborative - biomedical analyses: 2016 update." - Nucleic acids research, 44(W1), W3-W10. - -- Goecks, J, Nekrutenko, A, Taylor, J and The Galaxy Team. "Galaxy: a - comprehensive approach for supporting accessible, reproducible, and - transparent computational research in the life sciences." - Genome Biol. 2010 Aug 25;11(8):R86. - -- Blankenberg D, Von Kuster G, Coraor N, Ananda G, Lazarus R, Mangan M, - Nekrutenko A, Taylor J. "Galaxy: a web-based genome analysis tool for - experimentalists". Current Protocols in Molecular Biology. - 2010 Jan; Chapter 19:Unit 19.10.1-21. - -- Giardine B, Riemer C, Hardison RC, Burhans R, Elnitski L, Shah P, Zhang Y, - Blankenberg D, Albert I, Taylor J, Miller W, Kent WJ, Nekrutenko A. "Galaxy: - a platform for interactive large-scale genome analysis." - Genome Research. 2005 Oct; 15(10):1451-5. - -See also: https://galaxyproject.org/citing-galaxy - - + biomedical analyses: 2016 update" + Nucleic Acids Research (2016) 44(W1):W3-W10 doi:10.1093/nar/gkw343 BibTeX format: @article{afgan2016galaxy, - title={The Galaxy platform for accessible, reproducible and collaborative biomedical analyses: 2016 update}, - author={Afgan, Enis and Baker, Dannon and Van den Beek, Marius and Blankenberg, Daniel and Bouvier, Dave and Čech, Martin and Chilton, John and Clements, Dave and Coraor, Nate and Eberhard, Carl and others}, - journal={Nucleic acids research}, + title={The {Galaxy} platform for accessible, reproducible and collaborative biomedical analyses: 2016 update}, + author={Afgan, Enis and Baker, Dannon and van den Beek, Marius and Blankenberg, Daniel and Bouvier, Dave and \v{C}ech, Martin and Chilton, John and Clements, Dave and Coraor, Nate and Eberhard, Carl and Gr\"uning, Bj\"orn and Guerler, Aysam and Hillman-Jackson, Jennifer and Von Kuster, Greg and Rasche, Eric and Soranzo, Nicola and Turaga, Nitesh and Taylor, James and Nekrutenko, Anton and Goecks, Jeremy}, + journal={Nucleic Acids Res.}, volume={44}, number={W1}, - pages={W3--W10}, + pages={W3-W10}, year={2016}, publisher={Oxford University Press} } -@article{goecks2010galaxy, - title={Galaxy: a comprehensive approach for supporting accessible, reproducible, and transparent computational research in the life sciences}, - author={Goecks, Jeremy and Nekrutenko, Anton and Taylor, James and The Galaxy Team}, - journal={Genome Biol}, - volume={11}, - number={8}, - pages={R86}, - year={2010} -} -@article{blankenberg2010galaxy, - title={Galaxy: A Web-Based Genome Analysis Tool for Experimentalists}, - author={Blankenberg, Daniel and Kuster, Gregory Von and Coraor, Nathaniel and Ananda, Guruprasad and Lazarus, Ross and Mangan, Mary and Nekrutenko, Anton and Taylor, James}, - journal={Current protocols in molecular biology}, - pages={19--10}, - year={2010}, - publisher={John Wiley \& Sons, Inc.} -} +More publications +----------------- -@article{giardine2005galaxy, - title={Galaxy: a platform for interactive large-scale genome analysis}, - author={Giardine, Belinda and Riemer, Cathy and Hardison, Ross C and Burhans, Richard and Elnitski, Laura and Shah, Prachi and Zhang, Yi and Blankenberg, Daniel and Albert, Istvan and Taylor, James and Miller, Webb C and Kent, W James and Nekrutenko, Anton}, - journal={Genome research}, - volume={15}, - number={10}, - pages={1451--1455}, - year={2005}, - publisher={Cold Spring Harbor Lab} -} +To cite specific Galaxy components/features or other Galaxy project +publications, see +https://galaxyproject.org/citing-galaxy From 4311f5574dbf82574599b4947cc2721a1132675e Mon Sep 17 00:00:00 2001 From: E Rasche Date: Mon, 25 Sep 2017 17:16:24 +0200 Subject: [PATCH 61/66] nicer logging for sentry --- lib/galaxy/visualization/plugins/interactive_environments.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/visualization/plugins/interactive_environments.py b/lib/galaxy/visualization/plugins/interactive_environments.py index 161882e9409..6410b03ec08 100644 --- a/lib/galaxy/visualization/plugins/interactive_environments.py +++ b/lib/galaxy/visualization/plugins/interactive_environments.py @@ -373,7 +373,7 @@ class InteractiveEnvironmentRequest(object): p = Popen(raw_cmd, stdout=PIPE, stderr=PIPE, close_fds=True, shell=True) stdout, stderr = p.communicate() if p.returncode != 0: - log.error("%s\n%s" % (stdout, stderr)) + log.error("Container Launch error\n\n%s\n%s" % (stdout, stderr)) return None else: container_id = stdout.strip() @@ -484,7 +484,7 @@ class InteractiveEnvironmentRequest(object): p = Popen(command, stdout=PIPE, stderr=PIPE, close_fds=True, shell=True) stdout, stderr = p.communicate() if p.returncode != 0: - log.error("%s\n%s" % (stdout, stderr)) + log.error("Container Launch error\n\n%s\n%s" % (stdout, stderr)) return None inspect_data = json.loads(stdout) From 41574d022249bae4df4b89bfb50c3a599ee5585e Mon Sep 17 00:00:00 2001 From: E Rasche Date: Mon, 25 Sep 2017 17:32:05 +0200 Subject: [PATCH 62/66] swap for subprocess.call per @nsoranzo suggestion --- lib/galaxy/jobs/runners/util/job_script/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/jobs/runners/util/job_script/__init__.py b/lib/galaxy/jobs/runners/util/job_script/__init__.py index 4c4c6b86e45..2c717c8f13d 100644 --- a/lib/galaxy/jobs/runners/util/job_script/__init__.py +++ b/lib/galaxy/jobs/runners/util/job_script/__init__.py @@ -119,9 +119,8 @@ def _handle_script_integrity(path, config): sleep_amt = getattr(config, "check_job_script_integrity_sleep", DEFAULT_INTEGRITY_SLEEP) for i in range(count): try: - proc = subprocess.Popen([path], env={"ABC_TEST_JOB_SCRIPT_INTEGRITY_XYZ": "1"}) - proc.wait() - if proc.returncode == 42: + returncode = subprocess.call([path], env={"ABC_TEST_JOB_SCRIPT_INTEGRITY_XYZ": "1"}) + if returncode == 42: script_integrity_verified = True break From 8c984c9b3e4d1c7fb323448e55a936ec5112c8c1 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 25 Sep 2017 15:00:27 +0100 Subject: [PATCH 63/66] Remove unused binary_compatibility.py script Follow-up on commit 39d78c7226d8f0417577bdea94b15ac2c77c45a2 from PR https://github.com/galaxyproject/galaxy/pull/2383 --- scripts/binary_compatibility.py | 75 --------------------------------- 1 file changed, 75 deletions(-) delete mode 100644 scripts/binary_compatibility.py diff --git a/scripts/binary_compatibility.py b/scripts/binary_compatibility.py deleted file mode 100644 index 29141d15d82..00000000000 --- a/scripts/binary_compatibility.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python -""" -Creates a Python binary-compatibility.cfg as described here: - - https://mail.python.org/pipermail/distutils-sig/2015-July/026617.html -""" -from __future__ import print_function - -# Vaguely Python 2.6 compatibile ArgumentParser import -try: - from argparse import ArgumentParser -except ImportError: - from optparse import OptionParser - - class ArgumentParser(OptionParser): - - def __init__(self, **kwargs): - self.delegate = OptionParser(**kwargs) - - def add_argument(self, *args, **kwargs): - if "required" in kwargs: - del kwargs["required"] - return self.delegate.add_option(*args, **kwargs) - - def parse_args(self, args=None): - (options, args) = self.delegate.parse_args(args) - return options - -import sys -import json - -from pip.pep425tags import get_supported -from pip.platform import get_specific_platform - - -compatible_platforms = { - 'centos': 'rhel', - 'scientific': 'rhel', -} - - -def install_compat(): - spec_plat = get_specific_platform() - if spec_plat is None: - return None - this_plat = spec_plat[0] - compat_plat = compatible_platforms.get(this_plat, None) - rval = {} - if compat_plat: - print('{0} is binary compatible with {1} (and can install {2} wheels)' - .format(this_plat, compat_plat, compat_plat), - file=sys.stderr) - for py, abi, plat in get_supported(): - if this_plat in plat: - rval[plat] = {'install': [plat.replace(this_plat, compat_plat)]} - return rval - - -def main(): - arg_parser = ArgumentParser() - arg_parser.add_argument('-o', '--output', default=None, help='Output to file') - args = arg_parser.parse_args() - - compat = install_compat() - - if compat: - if args.output is not None: - with open(args.output, 'w') as out: - json.dump(compat, out) - else: - print(json.dumps(install_compat())) - - -if __name__ == '__main__': - main() From 9b1832abb11b79a244e0505b4afc059ba894d944 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 25 Sep 2017 19:42:33 +0100 Subject: [PATCH 64/66] Fix import order for cron/, scripts/ and test/ --- .ci/flake8_lint_include_list.txt | 185 +----------------- cron/build_chrom_db.py | 2 +- lib/galaxy/tools/toolbox/filters/__init__.py | 2 +- lib/galaxy/visualization/plugins/plugin.py | 15 +- scripts/api/data_manager_example_execute.py | 4 +- scripts/api/display.py | 2 +- scripts/api/search.py | 4 +- scripts/dump_tour.py | 2 +- scripts/edam_mapping.py | 2 +- scripts/fetch_eggs.py | 7 +- scripts/grt/export.py | 15 +- scripts/grt/upload.py | 5 +- scripts/manage_tool_dependencies.py | 2 +- scripts/microbes/BeautifulSoup.py | 7 +- scripts/microbes/harvest_bacteria.py | 7 +- scripts/paster.py | 2 +- scripts/rst2html.py | 6 +- scripts/summarize_timings.py | 2 +- scripts/tool_shed/api/export.py | 2 +- .../api/get_filtered_repository_revisions.py | 2 +- .../bootstrap_tool_shed/bootstrap_util.py | 2 +- .../create_user_with_api_key.py | 5 +- ...deprecate_repositories_without_metadata.py | 8 +- scripts/tools/maf/check_loc_file.py | 6 +- test/api/test_api_batch.py | 2 +- test/api/test_authenticate.py | 2 +- test/api/test_datatypes.py | 2 +- test/api/test_histories.py | 10 +- test/api/test_history_contents.py | 8 +- test/api/test_pages.py | 2 +- test/api/test_tools.py | 2 +- test/api/test_users.py | 7 +- test/api/test_visualizations.py | 4 +- test/api/test_workflows.py | 4 +- test/base/populators.py | 6 +- test/functional/twilltestcase.py | 2 +- test/galaxy_selenium/data.py | 3 +- test/galaxy_selenium/has_driver.py | 6 +- test/galaxy_selenium/navigates_galaxy.py | 7 +- test/manual/workflows_scaling.py | 4 +- test/selenium_tests/framework.py | 23 +-- test/selenium_tests/test_anon_history.py | 6 +- .../test_collection_builders.py | 6 +- .../test_history_dataset_state.py | 2 +- test/selenium_tests/test_history_options.py | 6 +- test/selenium_tests/test_history_panel.py | 6 +- test/selenium_tests/test_history_sharing.py | 6 +- test/selenium_tests/test_login.py | 6 +- test/selenium_tests/test_sizzle_loading.py | 6 +- test/selenium_tests/test_stock_tours.py | 8 +- test/selenium_tests/test_tool_form.py | 6 +- test/selenium_tests/test_uploads.py | 2 +- test/selenium_tests/test_workflow_editor.py | 11 +- test/selenium_tests/test_workflow_run.py | 11 +- test/shed_functional/base/test_db_util.py | 8 +- test/shed_functional/base/twilltestcase.py | 4 +- test/unit/__init__.py | 0 .../dataproviders/test_base_dataproviders.py | 6 +- test/unit/jobs/test_job_wrapper.py | 2 +- test/unit/jobs/test_runner_local.py | 2 +- test/unit/managers/base.py | 6 +- test/unit/managers/test_HistoryManager.py | 7 +- .../test_installed_repository_manager.py | 3 +- .../unit/shed_unit/test_tool_panel_manager.py | 3 +- test/unit/tools/test_actions.py | 3 +- .../tools/test_collect_primary_datasets.py | 2 +- test/unit/tools/test_column_parameters.py | 3 +- test/unit/tools/test_data_parameters.py | 8 +- test/unit/tools/test_dataset_matcher.py | 4 +- test/unit/tools/test_evaluation.py | 3 +- test/unit/tools/test_execution.py | 3 +- test/unit/tools/test_history_imp_exp.py | 5 +- test/unit/tools/test_parameter_parsing.py | 5 +- test/unit/tools/test_toolbox.py | 6 +- test/unit/tools/test_toolbox_filters.py | 3 +- .../plugins/test_VisualizationPlugin.py | 7 +- .../plugins/test_VisualizationsRegistry.py | 7 +- test/unit/web/framework/test_webapp.py | 6 +- 78 files changed, 205 insertions(+), 373 deletions(-) create mode 100644 test/unit/__init__.py diff --git a/.ci/flake8_lint_include_list.txt b/.ci/flake8_lint_include_list.txt index 4cb3e453782..6d90bc4ab6f 100644 --- a/.ci/flake8_lint_include_list.txt +++ b/.ci/flake8_lint_include_list.txt @@ -1,9 +1,6 @@ client/galaxy/style/source_material/circle.py contrib/ -cron/add_manual_builds.py -cron/cleanup_datasets.py -cron/parse_builds_3_sites.py -cron/parse_builds.py +cron/ doc/parse_gx_xsd.py lib/galaxy/actions/ lib/galaxy/auth/ @@ -77,6 +74,7 @@ lib/galaxy/visualization/genome/ lib/galaxy/visualization/genomes.py lib/galaxy/visualization/__init__.py lib/galaxy/visualization/plugins/__init__.py +lib/galaxy/visualization/plugins/plugin.py lib/galaxy/visualization/plugins/utils.py lib/galaxy/visualization/tracks/ lib/galaxy/web/base/ @@ -152,182 +150,7 @@ lib/log_tempfile.py lib/mimeparse.py lib/psyco_full.py lib/tool_shed/ -scripts/api/common.py -scripts/api/copy_hda_to_library_folder.py -scripts/api/create.py -scripts/api/delete.py -scripts/api/example_watch_folder.py -scripts/api/filter_failed_datasets_from_collection.py -scripts/api/form_create_from_xml.py -scripts/api/history_create_history.py -scripts/api/history_delete_history.py -scripts/api/import_library_dataset_to_history.py -scripts/api/import_workflows_from_installed_tool_shed_repository.py -scripts/api/install_tool_shed_repositories.py -scripts/api/library_create_folder.py -scripts/api/library_create_library.py -scripts/api/library_upload_dir.py -scripts/api/library_upload_from_import_dir.py -scripts/api/load_data_with_metadata.py -scripts/api/repair_tool_shed_repository.py -scripts/api/requests_update_state.py -scripts/api/request_type_create_from_xml.py -scripts/api/reset_metadata_on_installed_repositories.py -scripts/api/sample_dataset_update_status.py -scripts/api/sample_update_state.py -scripts/api/sequencer_configuration_create.py -scripts/api/update.py -scripts/api/upload_to_history.py -scripts/api/workflow_delete.py -scripts/api/workflow_execute_parameters.py -scripts/api/workflow_execute.py -scripts/api/workflow_import_from_file_rpark.py -scripts/api/workflow_import.py -scripts/auth/ -scripts/bootstrap_history.py -scripts/build_toolbox.py -scripts/build_universe_config.py -scripts/check_galaxy.py -scripts/check_python.py -scripts/cleanup_datasets/ -scripts/communication/ -scripts/create_db.py -scripts/data_libraries/ -scripts/db_shell.py -scripts/drmaa_external_killer.py -scripts/drmaa_external_runner.py -scripts/external_chown_script.py -scripts/extract_dataset_part.py -scripts/extract_toolbox_sections.py -scripts/functional_tests.py -scripts/get_platforms.py -scripts/helper.py -scripts/__init__.py -scripts/loc_files/ -scripts/manage_db.py -scripts/manage_tools.py -scripts/metagenomics/ -scripts/microbes/create_bacteria_loc_file.py -scripts/microbes/create_bacteria_table.py -scripts/microbes/create_nib_seq_loc_file.py -scripts/microbes/get_builds_lengths.py -scripts/microbes/ncbi_to_ucsc.py -scripts/microbes/util.py -scripts/migrate_tools/ -scripts/nosetests.py -scripts/others/ -scripts/runtime_stats.py -scripts/secret_decoder_ring.py -scripts/set_dataset_sizes.py -scripts/set_user_disk_usage.py -scripts/sync_reports_config.py -scripts/tool_shed/api/add_repository_registry_entry.py -scripts/tool_shed/api/common.py -scripts/tool_shed/api/create_categories.py -scripts/tool_shed/api/create_users.py -scripts/tool_shed/api/import_capsule.py -scripts/tool_shed/api/remove_repository_registry_entry.py -scripts/tool_shed/api/reset_metadata_on_repositories.py -scripts/tool_shed/api/tool_shed_repository_revision_update.py -scripts/tool_shed/build_ts_whoosh_index.py -scripts/tool_shed/check_download_urls.py -scripts/tool_shed/check_filesystem_for_empty_tool_dependency_installation_paths.py -scripts/tool_shed/clean_up_tool_dependency_directory.py -scripts/tool_shed/migrate_tools_to_repositories.py -scripts/tool_shed/show_tool_dependency_installation_dir_contents.py -scripts/tools/re_escape_output.py -test/api/__init__.py -test/api/test_dataset_collections.py -test/api/test_datasets.py -test/api/test_framework.py -test/api/test_history_contents_provenance.py -test/api/test_jobs.py -test/api/test_libraries.py -test/api/test_page_revisions.py -test/api/test_search.py -test/api/test_tool_data.py -test/api/test_tours.py -test/api/test_workflow_extraction.py -test/api/test_workflows_from_yaml.py -test/base/api_asserts.py -test/base/api.py -test/base/api_util.py -test/base/__init__.py -test/base/instrument.py -test/base/integration_util.py -test/base/interactor.py -test/base/nose_util.py -test/base/test_logging.py -test/base/tool_shed_util.py -test/base/twilltestcase.py -test/base/workflows_format_2/ -test/casperjs/ -test/functional/database_contexts.py -test/functional/__init__.py -test/functional/test_data_managers.py -test/functional/tool-data/ -test/functional/tools/ -test/integration/ -test/manual/__init__.py -test/shed_functional/base/common.py -test/shed_functional/base/__init__.py -test/shed_functional/functional/ -test/shed_functional/functional_tests.py -test/shed_functional/__init__.py -test/unit/dataset_collections/ -test/unit/datatypes/dataproviders/__init__.py -test/unit/datatypes/dataproviders/test_line_dataproviders.py -test/unit/datatypes/__init__.py -test/unit/datatypes/test_data.py -test/unit/jobs/dynamic_tool_destination/ -test/unit/jobs/__init__.py -test/unit/jobs/test_command_factory.py -test/unit/jobs/test_datasets.py -test/unit/jobs/test_job_configuration.py -test/unit/jobs/test_job_output_checker.py -test/unit/jobs/test_mapper.py -test/unit/jobs/test_rule_helper.py -test/unit/jobs/test_rules/ -test/unit/jobs/test_runner_params.py -test/unit/managers/__init__.py -test/unit/managers/test_CollectionManager.py -test/unit/managers/test_DatasetManager.py -test/unit/managers/test_HDAManager.py -test/unit/managers/test_HDCAManager.py -test/unit/managers/test_HistoryContentsManager.py -test/unit/managers/test_UserManager.py -test/unit/shed_unit/__init__.py -test/unit/shed_unit/test_fabric_util.py -test/unit/shed_unit/test_td_common_util.py -test/unit/test_galaxy_transactions.py -test/unit/test_lazy_process.py -test/unit/test_objectstore.py -test/unit/test_routes.py -test/unit/test_security_helper.py -test/unit/test_sockets.py -test/unit/test_sqlite_utils.py -test/unit/test_topsort.py -test/unit/tools/filter_modules/ -test/unit/tools/__init__.py -test/unit/tools/test_citations.py -test/unit/tools/test_conda_resolution.py -test/unit/tools/test_parsing.py -test/unit/tools/test_select_parameters.py -test/unit/tools/test_toolbox_filters.py -test/unit/tools/test_tool_dependency_description.py -test/unit/tools/test_tool_deps.py -test/unit/tools/test_tool_external_files.py -test/unit/tools/test_tool_loader.py -test/unit/tools/test_tool_panel.py -test/unit/tools/test_watcher.py -test/unit/tools/test_wrappers.py -test/unit/tools_support.py -test/unit/unittest_utils/ -test/unit/visualizations/__init__.py -test/unit/visualizations/plugins/__init__.py -test/unit/web/base/__init__.py -test/unit/web/framework/__init__.py -test/unit/web/__init__.py -test/unit/workflows/ +scripts/ +test/ tool_list.py tools/ diff --git a/cron/build_chrom_db.py b/cron/build_chrom_db.py index 4775897548f..60c79b27b9b 100644 --- a/cron/build_chrom_db.py +++ b/cron/build_chrom_db.py @@ -20,7 +20,7 @@ import sys import requests from six.moves.urllib.parse import urlencode -import parse_builds +import parse_builds # noqa: I100 def getchrominfo(url, db): diff --git a/lib/galaxy/tools/toolbox/filters/__init__.py b/lib/galaxy/tools/toolbox/filters/__init__.py index 6e638076482..5d36d24c19b 100644 --- a/lib/galaxy/tools/toolbox/filters/__init__.py +++ b/lib/galaxy/tools/toolbox/filters/__init__.py @@ -83,7 +83,7 @@ class FilterFactory(object): try: __import__(full_module_name) except ImportError: - # log.debug("Failed to load module.", exc_info=True) + # log.debug("Failed to load module %s using sys.path %s", full_module_name, sys.path, exc_info=True) continue module = sys.modules[full_module_name] if hasattr(module, function_name): diff --git a/lib/galaxy/visualization/plugins/plugin.py b/lib/galaxy/visualization/plugins/plugin.py index e1c5f0ddcfa..82ccdd701fd 100644 --- a/lib/galaxy/visualization/plugins/plugin.py +++ b/lib/galaxy/visualization/plugins/plugin.py @@ -2,17 +2,18 @@ Visualization plugins: instantiate/deserialize data and models from a query string and render a webpage based on those data. """ - -import logging import copy -import mako +import logging import os -from galaxy.managers import api_keys +import mako.lookup -from galaxy.visualization.plugins import interactive_environments -from galaxy.visualization.plugins import resource_parser -from galaxy.visualization.plugins import utils +from galaxy.managers import api_keys +from galaxy.visualization.plugins import ( + interactive_environments, + resource_parser, + utils +) log = logging.getLogger(__name__) diff --git a/scripts/api/data_manager_example_execute.py b/scripts/api/data_manager_example_execute.py index 27715ab726f..991bef50b6e 100644 --- a/scripts/api/data_manager_example_execute.py +++ b/scripts/api/data_manager_example_execute.py @@ -4,11 +4,13 @@ # Script makes the naive assumption that dbkey==sequence id, which in many cases is not true nor desired # *** This script is not recommended for use as-is on a production server *** from __future__ import print_function + import optparse import time + from six.moves.urllib.parse import urljoin -from common import get, post +from common import get, post # noqa: I100 DEFAULT_SLEEP_TIME = 3 FETCH_GENOME_TOOL_ID = 'testtoolshed.g2.bx.psu.edu/repos/blankenberg/data_manager_fetch_genome_all_fasta/data_manager_fetch_genome_all_fasta/0.0.1' diff --git a/scripts/api/display.py b/scripts/api/display.py index bcf3885641c..d6fe219b3de 100755 --- a/scripts/api/display.py +++ b/scripts/api/display.py @@ -6,7 +6,7 @@ import sys from six.moves.urllib.error import URLError -from common import display +from common import display # noqa: I100 try: display(*sys.argv[1:3]) diff --git a/scripts/api/search.py b/scripts/api/search.py index bbb905d621d..73fc22b0451 100644 --- a/scripts/api/search.py +++ b/scripts/api/search.py @@ -2,10 +2,12 @@ Sample script for Galaxy Search API """ from __future__ import print_function + import json -import requests import sys +import requests + class RemoteGalaxy(object): diff --git a/scripts/dump_tour.py b/scripts/dump_tour.py index dde0e8e556f..b13020581cb 100755 --- a/scripts/dump_tour.py +++ b/scripts/dump_tour.py @@ -2,8 +2,8 @@ import argparse import datetime import os -import time import sys +import time sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'test'))) diff --git a/scripts/edam_mapping.py b/scripts/edam_mapping.py index f483db8c1f5..c3756ad97c7 100644 --- a/scripts/edam_mapping.py +++ b/scripts/edam_mapping.py @@ -22,8 +22,8 @@ import requests sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib'))) -import galaxy.model import galaxy.datatypes.registry +import galaxy.model SCRIPTS_DIR = os.path.dirname(__file__) PROJECT_DIR = os.path.abspath(os.path.join(SCRIPTS_DIR, os.pardir)) diff --git a/scripts/fetch_eggs.py b/scripts/fetch_eggs.py index 5236a733266..61451a6ef1b 100755 --- a/scripts/fetch_eggs.py +++ b/scripts/fetch_eggs.py @@ -1,7 +1,12 @@ #!/usr/bin/env python from __future__ import print_function + from os import pardir -from os.path import join, abspath, dirname +from os.path import ( + abspath, + dirname, + join +) from sys import exit msg = """ diff --git a/scripts/grt/export.py b/scripts/grt/export.py index da8048522f2..f5a3480c14b 100644 --- a/scripts/grt/export.py +++ b/scripts/grt/export.py @@ -4,25 +4,24 @@ See doc/source/admin/grt.rst for more detailed usage information. """ import argparse -import tarfile import json +import logging import os -import sqlalchemy as sa import subprocess import sys +import tarfile import time -import yaml -import logging - from collections import defaultdict -sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib'))) +import sqlalchemy as sa +import yaml -from galaxy.util.properties import load_app_properties +sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib'))) import galaxy import galaxy.config -from galaxy.objectstore import build_object_store_from_config from galaxy.model import mapping +from galaxy.objectstore import build_object_store_from_config +from galaxy.util.properties import load_app_properties sample_config = os.path.abspath(os.path.join(os.path.dirname(__file__), 'grt.yml.sample')) default_config = os.path.abspath(os.path.join(os.path.dirname(__file__), 'grt.yml')) diff --git a/scripts/grt/upload.py b/scripts/grt/upload.py index a93efb86a62..8ec619fc5b7 100644 --- a/scripts/grt/upload.py +++ b/scripts/grt/upload.py @@ -4,11 +4,12 @@ See doc/source/admin/grt.rst for more detailed usage information. """ import argparse +import logging import os import sys -import yaml -import logging + import requests +import yaml sample_config = os.path.abspath(os.path.join(os.path.dirname(__file__), 'grt.yml.sample')) default_config = os.path.abspath(os.path.join(os.path.dirname(__file__), 'grt.yml')) diff --git a/scripts/manage_tool_dependencies.py b/scripts/manage_tool_dependencies.py index 45a89cc82ae..40e09217d48 100644 --- a/scripts/manage_tool_dependencies.py +++ b/scripts/manage_tool_dependencies.py @@ -4,7 +4,6 @@ import sys sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib'))) -from galaxy.util.properties import find_config_file, load_app_properties from galaxy.config import ( configure_logging, find_path, @@ -12,6 +11,7 @@ from galaxy.config import ( parse_dependency_options, ) from galaxy.tools.deps import CachedDependencyManager, DependencyManager, NullDependencyManager +from galaxy.util.properties import find_config_file, load_app_properties DESCRIPTION = "Script to manage tool dependencies (with focus on a Conda environments)." diff --git a/scripts/microbes/BeautifulSoup.py b/scripts/microbes/BeautifulSoup.py index cd104595187..b9262d7a828 100644 --- a/scripts/microbes/BeautifulSoup.py +++ b/scripts/microbes/BeautifulSoup.py @@ -45,12 +45,15 @@ from __future__ import generators import codecs import re +import sgmllib import string import sys import types -import sgmllib from htmlentitydefs import name2codepoint -from sgmllib import SGMLParser, SGMLParseError +from sgmllib import ( + SGMLParseError, + SGMLParser +) __author__ = "Leonard Richardson (crummy.com)" __contributors__ = ["Sam Ruby (intertwingly.net)", diff --git a/scripts/microbes/harvest_bacteria.py b/scripts/microbes/harvest_bacteria.py index 3226dec78ce..762d9cfa821 100644 --- a/scripts/microbes/harvest_bacteria.py +++ b/scripts/microbes/harvest_bacteria.py @@ -12,7 +12,12 @@ from urllib import urlretrieve import requests from BeautifulSoup import BeautifulSoup -from util import get_bed_from_genbank, get_bed_from_glimmer3, get_bed_from_GeneMarkHMM, get_bed_from_GeneMark +from util import ( + get_bed_from_genbank, + get_bed_from_GeneMark, + get_bed_from_GeneMarkHMM, + get_bed_from_glimmer3 +) assert sys.version_info[:2] >= (2, 4) diff --git a/scripts/paster.py b/scripts/paster.py index e1548b3ed6e..861a98075ae 100755 --- a/scripts/paster.py +++ b/scripts/paster.py @@ -11,7 +11,7 @@ sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pa from galaxy.util.pastescript import serve -from check_python import check_python +from check_python import check_python # noqa: I100 # ensure supported version try: diff --git a/scripts/rst2html.py b/scripts/rst2html.py index ef6e1fcc20a..409d8e26c2f 100755 --- a/scripts/rst2html.py +++ b/scripts/rst2html.py @@ -16,8 +16,10 @@ try: except: pass -from docutils.core import publish_cmdline, default_description - +from docutils.core import ( + default_description, + publish_cmdline +) description = ('Generates (X)HTML documents from standalone reStructuredText ' 'sources. ' + default_description) diff --git a/scripts/summarize_timings.py b/scripts/summarize_timings.py index 5cf4295bc79..cfa1169594b 100644 --- a/scripts/summarize_timings.py +++ b/scripts/summarize_timings.py @@ -1,8 +1,8 @@ """Script to parse timings out of a Galaxy log and summarize.""" from __future__ import print_function -from argparse import ArgumentParser import re +from argparse import ArgumentParser import numpy diff --git a/scripts/tool_shed/api/export.py b/scripts/tool_shed/api/export.py index 6a5d030c6a0..05b43f53f78 100644 --- a/scripts/tool_shed/api/export.py +++ b/scripts/tool_shed/api/export.py @@ -17,7 +17,7 @@ import requests sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, 'lib')) from tool_shed.util import basic_util -from common import display, submit +from common import display, submit # noqa: I100 CAPSULE_FILENAME = 'capsule' CAPSULE_WITH_DEPENDENCIES_FILENAME = 'capsule_with_dependencies' diff --git a/scripts/tool_shed/api/get_filtered_repository_revisions.py b/scripts/tool_shed/api/get_filtered_repository_revisions.py index a59911fc40c..b5805e697bc 100644 --- a/scripts/tool_shed/api/get_filtered_repository_revisions.py +++ b/scripts/tool_shed/api/get_filtered_repository_revisions.py @@ -28,7 +28,7 @@ sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, from galaxy.util import asbool from tool_shed.util import hg_util -from common import get_api_url, get_repository_dict, json_from_url +from common import get_api_url, get_repository_dict, json_from_url # noqa: I100 def main(options): diff --git a/scripts/tool_shed/bootstrap_tool_shed/bootstrap_util.py b/scripts/tool_shed/bootstrap_tool_shed/bootstrap_util.py index 2a899238b88..6d91d9bf87a 100755 --- a/scripts/tool_shed/bootstrap_tool_shed/bootstrap_util.py +++ b/scripts/tool_shed/bootstrap_tool_shed/bootstrap_util.py @@ -1,6 +1,6 @@ #!/usr/bin/python -import optparse import ConfigParser +import optparse import os import sys diff --git a/scripts/tool_shed/bootstrap_tool_shed/create_user_with_api_key.py b/scripts/tool_shed/bootstrap_tool_shed/create_user_with_api_key.py index 4d1d20658ad..001c001a87d 100644 --- a/scripts/tool_shed/bootstrap_tool_shed/create_user_with_api_key.py +++ b/scripts/tool_shed/bootstrap_tool_shed/create_user_with_api_key.py @@ -2,10 +2,10 @@ import ConfigParser import logging +import optparse import os import re import sys -import optparse sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, 'lib')) sys.path.insert(1, os.path.join(os.path.dirname(__file__))) @@ -13,7 +13,8 @@ sys.path.insert(1, os.path.join(os.path.dirname(__file__))) import galaxy.webapps.tool_shed.config as tool_shed_config from galaxy.web import security from galaxy.webapps.tool_shed.model import mapping -from bootstrap_util import admin_user_info + +from bootstrap_util import admin_user_info # noqa: I100 log = logging.getLogger(__name__) diff --git a/scripts/tool_shed/deprecate_repositories_without_metadata.py b/scripts/tool_shed/deprecate_repositories_without_metadata.py index 95357ceafbe..1a31604e6e1 100644 --- a/scripts/tool_shed/deprecate_repositories_without_metadata.py +++ b/scripts/tool_shed/deprecate_repositories_without_metadata.py @@ -8,8 +8,8 @@ import sys import textwrap import time from datetime import datetime, timedelta -from time import strftime from optparse import OptionParser +from time import strftime import sqlalchemy as sa from sqlalchemy import and_, distinct, false, not_ @@ -18,8 +18,10 @@ sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, import galaxy.webapps.tool_shed.config as tool_shed_config import galaxy.webapps.tool_shed.model.mapping -from galaxy.util import send_mail as galaxy_send_mail -from galaxy.util import build_url +from galaxy.util import ( + build_url, + send_mail as galaxy_send_mail +) log = logging.getLogger() log.setLevel(10) diff --git a/scripts/tools/maf/check_loc_file.py b/scripts/tools/maf/check_loc_file.py index cc0078a238d..96be0d8f7ce 100644 --- a/scripts/tools/maf/check_loc_file.py +++ b/scripts/tools/maf/check_loc_file.py @@ -1,10 +1,12 @@ # Dan Blankenberg # This script checks maf_index.loc file for inconsistencies between what is listed as available and what is really available. # Make sure that required dependencies (e.g. galaxy_root/lib) are included in your PYTHONPATH -import bx.align.maf -from galaxy.tools.util import maf_utilities import sys +import bx.align.maf + +from galaxy.tools.util import maf_utilities + assert sys.version_info[:2] >= (2, 4) diff --git a/test/api/test_api_batch.py b/test/api/test_api_batch.py index fef24d0a87d..e7a8d1ccc39 100644 --- a/test/api/test_api_batch.py +++ b/test/api/test_api_batch.py @@ -2,7 +2,7 @@ import json from requests import post -from base import api +from base import api # noqa: I100 class ApiBatchTestCase(api.ApiTestCase): diff --git a/test/api/test_authenticate.py b/test/api/test_authenticate.py index a99e2ff4728..5e25e69426d 100644 --- a/test/api/test_authenticate.py +++ b/test/api/test_authenticate.py @@ -2,7 +2,7 @@ import base64 from requests import get -from base import api +from base import api # noqa: I100 TEST_USER_EMAIL = "auth_user_test@bx.psu.edu" TEST_USER_PASSWORD = "testpassword1" diff --git a/test/api/test_datatypes.py b/test/api/test_datatypes.py index d41cd98596b..70341332e97 100644 --- a/test/api/test_datatypes.py +++ b/test/api/test_datatypes.py @@ -2,7 +2,7 @@ import time from requests import put -from base import api +from base import api # noqa: I100 HIDDEN_DURING_UPLOAD_DATATYPE = "fli" diff --git a/test/api/test_histories.py b/test/api/test_histories.py index cf57354d342..ed26d3056f6 100644 --- a/test/api/test_histories.py +++ b/test/api/test_histories.py @@ -1,14 +1,16 @@ # -*- coding: utf-8 -*- - from requests import ( get, post, put ) -from base import api - -from base.populators import DatasetPopulator, DatasetCollectionPopulator, wait_on +from base import api # noqa: I100 +from base.populators import ( # noqa: I100 + DatasetCollectionPopulator, + DatasetPopulator, + wait_on +) class HistoriesApiTestCase(api.ApiTestCase): diff --git a/test/api/test_history_contents.py b/test/api/test_history_contents.py index c82e8cb4cdd..f395452f97c 100644 --- a/test/api/test_history_contents.py +++ b/test/api/test_history_contents.py @@ -4,8 +4,12 @@ import json from requests import delete, put -from base import api -from base.populators import DatasetCollectionPopulator, LibraryPopulator, TestsDatasets +from base import api # noqa: I100 +from base.populators import ( # noqa: I100 + DatasetCollectionPopulator, + LibraryPopulator, + TestsDatasets +) # TODO: Test anonymous access. diff --git a/test/api/test_pages.py b/test/api/test_pages.py index f0b5755eec4..8d72b0601ad 100644 --- a/test/api/test_pages.py +++ b/test/api/test_pages.py @@ -1,6 +1,6 @@ from requests import delete -from base import api +from base import api # noqa: I100 from galaxy.exceptions import error_codes diff --git a/test/api/test_tools.py b/test/api/test_tools.py index 0dcfe546bac..7b7e7cd878b 100644 --- a/test/api/test_tools.py +++ b/test/api/test_tools.py @@ -6,8 +6,8 @@ from base.populators import ( DatasetCollectionPopulator, DatasetPopulator, LibraryPopulator, - skip_without_tool, skip_without_datatype, + skip_without_tool, ) from galaxy.tools.verify.test_data import TestDataResolver diff --git a/test/api/test_users.py b/test/api/test_users.py index 977cac886c3..7453e3210a2 100644 --- a/test/api/test_users.py +++ b/test/api/test_users.py @@ -1,8 +1,11 @@ import json -from requests import put, get +from requests import ( + get, + put +) -from base import api +from base import api # noqa: I100 TEST_USER_EMAIL = "user_for_users_index_test@bx.psu.edu" diff --git a/test/api/test_visualizations.py b/test/api/test_visualizations.py index 7ed68af6321..702dd890c02 100644 --- a/test/api/test_visualizations.py +++ b/test/api/test_visualizations.py @@ -5,8 +5,8 @@ from requests import ( put ) -from base import api -from base.api_asserts import assert_has_keys +from base import api # noqa: I100 +from base.api_asserts import assert_has_keys # noqa: I100 INDEX_KEYS = [ 'id', 'title', 'type', 'dbkey', 'url' diff --git a/test/api/test_workflows.py b/test/api/test_workflows.py index f3c45c03368..c0162be0883 100644 --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -8,8 +8,8 @@ from uuid import uuid4 import yaml from requests import delete, put -from base import api -from base.populators import ( +from base import api # noqa: I100 +from base.populators import ( # noqa: I100 DatasetCollectionPopulator, DatasetPopulator, skip_without_tool, diff --git a/test/base/populators.py b/test/base/populators.py index e7d19e463a2..336ef523918 100644 --- a/test/base/populators.py +++ b/test/base/populators.py @@ -1,17 +1,15 @@ import contextlib import json import time - from functools import wraps from operator import itemgetter import requests - from pkg_resources import resource_string from six import StringIO -from base import api_asserts -from base.workflows_format_2 import ( +from . import api_asserts +from .workflows_format_2 import ( convert_and_import_workflow, ImporterGalaxyInterface, ) diff --git a/test/functional/twilltestcase.py b/test/functional/twilltestcase.py index 249c8a0b889..56c0aea45b8 100644 --- a/test/functional/twilltestcase.py +++ b/test/functional/twilltestcase.py @@ -15,7 +15,7 @@ from six import string_types, StringIO from six.moves.urllib.parse import urlencode, urlparse from twill.other_packages._mechanize_dist import ClientForm -from base.testcase import FunctionalTestCase +from base.testcase import FunctionalTestCase # noqa: I100 # Force twill to log to a buffer -- FIXME: Should this go to stdout and be captured by nose? buffer = StringIO() diff --git a/test/galaxy_selenium/data.py b/test/galaxy_selenium/data.py index ef15e27197b..1ed3baf0a5a 100644 --- a/test/galaxy_selenium/data.py +++ b/test/galaxy_selenium/data.py @@ -1,6 +1,5 @@ -from pkg_resources import resource_string - import yaml +from pkg_resources import resource_string data_yaml = resource_string(__name__, 'navigation-data.yml').decode("UTF-8") NAVIGATION_DATA = yaml.safe_load(data_yaml) diff --git a/test/galaxy_selenium/has_driver.py b/test/galaxy_selenium/has_driver.py index 4dca4fbe5d3..30b4a4f9e88 100644 --- a/test/galaxy_selenium/has_driver.py +++ b/test/galaxy_selenium/has_driver.py @@ -4,9 +4,9 @@ This should be mixed into classes with a self.driver and self.default_timeout attribute. """ +from selenium.common.exceptions import TimeoutException from selenium.webdriver.common.action_chains import ActionChains from selenium.webdriver.common.by import By -from selenium.common.exceptions import TimeoutException from selenium.webdriver.common.keys import Keys from selenium.webdriver.support import expected_conditions as ec from selenium.webdriver.support.ui import WebDriverWait @@ -140,7 +140,7 @@ class HasDriver: ) -def execption_indicates_not_clickable(exception): +def exception_indicates_not_clickable(exception): return "not clickable" in str(exception) @@ -149,7 +149,7 @@ def exception_indicates_stale_element(exception): __all__ = ( - "execption_indicates_not_clickable", + "exception_indicates_not_clickable", "exception_indicates_stale_element", "HasDriver", "TimeoutException", diff --git a/test/galaxy_selenium/navigates_galaxy.py b/test/galaxy_selenium/navigates_galaxy.py index 8981b3779da..71a77e711ac 100644 --- a/test/galaxy_selenium/navigates_galaxy.py +++ b/test/galaxy_selenium/navigates_galaxy.py @@ -8,20 +8,19 @@ import contextlib import random import string import time - from functools import partial, wraps import requests import yaml +from . import sizzle from .data import NAVIGATION_DATA from .has_driver import ( - execption_indicates_not_clickable, + exception_indicates_not_clickable, exception_indicates_stale_element, HasDriver, TimeoutException, ) -from . import sizzle # Test case data DEFAULT_PASSWORD = '123456' @@ -50,7 +49,7 @@ def excepion_seems_to_indicate_transition(e): StaleElement exceptions (a DOM element grabbed at one step is no longer available) and "not clickable" exceptions (so perhaps a popup modal is blocking a click). """ - return exception_indicates_stale_element(e) or execption_indicates_not_clickable(e) + return exception_indicates_stale_element(e) or exception_indicates_not_clickable(e) def retry_call_during_transitions(f, attempts=RETRY_DURING_TRANSITIONS_ATTEMPTS_DEFAULT, sleep=RETRY_DURING_TRANSITIONS_SLEEP_DEFAULT, exception_check=excepion_seems_to_indicate_transition): diff --git a/test/manual/workflows_scaling.py b/test/manual/workflows_scaling.py index 69ed5514580..677eff160df 100644 --- a/test/manual/workflows_scaling.py +++ b/test/manual/workflows_scaling.py @@ -19,12 +19,12 @@ from bioblend import galaxy galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)) sys.path[1:1] = [os.path.join(galaxy_root, "lib"), os.path.join(galaxy_root, "test")] -from base.populators import ( +from base.populators import ( # noqa: I100 GiDatasetCollectionPopulator, GiDatasetPopulator, GiWorkflowPopulator, ) -from api.workflows_format_2.converter import python_to_workflow +from api.workflows_format_2.converter import python_to_workflow # noqa: I100 LONG_TIMEOUT = 1000000000 DESCRIPTION = "Script to exercise the workflow engine." diff --git a/test/selenium_tests/framework.py b/test/selenium_tests/framework.py index 01532fc8fe5..d4f318596f0 100644 --- a/test/selenium_tests/framework.py +++ b/test/selenium_tests/framework.py @@ -8,31 +8,26 @@ import os import time import traceback import unittest - from functools import partial, wraps import requests - -from galaxy_selenium import ( - driver_factory, -) -from galaxy_selenium.navigates_galaxy import NavigatesGalaxy, retry_during_transitions - try: from pyvirtualdisplay import Display except ImportError: Display = None - from six.moves.urllib.parse import urljoin -from base import populators -from base.driver_util import classproperty, DEFAULT_WEB_HOST, get_ip_address -from base.testcase import FunctionalTestCase -from base.workflows_format_2 import ( - ImporterGalaxyInterface, +from base import populators # noqa: I100 +from base.driver_util import classproperty, DEFAULT_WEB_HOST, get_ip_address # noqa: I100 +from base.testcase import FunctionalTestCase # noqa: I100 +from base.workflows_format_2 import ( # noqa: I100 convert_and_import_workflow, + ImporterGalaxyInterface, ) - +from galaxy_selenium import ( # noqa: I100 + driver_factory, +) +from galaxy_selenium.navigates_galaxy import NavigatesGalaxy, retry_during_transitions # noqa: I100 from galaxy.util import asbool DEFAULT_WAIT_TIMEOUT = 60 diff --git a/test/selenium_tests/test_anon_history.py b/test/selenium_tests/test_anon_history.py index f26b8655f44..b37c8448391 100644 --- a/test/selenium_tests/test_anon_history.py +++ b/test/selenium_tests/test_anon_history.py @@ -1,5 +1,7 @@ -from .framework import SeleniumTestCase -from .framework import selenium_test +from .framework import ( + selenium_test, + SeleniumTestCase +) class AnonymousHistoriesTestCase(SeleniumTestCase): diff --git a/test/selenium_tests/test_collection_builders.py b/test/selenium_tests/test_collection_builders.py index f63febae6b3..6b59e1070da 100644 --- a/test/selenium_tests/test_collection_builders.py +++ b/test/selenium_tests/test_collection_builders.py @@ -1,5 +1,7 @@ -from .framework import SeleniumTestCase -from .framework import selenium_test +from .framework import ( + selenium_test, + SeleniumTestCase +) class CollectionBuildersTestCase(SeleniumTestCase): diff --git a/test/selenium_tests/test_history_dataset_state.py b/test/selenium_tests/test_history_dataset_state.py index 07ecb9becf5..95b86b015de 100644 --- a/test/selenium_tests/test_history_dataset_state.py +++ b/test/selenium_tests/test_history_dataset_state.py @@ -1,6 +1,6 @@ from .framework import ( - SeleniumTestCase, selenium_test, + SeleniumTestCase, UsesHistoryItemAssertions, ) diff --git a/test/selenium_tests/test_history_options.py b/test/selenium_tests/test_history_options.py index f701f65d357..d4dad146da4 100644 --- a/test/selenium_tests/test_history_options.py +++ b/test/selenium_tests/test_history_options.py @@ -1,5 +1,7 @@ -from .framework import SeleniumTestCase -from .framework import selenium_test +from .framework import ( + selenium_test, + SeleniumTestCase +) class HistoryOptionsTestCase(SeleniumTestCase): diff --git a/test/selenium_tests/test_history_panel.py b/test/selenium_tests/test_history_panel.py index 12c514b7306..f1aae2d5026 100644 --- a/test/selenium_tests/test_history_panel.py +++ b/test/selenium_tests/test_history_panel.py @@ -1,7 +1,9 @@ import time -from .framework import SeleniumTestCase -from .framework import selenium_test +from .framework import ( + selenium_test, + SeleniumTestCase +) class HistoryPanelTestCase(SeleniumTestCase): diff --git a/test/selenium_tests/test_history_sharing.py b/test/selenium_tests/test_history_sharing.py index dc77592b02f..7129c2713ff 100644 --- a/test/selenium_tests/test_history_sharing.py +++ b/test/selenium_tests/test_history_sharing.py @@ -1,5 +1,7 @@ -from .framework import SeleniumTestCase -from .framework import selenium_test +from .framework import ( + selenium_test, + SeleniumTestCase +) # Remove hack when submit_login works more consistently. VALID_LOGIN_RETRIES = 3 diff --git a/test/selenium_tests/test_login.py b/test/selenium_tests/test_login.py index e782728ea0c..cbdac4fc3d1 100644 --- a/test/selenium_tests/test_login.py +++ b/test/selenium_tests/test_login.py @@ -1,5 +1,7 @@ -from .framework import SeleniumTestCase -from .framework import selenium_test +from .framework import ( + selenium_test, + SeleniumTestCase +) class LoginTestCase(SeleniumTestCase): diff --git a/test/selenium_tests/test_sizzle_loading.py b/test/selenium_tests/test_sizzle_loading.py index 022e6fdfb67..94fdd6bb588 100644 --- a/test/selenium_tests/test_sizzle_loading.py +++ b/test/selenium_tests/test_sizzle_loading.py @@ -1,5 +1,7 @@ -from .framework import SeleniumTestCase -from .framework import selenium_test +from .framework import ( + selenium_test, + SeleniumTestCase +) class SizzleLoadingTestCase(SeleniumTestCase): diff --git a/test/selenium_tests/test_stock_tours.py b/test/selenium_tests/test_stock_tours.py index 4e3b1059bab..66ac07a5409 100644 --- a/test/selenium_tests/test_stock_tours.py +++ b/test/selenium_tests/test_stock_tours.py @@ -1,10 +1,12 @@ import os -from .framework import SeleniumTestCase -from .framework import selenium_test - from galaxy.util import galaxy_root_path +from .framework import ( + selenium_test, + SeleniumTestCase +) + STOCK_TOURS_DIRECTORY = os.path.join(galaxy_root_path, "config", "plugins", "tours") diff --git a/test/selenium_tests/test_tool_form.py b/test/selenium_tests/test_tool_form.py index 6d726c13621..38ad877971c 100644 --- a/test/selenium_tests/test_tool_form.py +++ b/test/selenium_tests/test_tool_form.py @@ -1,11 +1,11 @@ +from galaxy_selenium.navigates_galaxy import retry_call_during_transitions + from .framework import ( - SeleniumTestCase, selenium_test, + SeleniumTestCase, UsesHistoryItemAssertions, ) -from galaxy_selenium.navigates_galaxy import retry_call_during_transitions - class ToolFormTestCase(SeleniumTestCase, UsesHistoryItemAssertions): diff --git a/test/selenium_tests/test_uploads.py b/test/selenium_tests/test_uploads.py index 7e5cdcd55d3..d9ad6cdd570 100644 --- a/test/selenium_tests/test_uploads.py +++ b/test/selenium_tests/test_uploads.py @@ -1,6 +1,6 @@ from .framework import ( - SeleniumTestCase, selenium_test, + SeleniumTestCase, UsesHistoryItemAssertions, ) diff --git a/test/selenium_tests/test_workflow_editor.py b/test/selenium_tests/test_workflow_editor.py index 458cb3c032a..d44d252bd03 100644 --- a/test/selenium_tests/test_workflow_editor.py +++ b/test/selenium_tests/test_workflow_editor.py @@ -1,17 +1,16 @@ import time +from ._workflow_fixtures import ( + WORKFLOW_SIMPLE_CAT_TWICE, + WORKFLOW_WITH_INVALID_STATE, + WORKFLOW_WITH_OLD_TOOL_VERSION, +) from .framework import ( retry_assertion_during_transitions, selenium_test, SeleniumTestCase ) -from ._workflow_fixtures import ( - WORKFLOW_SIMPLE_CAT_TWICE, - WORKFLOW_WITH_OLD_TOOL_VERSION, - WORKFLOW_WITH_INVALID_STATE, -) - class WorkflowEditorTestCase(SeleniumTestCase): diff --git a/test/selenium_tests/test_workflow_run.py b/test/selenium_tests/test_workflow_run.py index 4533ecacabf..8bdc834f7b2 100644 --- a/test/selenium_tests/test_workflow_run.py +++ b/test/selenium_tests/test_workflow_run.py @@ -1,15 +1,14 @@ import time -from .framework import ( - selenium_test, - SeleniumTestCase, - UsesHistoryItemAssertions, -) - from ._workflow_fixtures import ( WORKFLOW_SIMPLE_CAT_TWICE, WORKFLOW_WITH_OLD_TOOL_VERSION, ) +from .framework import ( + selenium_test, + SeleniumTestCase, + UsesHistoryItemAssertions, +) class WorkflowRunTestCase(SeleniumTestCase, UsesHistoryItemAssertions): diff --git a/test/shed_functional/base/test_db_util.py b/test/shed_functional/base/test_db_util.py index c9ba1356555..f1db34bc614 100644 --- a/test/shed_functional/base/test_db_util.py +++ b/test/shed_functional/base/test_db_util.py @@ -5,9 +5,11 @@ from sqlalchemy import and_, false, true import galaxy.model import galaxy.model.tool_shed_install import galaxy.webapps.tool_shed.model as model - -from base.driver_util import (galaxy_context as ga_session, - install_context as install_session, tool_shed_context as sa_session) +from base.driver_util import ( # noqa: I100 + galaxy_context as ga_session, + install_context as install_session, + tool_shed_context as sa_session +) log = logging.getLogger('test.tool_shed.test_db_util') diff --git a/test/shed_functional/base/twilltestcase.py b/test/shed_functional/base/twilltestcase.py index 0ab2da9ddf4..681e51b3a4d 100644 --- a/test/shed_functional/base/twilltestcase.py +++ b/test/shed_functional/base/twilltestcase.py @@ -15,8 +15,8 @@ from six.moves.urllib.parse import quote_plus, urlencode import galaxy.model.tool_shed_install as galaxy_model import galaxy.util import galaxy.webapps.tool_shed.util.hgweb_config -from base.tool_shed_util import repository_installation_timeout -from functional.twilltestcase import TwillTestCase +from base.tool_shed_util import repository_installation_timeout # noqa: I100 +from functional.twilltestcase import TwillTestCase # noqa: I100 from galaxy.web import security from tool_shed.util import hg_util, xml_util from tool_shed.util.encoding_util import tool_shed_encode diff --git a/test/unit/__init__.py b/test/unit/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/unit/datatypes/dataproviders/test_base_dataproviders.py b/test/unit/datatypes/dataproviders/test_base_dataproviders.py index 883e12ea2f5..e081cc41ac4 100644 --- a/test/unit/datatypes/dataproviders/test_base_dataproviders.py +++ b/test/unit/datatypes/dataproviders/test_base_dataproviders.py @@ -3,17 +3,13 @@ Unit tests for base DataProviders. .. seealso:: galaxy.datatypes.dataproviders.base """ import logging -import os.path -import sys import unittest from six import StringIO from galaxy.datatypes.dataproviders import base, exceptions -unit_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) -sys.path.insert(1, unit_root) -from unittest_utils import tempfilecache, utility +from ...unittest_utils import tempfilecache, utility log = logging.getLogger(__name__) diff --git a/test/unit/jobs/test_job_wrapper.py b/test/unit/jobs/test_job_wrapper.py index 7c64fb3faef..268daf2315f 100644 --- a/test/unit/jobs/test_job_wrapper.py +++ b/test/unit/jobs/test_job_wrapper.py @@ -14,7 +14,7 @@ from galaxy.model import ( from galaxy.tools import evaluation from galaxy.util.bunch import Bunch -from tools_support import UsesApp +from ..tools_support import UsesApp TEST_TOOL_ID = "cufftest" TEST_VERSION_COMMAND = "bwa --version" diff --git a/test/unit/jobs/test_runner_local.py b/test/unit/jobs/test_runner_local.py index 70857bb3bb6..e9c85a301c0 100644 --- a/test/unit/jobs/test_runner_local.py +++ b/test/unit/jobs/test_runner_local.py @@ -8,7 +8,7 @@ from galaxy.jobs import metrics from galaxy.jobs.runners import local from galaxy.util import bunch -from tools_support import ( +from ..tools_support import ( UsesApp, UsesTools ) diff --git a/test/unit/managers/base.py b/test/unit/managers/base.py index 5291d636e9a..c1722e3c8dd 100644 --- a/test/unit/managers/base.py +++ b/test/unit/managers/base.py @@ -3,8 +3,6 @@ from __future__ import print_function import json -import os -import sys import unittest import sqlalchemy @@ -12,9 +10,7 @@ from six import string_types from galaxy.managers.users import UserManager -unit_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) -sys.path.insert(1, unit_root) -from unittest_utils import galaxy_mock +from ..unittest_utils import galaxy_mock # ============================================================================= admin_email = 'admin@admin.admin' diff --git a/test/unit/managers/test_HistoryManager.py b/test/unit/managers/test_HistoryManager.py index 1d741400374..530295b9c82 100644 --- a/test/unit/managers/test_HistoryManager.py +++ b/test/unit/managers/test_HistoryManager.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- """ """ -import os -import sys import unittest import sqlalchemy @@ -24,11 +22,8 @@ from galaxy.managers.histories import ( HistorySerializer ) -unit_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) -sys.path.insert(1, unit_root) -from unittest_utils import galaxy_mock - from .base import BaseTestCase +from ..unittest_utils import galaxy_mock default_password = '123456' user2_data = dict(email='user2@user2.user2', username='user2', password=default_password) diff --git a/test/unit/shed_unit/test_installed_repository_manager.py b/test/unit/shed_unit/test_installed_repository_manager.py index 3fc33637363..ffd3f57ff06 100644 --- a/test/unit/shed_unit/test_installed_repository_manager.py +++ b/test/unit/shed_unit/test_installed_repository_manager.py @@ -1,5 +1,6 @@ from tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager -from tools.test_toolbox import BaseToolBoxTestCase + +from ..tools.test_toolbox import BaseToolBoxTestCase class InstalledRepositoryManagerTestCase(BaseToolBoxTestCase): diff --git a/test/unit/shed_unit/test_tool_panel_manager.py b/test/unit/shed_unit/test_tool_panel_manager.py index e6cab3a53cc..c3ab72fb727 100644 --- a/test/unit/shed_unit/test_tool_panel_manager.py +++ b/test/unit/shed_unit/test_tool_panel_manager.py @@ -4,7 +4,8 @@ from galaxy.util import parse_xml from tool_shed.galaxy_install.tools import tool_panel_manager from tool_shed.tools import tool_version_manager -from tools.test_toolbox import ( + +from ..tools.test_toolbox import ( BaseToolBoxTestCase, SimplifiedToolBox ) diff --git a/test/unit/tools/test_actions.py b/test/unit/tools/test_actions.py index 1978f6dd109..5d0e0d83db1 100644 --- a/test/unit/tools/test_actions.py +++ b/test/unit/tools/test_actions.py @@ -2,7 +2,6 @@ import string import unittest from xml.etree.ElementTree import XML -import tools_support from galaxy import model from galaxy.tools.actions import ( DefaultToolAction, @@ -11,6 +10,8 @@ from galaxy.tools.actions import ( ) from galaxy.tools.parser.output_objects import ToolOutput +from .. import tools_support + TEST_HANDLER_NAME = "test_handler_1" diff --git a/test/unit/tools/test_collect_primary_datasets.py b/test/unit/tools/test_collect_primary_datasets.py index b5317e1b358..9c04b3301e0 100644 --- a/test/unit/tools/test_collect_primary_datasets.py +++ b/test/unit/tools/test_collect_primary_datasets.py @@ -2,7 +2,6 @@ import json import os import unittest -import tools_support from galaxy import ( model, util @@ -10,6 +9,7 @@ from galaxy import ( from galaxy.tools.parameters.output_collect import LegacyToolProvidedMetadata, NullToolProvidedMetadata from galaxy.tools.parser import output_collection_def +from .. import tools_support DEFAULT_TOOL_OUTPUT = "out1" DEFAULT_EXTRA_NAME = "test1" diff --git a/test/unit/tools/test_column_parameters.py b/test/unit/tools/test_column_parameters.py index ab40fd07851..9df583cdfcc 100644 --- a/test/unit/tools/test_column_parameters.py +++ b/test/unit/tools/test_column_parameters.py @@ -1,12 +1,11 @@ """ Tests for tool parameters, more tests exist in test_data_parameters.py and test_select_parameters.py. """ - from galaxy import model from galaxy.util import bunch -from tools_support import datatypes_registry from .test_parameter_parsing import BaseParameterTestCase +from ..tools_support import datatypes_registry class DataColumnParameterTestCase(BaseParameterTestCase): diff --git a/test/unit/tools/test_data_parameters.py b/test/unit/tools/test_data_parameters.py index 561dff9e205..9fd0f521042 100644 --- a/test/unit/tools/test_data_parameters.py +++ b/test/unit/tools/test_data_parameters.py @@ -1,14 +1,8 @@ -import os -import sys - from galaxy import model from galaxy.util import bunch -unit_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) -sys.path.insert(1, unit_root) -from unittest_utils import galaxy_mock - from .test_parameter_parsing import BaseParameterTestCase +from ..unittest_utils import galaxy_mock class DataToolParameterTestCase(BaseParameterTestCase): diff --git a/test/unit/tools/test_dataset_matcher.py b/test/unit/tools/test_dataset_matcher.py index 98c2a7d30bd..fe5b231eeb5 100644 --- a/test/unit/tools/test_dataset_matcher.py +++ b/test/unit/tools/test_dataset_matcher.py @@ -1,7 +1,6 @@ from unittest import TestCase from xml.etree.ElementTree import XML -import tools_support from galaxy import model from galaxy.tools.parameters import ( basic, @@ -10,9 +9,10 @@ from galaxy.tools.parameters import ( from galaxy.util import bunch from .test_data_parameters import MockHistoryDatasetAssociation +from ..tools_support import UsesApp -class DatasetMatcherTestCase(TestCase, tools_support.UsesApp): +class DatasetMatcherTestCase(TestCase, UsesApp): def test_hda_accessible(self): # Cannot access errored or discard datasets. diff --git a/test/unit/tools/test_evaluation.py b/test/unit/tools/test_evaluation.py index be7c9db772e..d0e38fc561c 100644 --- a/test/unit/tools/test_evaluation.py +++ b/test/unit/tools/test_evaluation.py @@ -27,8 +27,9 @@ from galaxy.tools.parameters.grouping import ( ) from galaxy.tools.parser.output_objects import ToolOutput from galaxy.util.bunch import Bunch + # Test fixtures for Galaxy infrastructure. -from tools_support import UsesApp +from ..tools_support import UsesApp # To Test: # - param_file handling. diff --git a/test/unit/tools/test_execution.py b/test/unit/tools/test_execution.py index f28606ff989..3182a271995 100644 --- a/test/unit/tools/test_execution.py +++ b/test/unit/tools/test_execution.py @@ -5,11 +5,12 @@ from unittest import TestCase from paste import httpexceptions import galaxy.model -import tools_support from galaxy.tools.parameters import params_to_incoming from galaxy.util.bunch import Bunch from galaxy.util.odict import odict +from .. import tools_support + BASE_REPEAT_TOOL_CONTENTS = ''' echo "$param1" #for $r in $repeat# "$r.param2" #end for# < $out1 diff --git a/test/unit/tools/test_history_imp_exp.py b/test/unit/tools/test_history_imp_exp.py index 599cad1d37e..02a6f967e3f 100644 --- a/test/unit/tools/test_history_imp_exp.py +++ b/test/unit/tools/test_history_imp_exp.py @@ -1,5 +1,4 @@ import os -import sys import tarfile from shutil import rmtree from tempfile import mkdtemp @@ -8,9 +7,7 @@ from galaxy import model from galaxy.exceptions import MalformedContents from galaxy.tools.imp_exp import JobImportHistoryArchiveWrapper, unpack_tar_gz_archive -unit_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) -sys.path.insert(1, unit_root) -from unittest_utils.galaxy_mock import MockApp +from ..unittest_utils.galaxy_mock import MockApp # good enough for the very specific tests we're writing as of now... diff --git a/test/unit/tools/test_parameter_parsing.py b/test/unit/tools/test_parameter_parsing.py index 32da6acb482..e6b3939b0b1 100644 --- a/test/unit/tools/test_parameter_parsing.py +++ b/test/unit/tools/test_parameter_parsing.py @@ -1,13 +1,14 @@ from unittest import TestCase from xml.etree.ElementTree import XML -import tools_support from galaxy import model from galaxy.tools.parameters import basic from galaxy.util import bunch +from ..tools_support import UsesApp -class BaseParameterTestCase(TestCase, tools_support.UsesApp): + +class BaseParameterTestCase(TestCase, UsesApp): def setUp(self): self.setup_app() diff --git a/test/unit/tools/test_toolbox.py b/test/unit/tools/test_toolbox.py index cf941b22e1b..d9366a8dab6 100644 --- a/test/unit/tools/test_toolbox.py +++ b/test/unit/tools/test_toolbox.py @@ -7,7 +7,6 @@ import unittest import routes from six import string_types -import tools_support from galaxy import model from galaxy.model import tool_shed_install from galaxy.model.tool_shed_install import mapping @@ -15,11 +14,12 @@ from galaxy.tools import ToolBox from galaxy.tools.cache import ToolCache from galaxy.webapps.galaxy.config_watchers import ConfigWatchers -from .test_toolbox_filters import mock_trans from .test_tool_loader import ( SIMPLE_MACRO, SIMPLE_TOOL_WITH_MACRO ) +from .test_toolbox_filters import mock_trans +from ..tools_support import UsesApp, UsesTools CONFIG_TEST_TOOL_VERSION_TEMPLATE = string.Template( @@ -37,7 +37,7 @@ CONFIG_TEST_TOOL_VERSION_1 = CONFIG_TEST_TOOL_VERSION_TEMPLATE.safe_substitute(d CONFIG_TEST_TOOL_VERSION_2 = CONFIG_TEST_TOOL_VERSION_TEMPLATE.safe_substitute(dict(version="2")) -class BaseToolBoxTestCase(unittest.TestCase, tools_support.UsesApp, tools_support.UsesTools): +class BaseToolBoxTestCase(unittest.TestCase, UsesApp, UsesTools): @property def integerated_tool_panel_path(self): diff --git a/test/unit/tools/test_toolbox_filters.py b/test/unit/tools/test_toolbox_filters.py index e334faa7c1e..b1fcf1da3bb 100644 --- a/test/unit/tools/test_toolbox_filters.py +++ b/test/unit/tools/test_toolbox_filters.py @@ -1,5 +1,4 @@ from galaxy.tools.toolbox.filters import FilterFactory - from galaxy.util.bunch import Bunch @@ -50,7 +49,7 @@ def filter_factory(config_dict=None): tool_label_filters=["filtermod:filter_label_1", "filtermod:filter_label_2"], ) config = Bunch(**config_dict) - config.toolbox_filter_base_modules = "galaxy.tools.filters,tools.filter_modules" + config.toolbox_filter_base_modules = "galaxy.tools.filters,unit.tools.filter_modules" app = Bunch(config=config) toolbox = Bunch(app=app) return FilterFactory(toolbox) diff --git a/test/unit/visualizations/plugins/test_VisualizationPlugin.py b/test/unit/visualizations/plugins/test_VisualizationPlugin.py index 96460fabe21..eae02765c60 100644 --- a/test/unit/visualizations/plugins/test_VisualizationPlugin.py +++ b/test/unit/visualizations/plugins/test_VisualizationPlugin.py @@ -1,8 +1,6 @@ """ Test lib/galaxy/visualization/plugins/plugin. """ -import os -import sys import unittest from six import string_types @@ -13,12 +11,9 @@ from galaxy.visualization.plugins import ( utils as vis_utils ) -unit_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) -sys.path.insert(1, unit_root) -from unittest_utils import galaxy_mock, utility +from ...unittest_utils import galaxy_mock, utility -# ----------------------------------------------------------------------------- class VisualizationsPlugin_TestCase(unittest.TestCase): plugin_class = vis_plugin.VisualizationPlugin diff --git a/test/unit/visualizations/plugins/test_VisualizationsRegistry.py b/test/unit/visualizations/plugins/test_VisualizationsRegistry.py index dc00310ced2..7ed3b8af10a 100644 --- a/test/unit/visualizations/plugins/test_VisualizationsRegistry.py +++ b/test/unit/visualizations/plugins/test_VisualizationsRegistry.py @@ -3,7 +3,6 @@ Test lib/galaxy/visualization/plugins/registry. """ import os import re -import sys import unittest from six import string_types @@ -12,11 +11,8 @@ from galaxy import model from galaxy.visualization.plugins import plugin from galaxy.visualization.plugins.registry import VisualizationsRegistry -unit_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) -sys.path.insert(1, unit_root) -from unittest_utils import galaxy_mock, utility +from ...unittest_utils import galaxy_mock, utility -# ----------------------------------------------------------------------------- glx_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, os.pardir)) template_cache_dir = os.path.join(glx_dir, 'database', 'compiled_templates') addtional_templates_dir = os.path.join(glx_dir, 'config', 'plugins', 'visualizations', 'common', 'templates') @@ -41,7 +37,6 @@ config1 = """\ """ -# ----------------------------------------------------------------------------- class VisualizationsRegistry_TestCase(unittest.TestCase): def test_plugin_load_from_repo(self): diff --git a/test/unit/web/framework/test_webapp.py b/test/unit/web/framework/test_webapp.py index d6efd9bfb45..319b980498c 100644 --- a/test/unit/web/framework/test_webapp.py +++ b/test/unit/web/framework/test_webapp.py @@ -2,17 +2,13 @@ Unit tests for ``galaxy.web.framework.webapp`` """ import logging -import os import re -import sys import unittest import galaxy.config from galaxy.web.framework import webapp as Webapp -unit_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) -sys.path.insert(1, unit_root) -from unittest_utils import galaxy_mock +from ...unittest_utils import galaxy_mock log = logging.getLogger(__name__) From 67e9dbc37aa41a873546098cad81fe4a2557281e Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 26 Sep 2017 07:57:37 -0400 Subject: [PATCH 65/66] Fix invalid requests imports introduced in #4604. Even when we remove the requests controller the pyc files will stick around - so probably best to keep these absolute imports for these controllers indefinitely. This broke workflow import from URL coming through the GUI and so it broke a bunch of Selenium tests. --- lib/galaxy/webapps/galaxy/controllers/async.py | 2 ++ lib/galaxy/webapps/galaxy/controllers/library_common.py | 2 ++ lib/galaxy/webapps/galaxy/controllers/root.py | 2 ++ lib/galaxy/webapps/galaxy/controllers/workflow.py | 2 ++ 4 files changed, 8 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/controllers/async.py b/lib/galaxy/webapps/galaxy/controllers/async.py index aabb551bdfc..f002e29a553 100644 --- a/lib/galaxy/webapps/galaxy/controllers/async.py +++ b/lib/galaxy/webapps/galaxy/controllers/async.py @@ -2,6 +2,8 @@ Upload class """ +from __future__ import absolute_import + import logging import urllib diff --git a/lib/galaxy/webapps/galaxy/controllers/library_common.py b/lib/galaxy/webapps/galaxy/controllers/library_common.py index 2a1ac0a09dc..955fede10f0 100644 --- a/lib/galaxy/webapps/galaxy/controllers/library_common.py +++ b/lib/galaxy/webapps/galaxy/controllers/library_common.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import + import glob import json import logging diff --git a/lib/galaxy/webapps/galaxy/controllers/root.py b/lib/galaxy/webapps/galaxy/controllers/root.py index a8d2df5f953..17b82a9251b 100644 --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -1,6 +1,8 @@ """ Contains the main interface in the Universe class """ +from __future__ import absolute_import + import cgi import os diff --git a/lib/galaxy/webapps/galaxy/controllers/workflow.py b/lib/galaxy/webapps/galaxy/controllers/workflow.py index 55706557e22..823078c8ecc 100644 --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import + import base64 import httplib import json From afe1ea7f20aa49ced11bb49d0d6322bfc6e00371 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 26 Sep 2017 09:40:23 -0400 Subject: [PATCH 66/66] Fix test_saved_histories.py for #4681. --- test/selenium_tests/test_saved_histories.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/test/selenium_tests/test_saved_histories.py b/test/selenium_tests/test_saved_histories.py index 4272ec46b93..792e76413be 100644 --- a/test/selenium_tests/test_saved_histories.py +++ b/test/selenium_tests/test_saved_histories.py @@ -57,14 +57,11 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase): self.click_popup_option('Unnamed history', 'Rename') # Rename the history - history_name_input = self.wait_for_selector('input[name="name"]') + history_name_input = self.wait_for_selector('.ui-form-element input.ui-input') history_name_input.clear() history_name_input.send_keys(self.history1_name) - self.send_enter(history_name_input) - message = self.wait_for_selector_visible('.infomessagelarge') - expected_message = 'History: Unnamed history renamed to: %s' % self.history1_name - self.assertEqual(expected_message, message.text) + self.wait_for_and_click_selector("button#submit") self.navigate_to_saved_histories_page()