From 0c2b8b9216b12305a818afb233fc1138a646c623 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 19 Sep 2019 07:13:45 -0400 Subject: [PATCH] Improve CI reliability by annotating external resources. And skipping if github, usegalaxy.org, or the tool shed is down for tests that depend on these. --- test/api/test_dataset_collections.py | 3 +- test/api/test_tools_upload.py | 3 ++ test/base/populators.py | 28 +++++++++++++++++-- .../test_data_manager_table_reload.py | 3 +- test/integration/test_shed_tool_tests.py | 2 ++ test/selenium_tests/test_tool_form.py | 3 +- .../test_workflow_management.py | 3 ++ 7 files changed, 40 insertions(+), 5 deletions(-) diff --git a/test/api/test_dataset_collections.py b/test/api/test_dataset_collections.py index 946114e51af..a719af89ac1 100644 --- a/test/api/test_dataset_collections.py +++ b/test/api/test_dataset_collections.py @@ -2,7 +2,7 @@ import json import tarfile from base import api -from base.populators import DatasetCollectionPopulator, DatasetPopulator +from base.populators import DatasetCollectionPopulator, DatasetPopulator, skip_if_github_down from six import BytesIO @@ -233,6 +233,7 @@ class DatasetCollectionApiTestCase(api.ApiTestCase): element0 = hdca["elements"][0] assert element0["element_identifier"] == "samp1" + @skip_if_github_down def test_upload_collection_from_url(self): elements = [{"src": "url", "url": "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed", "info": "my cool bed"}] targets = [{ diff --git a/test/api/test_tools_upload.py b/test/api/test_tools_upload.py index 9366f219e79..a6cde5cb9ae 100644 --- a/test/api/test_tools_upload.py +++ b/test/api/test_tools_upload.py @@ -8,6 +8,7 @@ from base.constants import ( ) from base.populators import ( DatasetPopulator, + skip_if_site_down, skip_without_datatype, uses_test_history, ) @@ -484,11 +485,13 @@ class ToolsUploadTestCase(api.ApiTestCase): assert extra_file["path"] == "composite" assert extra_file["class"] == "File" + @skip_if_site_down("https://usegalaxy.org") def test_upload_from_invalid_url(self): history_id, new_dataset = self._upload('https://usegalaxy.org/bla123', assert_ok=False) dataset_details = self.dataset_populator.get_history_dataset_details(history_id, dataset_id=new_dataset["id"], assert_ok=False) assert dataset_details['state'] == 'error', "expected dataset state to be 'error', but got '%s'" % dataset_details['state'] + @skip_if_site_down("https://usegalaxy.org") def test_upload_from_valid_url(self): history_id, new_dataset = self._upload('https://usegalaxy.org/api/version') self.dataset_populator.get_history_dataset_details(history_id, dataset_id=new_dataset["id"], assert_ok=True) diff --git a/test/base/populators.py b/test/base/populators.py index bc4be223902..88fb63530f6 100644 --- a/test/base/populators.py +++ b/test/base/populators.py @@ -106,6 +106,30 @@ def skip_without_datatype(extension): return method_wrapper +def skip_if_site_down(url): + + def site_down(): + try: + response = requests.get(url) + return response.status_code != 200 + except Exception: + return False + + def method_wrapper(method): + @wraps(method) + def wrapped_method(api_test_case, *args, **kwargs): + _raise_skip_if(site_down(), "Test depends on [%s] being up and it appears to be down." % url) + method(api_test_case, *args, **kwargs) + + return wrapped_method + + return method_wrapper + + +skip_if_toolshed_down = skip_if_site_down("https://toolshed.g2.bx.psu.edu") +skip_if_github_down = skip_if_site_down("https://github.com/") + + def summarize_instance_history_on_error(method): @wraps(method) def wrapped_method(api_test_case, *args, **kwds): @@ -134,10 +158,10 @@ def uses_test_history(**test_history_kwd): return method_wrapper -def _raise_skip_if(check): +def _raise_skip_if(check, *args): if check: from nose.plugins.skip import SkipTest - raise SkipTest() + raise SkipTest(*args) # Deprecated mixin, use dataset populator instead. diff --git a/test/integration/test_data_manager_table_reload.py b/test/integration/test_data_manager_table_reload.py index f63a503cfd0..3b8e32596aa 100644 --- a/test/integration/test_data_manager_table_reload.py +++ b/test/integration/test_data_manager_table_reload.py @@ -2,7 +2,7 @@ import random import string from base import integration_util -from base.populators import DatasetPopulator +from base.populators import DatasetPopulator, skip_if_toolshed_down from nose.plugins.skip import SkipTest from .uses_shed import CONDA_AUTO_INSTALL_JOB_TIMEOUT, UsesShed @@ -43,6 +43,7 @@ class DataManagerIntegrationTestCase(integration_util.IntegrationTestCase, UsesS cls.username = cls.get_secure_ascii_digits() config["admin_users"] = "%s@galaxy.org" % cls.username + @skip_if_toolshed_down def test_data_manager_installation_table_reload(self): """ Test that we can install data managers, create a new dbkey, and use that dbkey in a downstream data manager. diff --git a/test/integration/test_shed_tool_tests.py b/test/integration/test_shed_tool_tests.py index 1ed0015e9cb..66d84bde41c 100644 --- a/test/integration/test_shed_tool_tests.py +++ b/test/integration/test_shed_tool_tests.py @@ -1,4 +1,5 @@ from base import integration_util +from base.populators import skip_if_toolshed_down from .uses_shed import UsesShed @@ -13,6 +14,7 @@ class ToolShedToolTestIntegrationTestCase(integration_util.IntegrationTestCase, def handle_galaxy_config_kwds(cls, config): cls.configure_shed_and_conda(config) + @skip_if_toolshed_down def test_tool_test(self): self.install_repository("devteam", "fastqc", "ff9530579d1f") self._run_tool_test("toolshed.g2.bx.psu.edu/repos/devteam/fastqc/fastqc/0.71") diff --git a/test/selenium_tests/test_tool_form.py b/test/selenium_tests/test_tool_form.py index f8aef81abf2..4ba932dcc69 100644 --- a/test/selenium_tests/test_tool_form.py +++ b/test/selenium_tests/test_tool_form.py @@ -1,7 +1,7 @@ import json from base import rules_test_data -from base.populators import flakey, load_data_dict +from base.populators import flakey, load_data_dict, skip_if_github_down from galaxy.selenium.navigates_galaxy import retry_call_during_transitions from .framework import ( @@ -185,6 +185,7 @@ class LoggedInToolFormTestCase(SeleniumTestCase): @selenium_test @managed_history + @skip_if_github_down def test_run_apply_rules_tutorial(self): self.home() self.upload_rule_start() diff --git a/test/selenium_tests/test_workflow_management.py b/test/selenium_tests/test_workflow_management.py index 3f7d614880b..2041836ba01 100644 --- a/test/selenium_tests/test_workflow_management.py +++ b/test/selenium_tests/test_workflow_management.py @@ -1,3 +1,5 @@ +from base.populators import skip_if_github_down + from .framework import ( retry_assertion_during_transitions, selenium_test, @@ -108,6 +110,7 @@ class WorkflowManagementTestCase(SeleniumTestCase): def _assert_showing_n_workflows(self, n): self.assertEqual(len(self.workflow_index_table_elements()), n) + @skip_if_github_down def _workflow_import_from_url(self): self.workflow_index_click_import() url = "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test/base/data/test_workflow_1.ga"