diff --git a/.ci/flake8_lint_include_list.txt b/.ci/flake8_lint_include_list.txt index 96897866df4..0ddc6d8a6c2 100644 --- a/.ci/flake8_lint_include_list.txt +++ b/.ci/flake8_lint_include_list.txt @@ -483,7 +483,6 @@ test/api/test_tool_data.py test/api/test_tours.py test/api/test_workflow_extraction.py test/api/test_workflows_from_yaml.py -test/api/workflows_format_2/ test/base/api_asserts.py test/base/api.py test/base/api_util.py @@ -495,6 +494,7 @@ test/base/nose_util.py test/base/test_logging.py test/base/tool_shed_util.py test/base/twilltestcase.py +test/base/workflows_format_2/ test/casperjs/ test/functional/database_contexts.py test/functional/__init__.py diff --git a/test/api/test_dataset_collections.py b/test/api/test_dataset_collections.py index 7e38350956f..2327ebfc2d1 100644 --- a/test/api/test_dataset_collections.py +++ b/test/api/test_dataset_collections.py @@ -1,8 +1,7 @@ import json from base import api - -from .helpers import DatasetCollectionPopulator, DatasetPopulator +from base.populators import DatasetCollectionPopulator, DatasetPopulator class DatasetCollectionApiTestCase( api.ApiTestCase ): diff --git a/test/api/test_datasets.py b/test/api/test_datasets.py index 709838f5316..3a07f21cb67 100644 --- a/test/api/test_datasets.py +++ b/test/api/test_datasets.py @@ -3,8 +3,7 @@ from __future__ import print_function import textwrap from base import api - -from .helpers import TestsDatasets +from base.populators import TestsDatasets class DatasetsApiTestCase( api.ApiTestCase, TestsDatasets ): diff --git a/test/api/test_histories.py b/test/api/test_histories.py index 76b0df61a8c..f026a6fdddd 100644 --- a/test/api/test_histories.py +++ b/test/api/test_histories.py @@ -8,7 +8,7 @@ from requests import ( from base import api -from .helpers import DatasetPopulator, wait_on +from base.populators import DatasetPopulator, wait_on class HistoriesApiTestCase( api.ApiTestCase ): diff --git a/test/api/test_history_contents.py b/test/api/test_history_contents.py index ac2750f6a24..8670a15714f 100644 --- a/test/api/test_history_contents.py +++ b/test/api/test_history_contents.py @@ -5,8 +5,7 @@ import json from requests import delete, put from base import api - -from .helpers import DatasetCollectionPopulator, LibraryPopulator, TestsDatasets +from base.populators import DatasetCollectionPopulator, LibraryPopulator, TestsDatasets # TODO: Test anonymous access. diff --git a/test/api/test_history_contents_provenance.py b/test/api/test_history_contents_provenance.py index 4222cec0193..02672f0e5f5 100644 --- a/test/api/test_history_contents_provenance.py +++ b/test/api/test_history_contents_provenance.py @@ -1,6 +1,5 @@ from base import api - -from .helpers import DatasetPopulator +from base.populators import DatasetPopulator class TestProvenance( api.ApiTestCase ): diff --git a/test/api/test_jobs.py b/test/api/test_jobs.py index 18c850f65bc..53ce9a1c574 100644 --- a/test/api/test_jobs.py +++ b/test/api/test_jobs.py @@ -4,8 +4,7 @@ import time from operator import itemgetter from base import api - -from .helpers import TestsDatasets +from base.populators import TestsDatasets class JobsApiTestCase( api.ApiTestCase, TestsDatasets ): diff --git a/test/api/test_libraries.py b/test/api/test_libraries.py index a6fafdf0bf1..56869c940f7 100644 --- a/test/api/test_libraries.py +++ b/test/api/test_libraries.py @@ -1,6 +1,5 @@ from base import api - -from .helpers import ( +from base.populators import ( LibraryPopulator, TestsDatasets, wait_on_state diff --git a/test/api/test_search.py b/test/api/test_search.py index 25699b04978..55d1549b1b1 100644 --- a/test/api/test_search.py +++ b/test/api/test_search.py @@ -1,8 +1,7 @@ from base import api +from base.populators import WorkflowPopulator from requests import delete -from .helpers import WorkflowPopulator - class SearchApiTestCase( api.ApiTestCase ): diff --git a/test/api/test_tools.py b/test/api/test_tools.py index 7b0033caf24..11734a4c006 100644 --- a/test/api/test_tools.py +++ b/test/api/test_tools.py @@ -2,11 +2,14 @@ import json from base import api +from base.populators import ( + DatasetCollectionPopulator, + DatasetPopulator, + LibraryPopulator, + skip_without_tool +) from galaxy.tools.verify.test_data import TestDataResolver -from .helpers import (DatasetCollectionPopulator, DatasetPopulator, - LibraryPopulator, skip_without_tool) - class ToolsTestCase( api.ApiTestCase ): diff --git a/test/api/test_workflow_extraction.py b/test/api/test_workflow_extraction.py index 53864393edd..afca15dfb9f 100644 --- a/test/api/test_workflow_extraction.py +++ b/test/api/test_workflow_extraction.py @@ -5,7 +5,8 @@ import operator from collections import namedtuple from json import dumps, loads -from .helpers import skip_without_tool +from base.populators import skip_without_tool + from .test_workflows import BaseWorkflowsApiTestCase diff --git a/test/api/test_workflows.py b/test/api/test_workflows.py index c3e00526b46..eca1f9f0ca5 100644 --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -9,12 +9,16 @@ import yaml from requests import delete, put from base import api +from base.populators import ( + DatasetCollectionPopulator, + DatasetPopulator, + skip_without_tool, + WorkflowPopulator +) from galaxy.exceptions import error_codes from galaxy.tools.verify.test_data import TestDataResolver -from .helpers import (DatasetCollectionPopulator, DatasetPopulator, - skip_without_tool, WorkflowPopulator) -from .workflows_format_2 import ( +from base.workflows_format_2 import ( convert_and_import_workflow, ImporterGalaxyInterface, ) @@ -1216,11 +1220,9 @@ test_data: def test_workflow_stability( self ): # Run this index stability test with following command: # ./run_tests.sh test/api/test_workflows.py:WorkflowsApiTestCase.test_workflow_stability - from pkg_resources import resource_string num_tests = 1 - for workflow_file in [ "test_workflow_topoambigouity.ga", "test_workflow_topoambigouity_auto_laidout.ga" ]: - workflow_str = resource_string( __name__, workflow_file ) - workflow = self.workflow_populator.load_workflow( "test1", content=workflow_str ) + for workflow_file in [ "test_workflow_topoambigouity", "test_workflow_topoambigouity_auto_laidout" ]: + workflow = self.workflow_populator.load_workflow_from_resource( workflow_file ) last_step_map = self._step_map( workflow ) for i in range(num_tests): uploaded_workflow_id = self.workflow_populator.create_workflow( workflow ) diff --git a/test/api/test_workflow_1.ga b/test/base/data/test_workflow_1.ga similarity index 100% rename from test/api/test_workflow_1.ga rename to test/base/data/test_workflow_1.ga diff --git a/test/api/test_workflow_2.ga b/test/base/data/test_workflow_2.ga similarity index 100% rename from test/api/test_workflow_2.ga rename to test/base/data/test_workflow_2.ga diff --git a/test/api/test_workflow_batch.ga b/test/base/data/test_workflow_batch.ga similarity index 100% rename from test/api/test_workflow_batch.ga rename to test/base/data/test_workflow_batch.ga diff --git a/test/api/test_workflow_map_reduce_pause.ga b/test/base/data/test_workflow_map_reduce_pause.ga similarity index 100% rename from test/api/test_workflow_map_reduce_pause.ga rename to test/base/data/test_workflow_map_reduce_pause.ga diff --git a/test/api/test_workflow_matching_lists.ga b/test/base/data/test_workflow_matching_lists.ga similarity index 100% rename from test/api/test_workflow_matching_lists.ga rename to test/base/data/test_workflow_matching_lists.ga diff --git a/test/api/test_workflow_missing_tool.ga b/test/base/data/test_workflow_missing_tool.ga similarity index 100% rename from test/api/test_workflow_missing_tool.ga rename to test/base/data/test_workflow_missing_tool.ga diff --git a/test/api/test_workflow_pause.ga b/test/base/data/test_workflow_pause.ga similarity index 100% rename from test/api/test_workflow_pause.ga rename to test/base/data/test_workflow_pause.ga diff --git a/test/api/test_workflow_topoambigouity.ga b/test/base/data/test_workflow_topoambigouity.ga similarity index 100% rename from test/api/test_workflow_topoambigouity.ga rename to test/base/data/test_workflow_topoambigouity.ga diff --git a/test/api/test_workflow_topoambigouity_auto_laidout.ga b/test/base/data/test_workflow_topoambigouity_auto_laidout.ga similarity index 100% rename from test/api/test_workflow_topoambigouity_auto_laidout.ga rename to test/base/data/test_workflow_topoambigouity_auto_laidout.ga diff --git a/test/api/test_workflow_validation_1.ga b/test/base/data/test_workflow_validation_1.ga similarity index 100% rename from test/api/test_workflow_validation_1.ga rename to test/base/data/test_workflow_validation_1.ga diff --git a/test/api/test_workflow_with_runtime_input.ga b/test/base/data/test_workflow_with_runtime_input.ga similarity index 100% rename from test/api/test_workflow_with_runtime_input.ga rename to test/base/data/test_workflow_with_runtime_input.ga diff --git a/test/api/helpers.py b/test/base/populators.py similarity index 91% rename from test/api/helpers.py rename to test/base/populators.py index 36b93024889..72e7f285008 100644 --- a/test/api/helpers.py +++ b/test/base/populators.py @@ -1,17 +1,20 @@ import json import time + from operator import itemgetter +import requests + from pkg_resources import resource_string from six import StringIO from base import api_asserts # Simple workflow that takes an input and call cat wrapper on it. -workflow_str = resource_string( __name__, "test_workflow_1.ga" ) +workflow_str = resource_string( __name__, "data/test_workflow_1.ga" ) # Simple workflow that takes an input and filters with random lines twice in a # row - first grabbing 8 lines at random and then 6. -workflow_random_x2_str = resource_string( __name__, "test_workflow_2.ga" ) +workflow_random_x2_str = resource_string( __name__, "data/test_workflow_2.ga" ) DEFAULT_TIMEOUT = 15 # Secs to wait for state to turn ok @@ -229,7 +232,7 @@ class BaseWorkflowPopulator( object ): def load_workflow_from_resource( self, name, filename=None ): if filename is None: - filename = "%s.ga" % name + filename = "data/%s.ga" % name content = resource_string( __name__, filename ) return self.load_workflow( name, content=content ) @@ -476,6 +479,54 @@ def wait_on_state( state_func, assert_ok=False, timeout=DEFAULT_TIMEOUT ): return wait_on( get_state, desc="state", timeout=timeout) +class GiPostGetMixin: + """Mixin for adapting Galaxy testing populators helpers to bioblend.""" + + def _get(self, route): + return self._gi.make_get_request(self.__url(route)) + + def _post(self, route, data={}): + data = data.copy() + data['key'] = self._gi.key + return requests.post(self.__url(route), data=data) + + def __url(self, route): + return self._gi.url + "/" + route + + +class GiDatasetPopulator(BaseDatasetPopulator, GiPostGetMixin): + + """Implementation of BaseDatasetPopulator backed by bioblend.""" + + def __init__(self, gi): + """Construct a dataset populator from a bioblend GalaxyInstance.""" + self._gi = gi + + +class GiDatasetCollectionPopulator(BaseDatasetCollectionPopulator, GiPostGetMixin): + + """Implementation of BaseDatasetCollectionPopulator backed by bioblend.""" + + def __init__(self, gi): + """Construct a dataset collection populator from a bioblend GalaxyInstance.""" + self._gi = gi + self.dataset_populator = GiDatasetPopulator(gi) + + def _create_collection(self, payload): + create_response = self._post( "dataset_collections", data=payload ) + return create_response + + +class GiWorkflowPopulator(BaseWorkflowPopulator, GiPostGetMixin): + + """Implementation of BaseWorkflowPopulator backed by bioblend.""" + + def __init__(self, gi): + """Construct a workflow populator from a bioblend GalaxyInstance.""" + self._gi = gi + self.dataset_populator = GiDatasetPopulator(gi) + + def wait_on( function, desc, timeout=DEFAULT_TIMEOUT ): delta = .25 iteration = 0 diff --git a/test/api/workflows_format_2/README.txt b/test/base/workflows_format_2/README.txt similarity index 100% rename from test/api/workflows_format_2/README.txt rename to test/base/workflows_format_2/README.txt diff --git a/test/api/workflows_format_2/__init__.py b/test/base/workflows_format_2/__init__.py similarity index 100% rename from test/api/workflows_format_2/__init__.py rename to test/base/workflows_format_2/__init__.py diff --git a/test/api/workflows_format_2/converter.py b/test/base/workflows_format_2/converter.py similarity index 100% rename from test/api/workflows_format_2/converter.py rename to test/base/workflows_format_2/converter.py diff --git a/test/api/workflows_format_2/interface.py b/test/base/workflows_format_2/interface.py similarity index 100% rename from test/api/workflows_format_2/interface.py rename to test/base/workflows_format_2/interface.py diff --git a/test/api/workflows_format_2/main.py b/test/base/workflows_format_2/main.py similarity index 100% rename from test/api/workflows_format_2/main.py rename to test/base/workflows_format_2/main.py diff --git a/test/manual/workflows_scaling.py b/test/manual/workflows_scaling.py index 354c8a5f9ed..f9cbaac7a0d 100644 --- a/test/manual/workflows_scaling.py +++ b/test/manual/workflows_scaling.py @@ -14,13 +14,16 @@ from argparse import ArgumentParser from threading import Thread from uuid import uuid4 -import requests from bioblend import galaxy galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)) sys.path[1:1] = [ os.path.join( galaxy_root, "lib" ), os.path.join( galaxy_root, "test" ) ] -from api import helpers +from base.populators import ( + GiDatasetCollectionPopulator, + GiDatasetPopulator, + GiWorkflowPopulator, +) from api.workflows_format_2.converter import python_to_workflow LONG_TIMEOUT = 1000000000 @@ -109,51 +112,6 @@ def _run(args, gi, workflow_id, uuid): ) -class GiPostGetMixin: - """Mixin for adapting Galaxy API testing helpers to bioblend.""" - - def _get(self, route): - return self._gi.make_get_request(self.__url(route)) - - def _post(self, route, data={}): - data = data.copy() - data['key'] = self._gi.key - return requests.post(self.__url(route), data=data) - - def __url(self, route): - return self._gi.url + "/" + route - - -class GiDatasetPopulator(helpers.BaseDatasetPopulator, GiPostGetMixin): - """Utility class for dealing with datasets and histories.""" - - def __init__(self, gi): - """Construct a dataset populator from a bioblend GalaxyInstance.""" - self._gi = gi - - -class GiDatasetCollectionPopulator(helpers.BaseDatasetCollectionPopulator, GiPostGetMixin): - """Utility class for dealing with dataset collections.""" - - def __init__(self, gi): - """Construct a dataset collection populator from a bioblend GalaxyInstance.""" - self._gi = gi - self.dataset_populator = GiDatasetPopulator(gi) - - def _create_collection(self, payload): - create_response = self._post( "dataset_collections", data=payload ) - return create_response - - -class GiWorkflowPopulator(helpers.BaseWorkflowPopulator, GiPostGetMixin): - """Utility class for dealing with workflows.""" - - def __init__(self, gi): - """Construct a workflow populator from a bioblend GalaxyInstance.""" - self._gi = gi - self.dataset_populator = GiDatasetPopulator(gi) - - def _workflow_struct(args, input_uuid): if args.two_outputs: return _workflow_struct_two_outputs(args, input_uuid)