Merge branch 'dev' into workflow_run_performance

client/galaxy/scripts/mvc/ui/ui-select-genomespace.js

@@ -26,16 +26,10 @@ var View = Backbone.View.extend({
         // create genomespace filepath textbox
         this.filename_textbox = new Ui.Input();
 
-        // create genomespace token textbox
-        this.token_textbox = new Ui.Input({
-            type: "password"
-        });
-
         // create elements
         this.setElement(this._template(options));
         this.$(".ui-gs-browse-button").append(this.browse_button.$el);
         this.$(".ui-gs-filename-textbox").append(this.filename_textbox.$el);
-        this.$(".ui-gs-token-textbox").append(this.token_textbox.$el);
     },
 
     /** Browse GenomeSpace */
@@ -43,7 +37,7 @@ var View = Backbone.View.extend({
         var self = this;
         GenomespaceBrowser.openFileBrowser({
             successCallback: function(data) {
-                self.value(`${data.destination}^${data.token}`);
+                self.value(data.destination);
             }
         });
     },
@@ -56,15 +50,11 @@ var View = Backbone.View.extend({
             '<span class="ui-gs-browse-button" />' +
             '<span class="ui-gs-filename-textbox" />' +
             "</div>" +
-            '<div class="ui-gs-token-field">' +
-            '<span class=ui-gs-label"><div class="ui-gs-token-label">Token</div></span>' +
-            '<span class="ui-gs-token-textbox" />' +
-            "</div>" +
             "</div>"
         );
     },
 
-    /** Return/Set currently selected genomespace filename/token */
+    /** Return/Set currently selected genomespace filename */
     value: function(new_value) {
         // check if new_value is defined
         if (new_value !== undefined) {
@@ -76,15 +66,13 @@ var View = Backbone.View.extend({
 
     // get value
     _getValue: function() {
-        return `${this.filename_textbox.value()}^${this.token_textbox.value()}`;
+        return this.filename_textbox.value();
     },
 
     // set value
     _setValue: function(new_value) {
         if (new_value) {
-            var values = new_value.split("^");
-            this.filename_textbox.value(values[0]);
-            this.token_textbox.value(values[1]);
+            this.filename_textbox.value(new_value);
         }
         this.options.onchange && this.options.onchange(new_value);
     }

client/galaxy/style/less/ui.less

@@ -909,10 +909,6 @@
         float: right;
         width: ~'calc(100% - 76px)';
     }
-    .ui-gs-token-textbox {
-        float: right;
-        width: ~'calc(100% - 76px)';
-    }
     .ui-gs-browse-button {
         float: left;
         .ui-button-icon {
@@ -920,15 +916,9 @@
             margin-right: 5px;
         }
     }
-    .ui-gs-token-label {
-        margin-top: 5px;            
-        margin-left: 32px;
-        margin-right: 5px;
-        float: left;
-    }
-    .ui-gs-token-field {
-        clear: both;
-        padding-top: 5px;
+    .ui-gs-browse-field {
+        height: @input-height-base;
+        line-height: @line-height-base;
     }
 }
 

config/galaxy.yml.sample

@@ -1167,6 +1167,15 @@ galaxy:
   # Set maximum size of ngrams
   #tool_ngram_maxsize: 4
 
+  # Set tool test data directory. The test framework sets this value to
+  # 'test-data,https://github.com/galaxyproject/galaxy-test-data.git'
+  # which will cause Galaxy to clone down extra test data on the fly for
+  # certain tools distributed with Galaxy but this is likely not
+  # appropriate for production systems. Instead one can simply clone
+  # that repository directly and specify a path here instead of a Git
+  # HTTP repository.
+  #tool_test_data_directories: test-data
+
   # Galaxy encodes various internal values when these values will be
   # output in some format (for example, in a URL or cookie).  You should
   # set a key to be used by the algorithm that encodes and decodes these

config/plugins/webhooks/demo/tour_generator/__init__.py

@@ -64,26 +64,13 @@ class TourGenerator(object):
                 self._use_datasets = False
                 return
 
-        test_data_paths = [os.path.abspath('test-data')]
-        test_data_cache_dir = os.path.abspath(
-            os.environ.get('GALAXY_TEST_DATA_REPO_CACHE', 'test-data-cache'))
-        test_data_paths.extend([
-            x[0] for x in os.walk(test_data_cache_dir) if '.git' not in x[0]])
-        if self._tool.tool_shed:
-            test_data_paths.append(os.path.abspath(os.path.join(
-                self._tool.tool_dir, 'test-data')))
-
         # Upload all test datasets
         for input_name, input in self._data_inputs.items():
             if input_name in test_datasets.keys():
-                for i, data_path in enumerate(test_data_paths):
-                    input_path = os.path.join(data_path,
-                                              test_datasets[input_name])
-                    if os.path.exists(input_path):
-                        break
-                    elif i + 1 == len(test_data_paths):  # the last path
-                        raise ValueError('Test dataset "%s" doesn\'t exist.' %
-                                         input_name)
+                filename = test_datasets[input_name]
+                input_path = self._tool.test_data_path(filename)
+                if not input_path:
+                    raise ValueError('Test dataset "%s" doesn\'t exist.' % input_name)
 
                 upload_tool = self._trans.app.toolbox.get_tool('upload1')
                 filename = os.path.basename(input_path)

doc/source/admin/galaxy_options.rst

@@ -2439,6 +2439,22 @@
 :Type: int
 
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``tool_test_data_directories``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Set tool test data directory. The test framework sets this value
+    to 'test-data,https://github.com/galaxyproject/galaxy-test-
+    data.git' which will cause Galaxy to clone down extra test data on
+    the fly for certain tools distributed with Galaxy but this is
+    likely not appropriate for production systems. Instead one can
+    simply clone that repository directly and specify a path here
+    instead of a Git HTTP repository.
+:Default: ``test-data``
+:Type: str
+
+
 ~~~~~~~~~~~~~
 ``id_secret``
 ~~~~~~~~~~~~~

lib/galaxy/app.py

@@ -12,6 +12,7 @@ import galaxy.security
 from galaxy import config, jobs
 from galaxy.jobs import metrics as job_metrics
 from galaxy.managers.collections import DatasetCollectionManager
+from galaxy.managers.histories import HistoryManager
 from galaxy.managers.tags import GalaxyTagManager
 from galaxy.openid.providers import OpenIDProviders
 from galaxy.queue_worker import GalaxyQueueWorker
@@ -20,8 +21,10 @@ from galaxy.tools.cache import (
     ToolShedRepositoryCache
 )
 from galaxy.tools.data_manager.manager import DataManagers
+from galaxy.tools.deps.views import DependencyResolversView
 from galaxy.tools.error_reports import ErrorReports
 from galaxy.tools.special_tools import load_lib_tools
+from galaxy.tools.verify import test_data
 from galaxy.tours import ToursRegistry
 from galaxy.util import (
     ExecutionTimer,
@@ -90,6 +93,9 @@ class UniverseApplication(config.ConfiguresGalaxyMixin):
         self.tag_handler = GalaxyTagManager(self.model.context)
         # Dataset Collection Plugins
         self.dataset_collections_service = DatasetCollectionManager(self)
+        self.history_manager = HistoryManager(self)
+        self.dependency_resolvers_view = DependencyResolversView(self)
+        self.test_data_resolver = test_data.TestDataResolver(file_dirs=self.config.tool_test_data_directories)
 
         # Tool Data Tables
         self._configure_tool_data_tables(from_shed_config=False)

lib/galaxy/config.py

@@ -462,6 +462,8 @@ class Configuration(object):
         self.tool_enable_ngram_search = kwargs.get("tool_enable_ngram_search", False)
         self.tool_ngram_minsize = kwargs.get("tool_ngram_minsize", 3)
         self.tool_ngram_maxsize = kwargs.get("tool_ngram_maxsize", 4)
+        default_tool_test_data_directories = os.environ.get("GALAXY_TEST_FILE_DIR", resolve_path("test-data", self.root))
+        self.tool_test_data_directories = kwargs.get("tool_test_data_directories", default_tool_test_data_directories)
         # Location for tool dependencies.
         use_tool_dependencies, tool_dependency_dir, use_cached_dependency_manager, tool_dependency_cache_dir, precache_dependencies = \
             parse_dependency_options(kwargs, self.root, self.dependency_resolvers_config_file)

lib/galaxy/datatypes/binary.py

@@ -506,6 +506,10 @@ class BamInputSorted(BamNative):
     This notaby keeps alignments produced by paired end sequencing adjacent.
     """
 
+    def sniff(self, file_name):
+        # We never want to sniff to this datatype
+        return False
+
     def dataset_content_needs_grooming(self, file_name):
         """
         Groom if the file is coordinate sorted

lib/galaxy/dependencies/pinned-requirements.txt

@@ -15,6 +15,7 @@ uWSGI==2.0.15
 pysam==0.14
 
 # pure Python packages
+bleach==2.1.3
 bz2file==0.98; python_version < '3.3'
 ipaddress==1.0.18; python_version < '3.3'
 boltons==17.1.0

lib/galaxy/dependencies/requirements.txt

@@ -13,6 +13,7 @@ pycrypto
 pysam>=0.13
 
 # pure Python packages
+bleach
 bz2file; python_version < '3.3'
 ipaddress; python_version < '3.3'
 boltons

lib/galaxy/managers/workflows.py

@@ -241,7 +241,7 @@ class WorkflowContentsManager(UsesAnnotations):
             stored.user = trans.user
             stored.published = publish
             if data['annotation']:
-                annotation = sanitize_html(data['annotation'], 'utf-8', 'text/html')
+                annotation = sanitize_html(data['annotation'])
                 self.add_item_annotation(trans.sa_session, stored.user, stored, annotation)
             workflow_tags = data.get('tags', [])
             trans.app.tag_handler.set_tags_from_list(user=trans.user, item=stored, new_tags_list=workflow_tags)
@@ -864,7 +864,7 @@ class WorkflowContentsManager(UsesAnnotations):
 
         annotation = step_dict['annotation']
         if annotation:
-            annotation = sanitize_html(annotation, 'utf-8', 'text/html')
+            annotation = sanitize_html(annotation)
             self.add_item_annotation(trans.sa_session, trans.get_user(), step, annotation)
 
         # Stick this in the step temporarily

lib/galaxy/tools/__init__.py

@@ -26,7 +26,6 @@ from galaxy import (
     model
 )
 from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper
-from galaxy.managers import histories
 from galaxy.managers.jobs import JobSearch
 from galaxy.managers.tags import GalaxyTagManager
 from galaxy.queue_worker import send_control_task
@@ -36,7 +35,6 @@ from galaxy.tools.actions.data_source import DataSourceToolAction
 from galaxy.tools.actions.model_operations import ModelOperationToolAction
 from galaxy.tools.deps import (
     CachedDependencyManager,
-    views
 )
 from galaxy.tools.fetcher import ToolLocationFetcher
 from galaxy.tools.parameters import (
@@ -69,6 +67,7 @@ from galaxy.tools.test import parse_tests
 from galaxy.tools.toolbox import BaseGalaxyToolBox
 from galaxy.util import (
     ExecutionTimer,
+    in_directory,
     listify,
     Params,
     rst_to_html,
@@ -439,13 +438,19 @@ class Tool(Dictifiable):
         except Exception as e:
             global_tool_errors.add_error(config_file, "Tool Loading", e)
             raise e
-        self.history_manager = histories.HistoryManager(app)
-        self._view = views.DependencyResolversView(app)
         # The job search is only relevant in a galaxy context, and breaks
         # loading tools into the toolshed for validation.
         if self.app.name == 'galaxy':
             self.job_search = JobSearch(app=self.app)
 
+    @property
+    def history_manager(self):
+        return self.app.history_manager
+
+    @property
+    def _view(self):
+        return self.app.dependency_resolvers_view
+
     @property
     def version_object(self):
         return packaging.version.parse(self.version)
@@ -826,6 +831,44 @@ class Tool(Dictifiable):
             self.__tests_populated = True
         return self.__tests
 
+    @property
+    def _repository_dir(self):
+        """If tool shed installed tool, the base directory of the repository installed."""
+        repository_dir = None
+
+        if hasattr(self, 'tool_shed') and self.tool_shed:
+            repository_dir = self.tool_dir
+            while True:
+                repository_dir_name = os.path.basename(repository_dir)
+                if repository_dir_name == self.repository_name:
+                    break
+
+                parent_repository_dir = os.path.dirname(repository_dir)
+                if repository_dir == parent_repository_dir:
+                    log.error("Problem finding repository dir for tool [%s]" % self.id)
+                    repository_dir = None
+
+        return repository_dir
+
+    def test_data_path(self, filename):
+        repository_dir = self._repository_dir
+        if repository_dir:
+            for root, dirs, files in os.walk(repository_dir):
+                if '.hg' in dirs:
+                    dirs.remove('.hg')
+                if 'test-data' in dirs:
+                    test_data_dir = os.path.join(root, 'test-data')
+                    result = os.path.abspath(os.path.join(test_data_dir, filename))
+                    if not in_directory(result, test_data_dir):
+                        # Don't raise an explicit exception and reveal details about what
+                        # files are or are not on the path, simply return None and let the
+                        # API raise a 404.
+                        return None
+                    else:
+                        return result
+        else:
+            return self.app.test_data_resolver.get_filename(filename)
+
     def tool_provided_metadata(self, job_wrapper):
         meta_file = os.path.join(job_wrapper.tool_working_directory, self.provided_metadata_file)
         # LEGACY: Remove in 17.XX

lib/galaxy/tools/deps/mulled/mulled_build.py

@@ -18,6 +18,7 @@ import subprocess
 import sys
 from sys import platform as _platform
 
+from six.moves import shlex_quote
 try:
     import yaml
 except ImportError:
@@ -47,6 +48,7 @@ DEFAULT_WORKING_DIR = '/source/'
 IS_OS_X = _platform == "darwin"
 INVOLUCRO_VERSION = "1.1.2"
 DEST_BASE_IMAGE = os.environ.get('DEST_BASE_IMAGE', None)
+CONDA_IMAGE = os.environ.get('CONDA_IMAGE', None)
 
 SINGULARITY_TEMPLATE = """Bootstrap: docker
 From: bgruening/busybox-bash:0.1
@@ -195,7 +197,7 @@ def mull_targets(
     involucro_args = [
         '-f', '%s/invfile.lua' % DIRNAME,
         '-set', "CHANNELS='%s'" % channels,
-        '-set', "TEST='%s'" % test,
+        '-set', "TEST=%s" % shlex_quote(test),
         '-set', "TARGETS='%s'" % target_str,
         '-set', "REPO='%s'" % repo,
         '-set', "BINDS='%s'" % bind_str,
@@ -203,6 +205,8 @@ def mull_targets(
 
     if DEST_BASE_IMAGE:
         involucro_args.extend(["-set", "DEST_BASE_IMAGE='%s'" % DEST_BASE_IMAGE])
+    if CONDA_IMAGE:
+        involucro_args.extend(["-set", "CONDA_IMAGE='%s'" % CONDA_IMAGE])
     if verbose:
         involucro_args.extend(["-set", "VERBOSE='1'"])
     if singularity:

lib/galaxy/tools/parser/interface.py

@@ -357,26 +357,77 @@ class ToolStdioExitCode(object):
 
 
 class TestCollectionDef(object):
-    # TODO: do not require XML directly here.
 
-    def __init__(self, elem, parse_param_elem):
-        self.elements = []
+    def __init__(self, attrib, name, collection_type, elements):
+        self.attrib = attrib
+        self.collection_type = collection_type
+        self.elements = elements
+        self.name = name
+
+    @staticmethod
+    def from_xml(elem, parse_param_elem):
+        elements = []
         attrib = dict(elem.attrib)
-        self.collection_type = attrib["type"]
-        self.name = attrib.get("name", "Unnamed Collection")
+        collection_type = attrib["type"]
+        name = attrib.get("name", "Unnamed Collection")
         for element in elem.findall("element"):
             element_attrib = dict(element.attrib)
             element_identifier = element_attrib["name"]
             nested_collection_elem = element.find("collection")
             if nested_collection_elem is not None:
-                self.elements.append((element_identifier, TestCollectionDef(nested_collection_elem, parse_param_elem)))
+                element_definition = TestCollectionDef.from_xml(nested_collection_elem, parse_param_elem)
             else:
-                self.elements.append((element_identifier, parse_param_elem(element)))
+                element_definition = parse_param_elem(element)
+            elements.append({"element_identifier": element_identifier, "element_definition": element_definition})
+
+        return TestCollectionDef(
+            attrib=attrib,
+            collection_type=collection_type,
+            elements=elements,
+            name=name,
+        )
+
+    def to_dict(self):
+        def element_to_dict(element_dict):
+            element_identifier, element_def = element_dict["element_identifier"], element_dict["element_definition"]
+            if isinstance(element_def, TestCollectionDef):
+                element_def = element_def.to_dict()
+            return {
+                "element_identifier": element_identifier,
+                "element_definition": element_def,
+            }
+
+        return {
+            "model_class": "TestCollectionDef",
+            "attributes": self.attrib,
+            "collection_type": self.collection_type,
+            "elements": map(element_to_dict, self.elements or []),
+            "name": self.name,
+        }
+
+    @staticmethod
+    def from_dict(as_dict):
+        assert as_dict["model_class"] == "TestCollectionDef"
+
+        def element_from_dict(element_dict):
+            if "element_definition" not in element_dict:
+                raise Exception("Invalid element_dict %s" % element_dict)
+            element_def = element_dict["element_definition"]
+            if element_def.get("model_class", None) == "TestCollectionDef":
+                element_def = TestCollectionDef.from_dict(element_def)
+            return {"element_identifier": element_dict["element_identifier"], "element_definition": element_def}
+
+        return TestCollectionDef(
+            attrib=as_dict["attributes"],
+            name=as_dict["name"],
+            elements=list(map(element_from_dict, as_dict["elements"] or [])),
+            collection_type=as_dict["collection_type"],
+        )
 
     def collect_inputs(self):
         inputs = []
         for element in self.elements:
-            value = element[1]
+            value = element["element_definition"]
             if isinstance(value, TestCollectionDef):
                 inputs.extend(value.collect_inputs())
             else:
@@ -393,3 +444,18 @@ class TestCollectionOutputDef(object):
         self.count = int(count) if count is not None else None
         self.attrib = attrib
         self.element_tests = element_tests
+
+    @staticmethod
+    def from_dict(as_dict):
+        return TestCollectionOutputDef(
+            name=as_dict["name"],
+            attrib=as_dict["attributes"],
+            element_tests=as_dict["element_tests"],
+        )
+
+    def to_dict(self):
+        return dict(
+            name=self.name,
+            attributes=self.attrib,
+            element_tests=self.element_tests
+        )

lib/galaxy/tools/parser/xml.py

@@ -391,8 +391,6 @@ class XmlToolSource(ToolSource):
             for i, test_elem in enumerate(tests_elem.findall("test")):
                 tests.append(_test_elem_to_dict(test_elem, i))
 
-            _copy_to_dict_if_present(tests_elem, rval, ["interactor"])
-
         return rval
 
     def parse_profile(self):
@@ -418,7 +416,7 @@ def _test_elem_to_dict(test_elem, i):
         expect_failure=string_as_bool(test_elem.get("expect_failure", False)),
         maxseconds=test_elem.get("maxseconds", None),
     )
-    _copy_to_dict_if_present(test_elem, rval, ["interactor", "num_outputs"])
+    _copy_to_dict_if_present(test_elem, rval, ["num_outputs"])
     return rval
 
 
@@ -431,7 +429,7 @@ def __parse_output_elems(test_elem):
     outputs = []
     for output_elem in test_elem.findall("output"):
         name, file, attributes = __parse_output_elem(output_elem)
-        outputs.append((name, file, attributes))
+        outputs.append({"name": name, "value": file, "attributes": attributes})
     return outputs
 
 
@@ -464,7 +462,7 @@ def __parse_output_collection_elem(output_collection_elem):
     if name is None:
         raise Exception("Test output collection does not have a 'name'")
     element_tests = __parse_element_tests(output_collection_elem)
-    return TestCollectionOutputDef(name, attrib, element_tests)
+    return TestCollectionOutputDef(name, attrib, element_tests).to_dict()
 
 
 def __parse_element_tests(parent_element):
@@ -565,7 +563,12 @@ def __parse_extra_files_elem(extra):
     assert extra_type == 'directory' or extra_name is not None, \
         'extra_files type (%s) requires a name attribute' % extra_type
     extra_value, extra_attributes = __parse_test_attributes(extra, attrib)
-    return extra_type, extra_value, extra_name, extra_attributes
+    return {
+        "value": extra_value,
+        "name": extra_name,
+        "type": extra_type,
+        "attributes": extra_attributes
+    }
 
 
 def __expand_input_elems(root_elem, prefix=""):
@@ -628,8 +631,8 @@ def _copy_to_dict_if_present(elem, rval, attributes):
 def __parse_inputs_elems(test_elem, i):
     raw_inputs = []
     for param_elem in test_elem.findall("param"):
-        name, value, attrib = __parse_param_elem(param_elem, i)
-        raw_inputs.append((name, value, attrib))
+        raw_inputs.append(__parse_param_elem(param_elem, i))
+
     return raw_inputs
 
 
@@ -641,40 +644,43 @@ def __parse_param_elem(param_elem, i=0):
         value = attrib['value']
     else:
         value = None
-    attrib['children'] = param_elem
-    if attrib['children'] is not None:
+    children_elem = param_elem
+    if children_elem is not None:
         # At this time, we can assume having children only
         # occurs on DataToolParameter test items but this could
         # change and would cause the below parsing to change
         # based upon differences in children items
-        attrib['metadata'] = []
+        attrib['metadata'] = {}
         attrib['composite_data'] = []
         attrib['edit_attributes'] = []
         # Composite datasets need to be renamed uniquely
         composite_data_name = None
-        for child in attrib['children']:
+        for child in children_elem:
             if child.tag == 'composite_data':
-                attrib['composite_data'].append(child)
+                file_name = child.get("value")
+                attrib['composite_data'].append(file_name)
                 if composite_data_name is None:
                     # Generate a unique name; each test uses a
                     # fresh history.
                     composite_data_name = '_COMPOSITE_RENAMED_t%d_%s' \
                         % (i, uuid.uuid1().hex)
             elif child.tag == 'metadata':
-                attrib['metadata'].append(child)
-            elif child.tag == 'metadata':
-                attrib['metadata'].append(child)
+                attrib['metadata'][child.get("name")] = child.get("value")
             elif child.tag == 'edit_attributes':
                 attrib['edit_attributes'].append(child)
             elif child.tag == 'collection':
-                attrib['collection'] = TestCollectionDef(child, __parse_param_elem)
+                attrib['collection'] = TestCollectionDef.from_xml(child, __parse_param_elem)
         if composite_data_name:
             # Composite datasets need implicit renaming;
             # inserted at front of list so explicit declarations
             # take precedence
             attrib['edit_attributes'].insert(0, {'type': 'name', 'value': composite_data_name})
     name = attrib.pop('name')
-    return (name, value, attrib)
+    return {
+        "name": name,
+        "value": value,
+        "attributes": attrib
+    }
 
 
 class StdioParser(object):

lib/galaxy/tools/parser/yaml.py

@@ -188,7 +188,7 @@ def _parse_test(i, test_dict):
     if _is_dict(inputs):
         new_inputs = []
         for key, value in inputs.items():
-            new_inputs.append((key, value, {}))
+            new_inputs.append({"name": key, "value": value, "attributes": {}})
         test_dict["inputs"] = new_inputs
 
     outputs = test_dict["outputs"]
@@ -202,7 +202,11 @@ def _parse_test(i, test_dict):
             else:
                 file = value
                 attributes = {}
-            new_outputs.append((key, file, attributes))
+            new_outputs.append({
+                "name": key,
+                "value": file,
+                "attributes": attributes
+            })
     else:
         for output in outputs:
             name = output["name"]
@@ -211,7 +215,7 @@ def _parse_test(i, test_dict):
             new_outputs.append((name, value, attributes))
 
     for output in new_outputs:
-        attributes = output[2]
+        attributes = output["attributes"]
         defaults = {
             'compare': 'diff',
             'lines_diff': 0,

lib/galaxy/tools/test.py

@@ -6,6 +6,7 @@ from six import string_types
 
 import galaxy.tools.parameters.basic
 import galaxy.tools.parameters.grouping
+from galaxy.tools.verify.interactor import ToolTestDescription
 from galaxy.util import string_as_bool
 
 try:
@@ -16,11 +17,6 @@ except ImportError:
 
 log = logging.getLogger(__name__)
 
-DEFAULT_FTYPE = 'auto'
-DEFAULT_DBKEY = 'hg17'
-DEFAULT_INTERACTOR = "api"  # Default mechanism test code uses for interacting with Galaxy instance.
-DEFAULT_MAX_SECS = None
-
 
 @nottest
 def parse_tests(tool, tests_source):
@@ -28,207 +24,128 @@ def parse_tests(tool, tests_source):
     Build ToolTestBuilder objects for each "<test>" elements and
     return default interactor (if any).
     """
-    default_interactor = os.environ.get('GALAXY_TEST_DEFAULT_INTERACTOR', DEFAULT_INTERACTOR)
-    tests_dict = tests_source.parse_tests_to_dict()
-    tests_default_interactor = tests_dict.get('interactor', default_interactor)
+    raw_tests_dict = tests_source.parse_tests_to_dict()
     tests = []
-    for i, test_dict in enumerate(tests_dict.get('tests', [])):
-        test = ToolTestBuilder(tool, test_dict, i, default_interactor=tests_default_interactor)
+    for i, raw_test_dict in enumerate(raw_tests_dict.get('tests', [])):
+        test = description_from_tool_object(tool, i, raw_test_dict)
         tests.append(test)
     return tests
 
 
-class ToolTestBuilder(object):
+def description_from_tool_object(tool, test_index, raw_test_dict):
+    required_files = []
+
+    num_outputs = raw_test_dict.get('expect_num_outputs', None)
+    if num_outputs:
+        num_outputs = int(num_outputs)
+
+    try:
+        processed_inputs = _process_raw_inputs(tool, tool.inputs, raw_test_dict["inputs"], required_files)
+        processed_test_dict = {
+            "inputs": processed_inputs,
+            "outputs": raw_test_dict["outputs"],
+            "output_collections": raw_test_dict["output_collections"],
+            "num_outputs": num_outputs,
+            "command_line": raw_test_dict.get("command", None),
+            "stdout": raw_test_dict.get("stdout", None),
+            "stderr": raw_test_dict.get("stderr", None),
+            "expect_exit_code": raw_test_dict.get("expect_exit_code", None),
+            "expect_failure": raw_test_dict.get("expect_failure", False),
+            "required_files": required_files,
+            "tool_id": tool.id,
+            "test_index": test_index,
+            "error": False,
+        }
+    except Exception as e:
+        log.exception("Failed to load tool test number [%d] for %s" % (test_index, tool.id))
+        processed_test_dict = {
+            "tool_id": tool.id,
+            "test_index": test_index,
+            "inputs": {},
+            "error": True,
+            "exception": str(e),
+        }
+
+    return ToolTestDescription(processed_test_dict)
+
+
+def _process_raw_inputs(tool, tool_inputs, raw_inputs, required_files, parent_context=None):
     """
-    Encapsulates information about a tool test, and allows creation of a
-    dynamic TestCase class (the unittest framework is very class oriented,
-    doing dynamic tests in this way allows better integration)
+    Recursively expand flat list of inputs into "tree" form of flat list
+    (| using to nest to new levels) structure and expand dataset
+    information as proceeding to populate self.required_files.
     """
-
-    def __init__(self, tool, test_dict, i, default_interactor):
-        name = test_dict.get('name', 'Test-%d' % (i + 1))
-        maxseconds = test_dict.get('maxseconds', DEFAULT_MAX_SECS)
-        if maxseconds is not None:
-            maxseconds = int(maxseconds)
-
-        self.tool = tool
-        self.name = name
-        self.maxseconds = maxseconds
-        self.required_files = []
-        self.inputs = {}
-        self.outputs = []
-        # By default do not making assertions on number of outputs - but to
-        # test filtering allow explicitly state number of outputs.
-        self.num_outputs = None
-        self.error = False
-        self.exception = None
-
-        self.__handle_test_dict(test_dict, i, default_interactor)
-
-    def test_data(self):
-        """
-        Iterator over metadata representing the required files for upload.
-        """
-        return test_data_iter(self.required_files)
-
-    def __matching_case_for_value(self, cond, declared_value):
-        test_param = cond.test_param
-        if isinstance(test_param, galaxy.tools.parameters.basic.BooleanToolParameter):
-            if declared_value is None:
-                # No explicit value for param in test case, determine from default
-                query_value = test_param.checked
-            else:
-                query_value = _process_bool_param_value(test_param, declared_value)
-
-            def matches_declared_value(case_value):
-                return _process_bool_param_value(test_param, case_value) == query_value
-        elif isinstance(test_param, galaxy.tools.parameters.basic.SelectToolParameter):
-            if declared_value is not None:
-                # Test case supplied explicit value to check against.
-
-                def matches_declared_value(case_value):
-                    return case_value == declared_value
-            elif test_param.static_options:
-                # No explicit value in test case, not much to do if options are dynamic but
-                # if static options are available can find the one specified as default or
-                # fallback on top most option (like GUI).
-                for (name, value, selected) in test_param.static_options:
-                    if selected:
-                        default_option = name
-                else:
-                    first_option = test_param.static_options[0]
-                    first_option_value = first_option[1]
-                    default_option = first_option_value
-
-                def matches_declared_value(case_value):
-                    return case_value == default_option
-            else:
-                # No explicit value for this param and cannot determine a
-                # default - give up. Previously this would just result in a key
-                # error exception.
-                msg = "Failed to find test parameter value specification required for conditional %s" % cond.name
-                raise Exception(msg)
-
-        # Check the tool's defined cases against predicate to determine
-        # selected or default.
-        for i, case in enumerate(cond.cases):
-            if matches_declared_value(case.value):
-                return case
-        else:
-            msg_template = "%s - Failed to find case matching value (%s) for test parameter specification for conditional %s. Remainder of test behavior is unspecified."
-            msg = msg_template % (self.tool.id, declared_value, cond.name)
-            log.info(msg)
-
-    def __split_if_str(self, value):
-        split = isinstance(value, string_types)
-        if split:
-            value = value.split(",")
-        return value
-
-    def __handle_test_dict(self, test_dict, i, default_interactor):
-        try:
-            # Mechanism test code uses for interacting with Galaxy instance,
-            # until 'api' is the default switch this to API to use its new
-            # features. Once 'api' is the default set to 'twill' to use legacy
-            # features or workarounds.
-            self.interactor = test_dict.get('interactor', default_interactor)
-
-            self.inputs = self.__process_raw_inputs(self.tool.inputs, test_dict["inputs"])
-            self.outputs = test_dict["outputs"]
-            self.output_collections = test_dict["output_collections"]
-            num_outputs = test_dict.get('expect_num_outputs', None)
-            if num_outputs:
-                num_outputs = int(num_outputs)
-            self.num_outputs = num_outputs
-            self.command_line = test_dict.get("command", None)
-            self.stdout = test_dict.get("stdout", None)
-            self.stderr = test_dict.get("stderr", None)
-            self.expect_exit_code = test_dict.get("expect_exit_code", None)
-            self.expect_failure = test_dict.get("expect_failure", False)
-            self.md5 = test_dict.get("md5", None)
-        except Exception as e:
-            self.inputs = {}
-            self.error = True
-            self.exception = e
-
-    def __process_raw_inputs(self, tool_inputs, raw_inputs, parent_context=None):
-        """
-        Recursively expand flat list of inputs into "tree" form of flat list
-        (| using to nest to new levels) structure and expand dataset
-        information as proceeding to populate self.required_files.
-        """
-        parent_context = parent_context or RootParamContext()
-        expanded_inputs = {}
-        for key, value in tool_inputs.items():
-            if isinstance(value, galaxy.tools.parameters.grouping.Conditional):
-                cond_context = ParamContext(name=value.name, parent_context=parent_context)
-                case_context = ParamContext(name=value.test_param.name, parent_context=cond_context)
-                raw_input = case_context.extract_value(raw_inputs)
-                case_value = raw_input[1] if raw_input else None
-                case = self.__matching_case_for_value(value, case_value)
-                if case:
-                    for input_name, input_value in case.inputs.items():
-                        case_inputs = self.__process_raw_inputs({input_name: input_value}, raw_inputs, parent_context=cond_context)
-                        expanded_inputs.update(case_inputs)
-                    if not value.type == "text":
-                        expanded_case_value = self.__split_if_str(case.value)
-                    if case_value is not None:
-                        # A bit tricky here - we are growing inputs with value
-                        # that may be implicit (i.e. not defined by user just
-                        # a default defined in tool). So we do not want to grow
-                        # expanded_inputs and risk repeat block viewing this
-                        # as a new instance with value defined and hence enter
-                        # an infinite loop - hence the "case_value is not None"
-                        # check.
-                        processed_value = _process_simple_value(value.test_param, expanded_case_value)
-                        expanded_inputs[case_context.for_state()] = processed_value
-            elif isinstance(value, galaxy.tools.parameters.grouping.Section):
-                context = ParamContext(name=value.name, parent_context=parent_context)
+    parent_context = parent_context or RootParamContext()
+    expanded_inputs = {}
+    for key, value in tool_inputs.items():
+        if isinstance(value, galaxy.tools.parameters.grouping.Conditional):
+            cond_context = ParamContext(name=value.name, parent_context=parent_context)
+            case_context = ParamContext(name=value.test_param.name, parent_context=cond_context)
+            raw_input_dict = case_context.extract_value(raw_inputs)
+            case_value = raw_input_dict["value"] if raw_input_dict else None
+            case = _matching_case_for_value(tool, value, case_value)
+            if case:
+                for input_name, input_value in case.inputs.items():
+                    case_inputs = _process_raw_inputs(tool, {input_name: input_value}, raw_inputs, required_files, parent_context=cond_context)
+                    expanded_inputs.update(case_inputs)
+                if not value.type == "text":
+                    expanded_case_value = _split_if_str(case.value)
+                if case_value is not None:
+                    # A bit tricky here - we are growing inputs with value
+                    # that may be implicit (i.e. not defined by user just
+                    # a default defined in tool). So we do not want to grow
+                    # expanded_inputs and risk repeat block viewing this
+                    # as a new instance with value defined and hence enter
+                    # an infinite loop - hence the "case_value is not None"
+                    # check.
+                    processed_value = _process_simple_value(value.test_param, expanded_case_value)
+                    expanded_inputs[case_context.for_state()] = processed_value
+        elif isinstance(value, galaxy.tools.parameters.grouping.Section):
+            context = ParamContext(name=value.name, parent_context=parent_context)
+            for r_name, r_value in value.inputs.items():
+                expanded_input = _process_raw_inputs(tool, {context.for_state(): r_value}, raw_inputs, required_files, parent_context=context)
+                if expanded_input:
+                    expanded_inputs.update(expanded_input)
+        elif isinstance(value, galaxy.tools.parameters.grouping.Repeat):
+            repeat_index = 0
+            while True:
+                context = ParamContext(name=value.name, index=repeat_index, parent_context=parent_context)
+                updated = False
                 for r_name, r_value in value.inputs.items():
-                    expanded_input = self.__process_raw_inputs({context.for_state(): r_value}, raw_inputs, parent_context=context)
+                    expanded_input = _process_raw_inputs(tool, {context.for_state(): r_value}, raw_inputs, required_files, parent_context=context)
                     if expanded_input:
                         expanded_inputs.update(expanded_input)
-            elif isinstance(value, galaxy.tools.parameters.grouping.Repeat):
-                repeat_index = 0
-                while True:
-                    context = ParamContext(name=value.name, index=repeat_index, parent_context=parent_context)
-                    updated = False
-                    for r_name, r_value in value.inputs.items():
-                        expanded_input = self.__process_raw_inputs({context.for_state(): r_value}, raw_inputs, parent_context=context)
-                        if expanded_input:
-                            expanded_inputs.update(expanded_input)
-                            updated = True
-                    if not updated:
-                        break
-                    repeat_index += 1
-            else:
-                context = ParamContext(name=value.name, parent_context=parent_context)
-                raw_input = context.extract_value(raw_inputs)
-                if raw_input:
-                    (name, param_value, param_extra) = raw_input
-                    if not value.type == "text":
-                        param_value = self.__split_if_str(param_value)
-                    if isinstance(value, galaxy.tools.parameters.basic.DataToolParameter):
-                        if not isinstance(param_value, list):
-                            param_value = [param_value]
-                        map(lambda v: self.__add_uploaded_dataset(context.for_state(), v, param_extra, value), param_value)
-                        processed_value = param_value
-                    elif isinstance(value, galaxy.tools.parameters.basic.DataCollectionToolParameter):
-                        assert 'collection' in param_extra
-                        collection_def = param_extra['collection']
-                        for (name, value, extra) in collection_def.collect_inputs():
-                            require_file(name, value, extra, self.required_files)
-                        processed_value = collection_def
-                    else:
-                        processed_value = _process_simple_value(value, param_value)
-                    expanded_inputs[context.for_state()] = processed_value
-        return expanded_inputs
-
-    def __add_uploaded_dataset(self, name, value, extra, input_parameter):
-        if value is None:
-            assert input_parameter.optional, '%s is not optional. You must provide a valid filename.' % name
-            return value
-        return require_file(name, value, extra, self.required_files)
+                        updated = True
+                if not updated:
+                    break
+                repeat_index += 1
+        else:
+            context = ParamContext(name=value.name, parent_context=parent_context)
+            raw_input_dict = context.extract_value(raw_inputs)
+            if raw_input_dict:
+                name = raw_input_dict["name"]
+                param_value = raw_input_dict["value"]
+                param_extra = raw_input_dict["attributes"]
+                if not value.type == "text":
+                    param_value = _split_if_str(param_value)
+                if isinstance(value, galaxy.tools.parameters.basic.DataToolParameter):
+                    if not isinstance(param_value, list):
+                        param_value = [param_value]
+                    map(lambda v: _add_uploaded_dataset(context.for_state(), v, param_extra, value, required_files), param_value)
+                    processed_value = param_value
+                elif isinstance(value, galaxy.tools.parameters.basic.DataCollectionToolParameter):
+                    assert 'collection' in param_extra
+                    collection_def = param_extra['collection']
+                    for input_dict in collection_def.collect_inputs():
+                        name = input_dict["name"]
+                        value = input_dict["value"]
+                        attributes = input_dict["attributes"]
+                        require_file(name, value, attributes, required_files)
+                    processed_value = collection_def
+                else:
+                    processed_value = _process_simple_value(value, param_value)
+                expanded_inputs[context.for_state()] = processed_value
+    return expanded_inputs
 
 
 def _process_simple_value(param, param_value):
@@ -266,6 +183,69 @@ def _process_simple_value(param, param_value):
     return processed_value
 
 
+def _matching_case_for_value(tool, cond, declared_value):
+    test_param = cond.test_param
+    if isinstance(test_param, galaxy.tools.parameters.basic.BooleanToolParameter):
+        if declared_value is None:
+            # No explicit value for param in test case, determine from default
+            query_value = test_param.checked
+        else:
+            query_value = _process_bool_param_value(test_param, declared_value)
+
+        def matches_declared_value(case_value):
+            return _process_bool_param_value(test_param, case_value) == query_value
+    elif isinstance(test_param, galaxy.tools.parameters.basic.SelectToolParameter):
+        if declared_value is not None:
+            # Test case supplied explicit value to check against.
+
+            def matches_declared_value(case_value):
+                return case_value == declared_value
+        elif test_param.static_options:
+            # No explicit value in test case, not much to do if options are dynamic but
+            # if static options are available can find the one specified as default or
+            # fallback on top most option (like GUI).
+            for (name, value, selected) in test_param.static_options:
+                if selected:
+                    default_option = name
+            else:
+                first_option = test_param.static_options[0]
+                first_option_value = first_option[1]
+                default_option = first_option_value
+
+            def matches_declared_value(case_value):
+                return case_value == default_option
+        else:
+            # No explicit value for this param and cannot determine a
+            # default - give up. Previously this would just result in a key
+            # error exception.
+            msg = "Failed to find test parameter value specification required for conditional %s" % cond.name
+            raise Exception(msg)
+
+    # Check the tool's defined cases against predicate to determine
+    # selected or default.
+    for i, case in enumerate(cond.cases):
+        if matches_declared_value(case.value):
+            return case
+    else:
+        msg_template = "%s - Failed to find case matching value (%s) for test parameter specification for conditional %s. Remainder of test behavior is unspecified."
+        msg = msg_template % (tool.id, declared_value, cond.name)
+        log.info(msg)
+
+
+def _add_uploaded_dataset(name, value, extra, input_parameter, required_files):
+    if value is None:
+        assert input_parameter.optional, '%s is not optional. You must provide a valid filename.' % name
+        return value
+    return require_file(name, value, extra, required_files)
+
+
+def _split_if_str(value):
+    split = isinstance(value, string_types)
+    if split:
+        value = value.split(",")
+    return value
+
+
 def _process_bool_param_value(param, param_value):
     assert isinstance(param, galaxy.tools.parameters.basic.BooleanToolParameter)
     was_list = False
@@ -281,30 +261,6 @@ def _process_bool_param_value(param, param_value):
     return [processed_value] if was_list else processed_value
 
 
-@nottest
-def test_data_iter(required_files):
-    for fname, extra in required_files:
-        data_dict = dict(
-            fname=fname,
-            metadata=extra.get('metadata', []),
-            composite_data=extra.get('composite_data', []),
-            ftype=extra.get('ftype', DEFAULT_FTYPE),
-            dbkey=extra.get('dbkey', DEFAULT_DBKEY),
-        )
-        edit_attributes = extra.get('edit_attributes', [])
-
-        # currently only renaming is supported
-        for edit_att in edit_attributes:
-            if edit_att.get('type', None) == 'name':
-                new_name = edit_att.get('value', None)
-                assert new_name, 'You must supply the new dataset name as the value tag of the edit_attributes tag'
-                data_dict['name'] = new_name
-            else:
-                raise Exception('edit_attributes type (%s) is unimplemented' % edit_att.get('type', None))
-
-        yield data_dict
-
-
 def require_file(name, value, extra, required_files):
     if (value, extra) not in required_files:
         required_files.append((value, extra))  # these files will be uploaded
@@ -359,13 +315,13 @@ class ParamContext(object):
 
     def __raw_param_found(self, param_name, raw_inputs):
         index = None
-        for i, raw_input in enumerate(raw_inputs):
-            if raw_input[0] == param_name:
+        for i, raw_input_dict in enumerate(raw_inputs):
+            if raw_input_dict["name"] == param_name:
                 index = i
         if index is not None:
-            raw_input = raw_inputs[index]
+            raw_input_dict = raw_inputs[index]
             del raw_inputs[index]
-            return raw_input
+            return raw_input_dict
         else:
             return None
 

lib/galaxy/tools/verify/asserts/__init__.py

@@ -2,6 +2,8 @@ import inspect
 import logging
 import sys
 
+from galaxy.util import unicodify
+
 log = logging.getLogger(__name__)
 
 assertion_module_names = ['text', 'tabular', 'xml']
@@ -66,7 +68,11 @@ def verify_assertion(data, assertion_description):
     # - <has_column_titles><with_name name="sequence"><with_name
     # name="probability"></has_column_titles>.)
     if "output" in assert_function_args:
-        args["output"] = data
+        # This was read in as bytes for checksum and such, but all current
+        # assertions expect text data. If binary assertions are added at
+        # some point, just checkout for "output_bytes" for instance and pass
+        # data in unchanged.
+        args["output"] = unicodify(data)
 
     if "verify_assertions_function" in assert_function_args:
         args["verify_assertions_function"] = verify_assertions

lib/galaxy/tools/verify/interactor.py

@@ -0,0 +1,971 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+import os
+import re
+import sys
+import time
+from json import dumps
+from logging import getLogger
+
+try:
+    from nose.tools import nottest
+except ImportError:
+    def nottest(x):
+        return x
+try:
+    import requests
+except ImportError:
+    requests = None
+from six import StringIO, text_type
+
+from galaxy import util
+from galaxy.tools.parser.interface import TestCollectionDef, TestCollectionOutputDef
+from galaxy.util.bunch import Bunch
+from galaxy.util.odict import odict
+from .asserts import verify_assertions
+from ..verify import verify
+
+log = getLogger(__name__)
+
+# Off by default because it can pound the database pretty heavily
+# and result in sqlite errors on larger tests or larger numbers of
+# tests.
+VERBOSE_ERRORS = util.asbool(os.environ.get("GALAXY_TEST_VERBOSE_ERRORS", False))
+UPLOAD_ASYNC = util.asbool(os.environ.get("GALAXY_TEST_UPLOAD_ASYNC", True))
+ERROR_MESSAGE_DATASET_SEP = "--------------------------------------"
+DEFAULT_TOOL_TEST_WAIT = os.environ.get("GALAXY_TEST_DEFAULT_WAIT", 86400)
+
+DEFAULT_FTYPE = 'auto'
+# This following default dbkey was traditionally hg17 before Galaxy 18.05,
+# restore this behavior by setting GALAXY_TEST_DEFAULT_DBKEY to hg17.
+DEFAULT_DBKEY = os.environ.get("GALAXY_TEST_DEFAULT_DBKEY", "?")
+DEFAULT_MAX_SECS = DEFAULT_TOOL_TEST_WAIT
+
+
+def stage_data_in_history(galaxy_interactor, tool_id, all_test_data, history):
+    # Upload any needed files
+    upload_waits = []
+
+    assert tool_id
+
+    if UPLOAD_ASYNC:
+        for test_data in all_test_data:
+            upload_waits.append(galaxy_interactor.stage_data_async(test_data, history, tool_id))
+        for upload_wait in upload_waits:
+            upload_wait()
+    else:
+        for test_data in all_test_data:
+            upload_wait = galaxy_interactor.stage_data_async(test_data, history, tool_id)
+            upload_wait()
+
+
+class GalaxyInteractorApi(object):
+
+    def __init__(self, **kwds):
+        self.api_url = "%s/api" % kwds["galaxy_url"].rstrip("/")
+        self.master_api_key = kwds["master_api_key"]
+        self.api_key = self.__get_user_key(kwds.get("api_key"), kwds.get("master_api_key"), test_user=kwds.get("test_user"))
+        self.keep_outputs_dir = kwds["keep_outputs_dir"]
+
+        self.uploads = {}
+
+    def __get_user_key(self, user_key, admin_key, test_user=None):
+        if not test_user:
+            test_user = "test@bx.psu.edu"
+        if user_key:
+            return user_key
+        test_user = self.ensure_user_with_email(test_user)
+        return self._post("users/%s/api_key" % test_user['id'], key=admin_key).json()
+
+    # def get_tools(self):
+    #    response = self._get("tools?in_panel=false")
+    #    assert response.status_code == 200, "Non 200 response from tool index API. [%s]" % response.content
+    #    return response.json()
+
+    def get_tests_summary(self):
+        response = self._get("tools/tests_summary")
+        assert response.status_code == 200, "Non 200 response from tool tests available API. [%s]" % response.content
+        return response.json()
+
+    def get_tool_tests(self, tool_id, tool_version=None):
+        url = "tools/%s/test_data" % tool_id
+        if tool_version is not None:
+            url += "?tool_version=%s" % tool_version
+        response = self._get(url)
+        assert response.status_code == 200, "Non 200 response from tool test API. [%s]" % response.content
+        return response.json()
+
+    def verify_output(self, history_id, jobs, output_data, output_testdef, tool_id, maxseconds):
+        outfile = output_testdef.outfile
+        attributes = output_testdef.attributes
+        name = output_testdef.name
+        self.wait_for_jobs(history_id, jobs, maxseconds)
+        hid = self.__output_id(output_data)
+        # TODO: Twill version verifys dataset is 'ok' in here.
+        self.verify_output_dataset(history_id=history_id, hda_id=hid, outfile=outfile, attributes=attributes, tool_id=tool_id)
+
+        primary_datasets = attributes.get('primary_datasets', {})
+        if primary_datasets:
+            job_id = self._dataset_provenance(history_id, hid)["job_id"]
+            outputs = self._get("jobs/%s/outputs" % (job_id)).json()
+
+        for designation, (primary_outfile, primary_attributes) in primary_datasets.items():
+            primary_output = None
+            for output in outputs:
+                if output["name"] == '__new_primary_file_%s|%s__' % (name, designation):
+                    primary_output = output
+                    break
+
+            if not primary_output:
+                msg_template = "Failed to find primary dataset with designation [%s] for output with name [%s]"
+                msg_args = (designation, name)
+                raise Exception(msg_template % msg_args)
+
+            primary_hda_id = primary_output["dataset"]["id"]
+            self.verify_output_dataset(history_id, primary_hda_id, primary_outfile, primary_attributes, tool_id=tool_id)
+
+    def wait_for_jobs(self, history_id, jobs, maxseconds):
+        for job in jobs:
+            self.wait_for_job(job['id'], history_id, maxseconds)
+
+    def verify_output_dataset(self, history_id, hda_id, outfile, attributes, tool_id):
+        fetcher = self.__dataset_fetcher(history_id)
+        test_data_path_builder = self.__test_data_path_builder(tool_id)
+        verify_hid(
+            outfile,
+            hda_id=hda_id,
+            attributes=attributes,
+            dataset_fetcher=fetcher,
+            test_data_path_builder=test_data_path_builder,
+            keep_outputs_dir=self.keep_outputs_dir
+        )
+        self._verify_metadata(history_id, hda_id, attributes)
+
+    def _verify_metadata(self, history_id, hid, attributes):
+        """Check dataset metadata.
+
+        ftype on output maps to `file_ext` on the hda's API description, `name`, `info`,
+        `dbkey` and `tags` all map to the API description directly. Other metadata attributes
+        are assumed to be datatype-specific and mapped with a prefix of `metadata_`.
+        """
+        metadata = attributes.get('metadata', {}).copy()
+        for key, value in metadata.copy().items():
+            if key not in ['name', 'info', 'tags']:
+                new_key = "metadata_%s" % key
+                metadata[new_key] = metadata[key]
+                del metadata[key]
+            elif key == "info":
+                metadata["misc_info"] = metadata["info"]
+                del metadata["info"]
+        expected_file_type = attributes.get('ftype', None)
+        if expected_file_type:
+            metadata["file_ext"] = expected_file_type
+
+        if metadata:
+            time.sleep(5)
+            dataset = self._get("histories/%s/contents/%s" % (history_id, hid)).json()
+            for key, value in metadata.items():
+                try:
+                    dataset_value = dataset.get(key, None)
+
+                    def compare(val, expected):
+                        if text_type(val) != text_type(expected):
+                            msg = "Dataset metadata verification for [%s] failed, expected [%s] but found [%s]. Dataset API value was [%s]."
+                            msg_params = (key, value, dataset_value, dataset)
+                            msg = msg % msg_params
+                            raise Exception(msg)
+
+                    if isinstance(dataset_value, list):
+                        value = text_type(value).split(",")
+                        if len(value) != len(dataset_value):
+                            msg = "Dataset metadata verification for [%s] failed, expected [%s] but found [%s], lists differ in length. Dataset API value was [%s]."
+                            msg_params = (key, value, dataset_value, dataset)
+                            msg = msg % msg_params
+                            raise Exception(msg)
+                        for val, expected in zip(dataset_value, value):
+                            compare(val, expected)
+                    else:
+                        compare(dataset_value, value)
+                except KeyError:
+                    msg = "Failed to verify dataset metadata, metadata key [%s] was not found." % key
+                    raise Exception(msg)
+
+    def wait_for_job(self, job_id, history_id, maxseconds):
+        self.wait_for(lambda: not self.__job_ready(job_id, history_id), maxseconds=maxseconds)
+
+    def wait_for(self, func, **kwd):
+        sleep_amount = 0.2
+        slept = 0
+        walltime_exceeded = kwd.get("maxseconds", DEFAULT_TOOL_TEST_WAIT)
+
+        while slept <= walltime_exceeded:
+            result = func()
+            if result:
+                time.sleep(sleep_amount)
+                slept += sleep_amount
+                sleep_amount *= 2
+            else:
+                return
+
+        message = 'Tool test run exceeded walltime [total %s, max %s], terminating.' % (slept, walltime_exceeded)
+        log.info(message)
+        raise AssertionError(message)
+
+    def get_job_stdio(self, job_id):
+        job_stdio = self.__get_job_stdio(job_id).json()
+        return job_stdio
+
+    def __get_job(self, job_id):
+        return self._get('jobs/%s' % job_id)
+
+    def __get_job_stdio(self, job_id):
+        return self._get('jobs/%s?full=true' % job_id)
+
+    def new_history(self):
+        history_json = self._post("histories", {"name": "test_history"}).json()
+        return history_json['id']
+
+    @nottest
+    def test_data_path(self, tool_id, filename):
+        return self._get("tools/%s/test_data_path?filename=%s" % (tool_id, filename)).json()
+
+    def __output_id(self, output_data):
+        # Allow data structure coming out of tools API - {id: <id>, output_name: <name>, etc...}
+        # or simple id as comes out of workflow API.
+        try:
+            output_id = output_data.get('id')
+        except AttributeError:
+            output_id = output_data
+        return output_id
+
+    def stage_data_async(self, test_data, history_id, tool_id, async=True):
+        fname = test_data['fname']
+        tool_input = {
+            "file_type": test_data['ftype'],
+            "dbkey": test_data['dbkey'],
+        }
+        metadata = test_data.get("metadata", {})
+        if not hasattr(metadata, "items"):
+            raise Exception("Invalid metadata description found for input [%s] - [%s]" % (fname, metadata))
+        for name, value in test_data.get('metadata', {}).items():
+            tool_input["files_metadata|%s" % name] = value
+
+        composite_data = test_data['composite_data']
+        if composite_data:
+            files = {}
+            for i, file_name in enumerate(composite_data):
+                file_name = self.test_data_path(tool_id, file_name)
+                files["files_%s|file_data" % i] = open(file_name, 'rb')
+                tool_input.update({
+                    "files_%d|type" % i: "upload_dataset",
+                })
+            name = test_data['name']
+        else:
+            file_name = self.test_data_path(tool_id, fname)
+            name = test_data.get('name', None)
+            if not name:
+                name = os.path.basename(file_name)
+
+            tool_input.update({
+                "files_0|NAME": name,
+                "files_0|type": "upload_dataset",
+            })
+            # TODO: Option to upload by path since we are getting the paths from Galaxy now it makes more
+            # sense to move this there.
+            files = {
+                "files_0|file_data": open(file_name, 'rb')
+            }
+        submit_response_object = self.__submit_tool(history_id, "upload1", tool_input, extra_data={"type": "upload_dataset"}, files=files)
+        if submit_response_object.status_code != 200:
+            raise Exception("Request to upload dataset failed [%s]" % submit_response_object.content)
+        submit_response = submit_response_object.json()
+        assert "outputs" in submit_response, "Invalid response from server [%s], expecteding outputs in response." % submit_response
+        outputs = submit_response["outputs"]
+        assert len(outputs) > 0, "Invalid response from server [%s], expecting an output dataset." % submit_response
+        dataset = outputs[0]
+        hid = dataset['id']
+        self.uploads[os.path.basename(fname)] = self.uploads[fname] = self.uploads[name] = {"src": "hda", "id": hid}
+        assert "jobs" in submit_response, "Invalid response from server [%s], expecting jobs in response." % submit_response
+        jobs = submit_response["jobs"]
+        assert len(jobs) > 0, "Invalid response from server [%s], expecting a job." % submit_response
+        return lambda: self.wait_for_job(jobs[0]["id"], history_id, DEFAULT_TOOL_TEST_WAIT)
+
+    def run_tool(self, testdef, history_id, resource_parameters={}):
+        # We need to handle the case where we've uploaded a valid compressed file since the upload
+        # tool will have uncompressed it on the fly.
+
+        inputs_tree = testdef.inputs.copy()
+        for key, value in inputs_tree.items():
+            values = [value] if not isinstance(value, list) else value
+            new_values = []
+            for value in values:
+                if isinstance(value, TestCollectionDef):
+                    hdca_id = self._create_collection(history_id, value)
+                    new_values = [dict(src="hdca", id=hdca_id)]
+                elif value in self.uploads:
+                    new_values.append(self.uploads[value])
+                else:
+                    new_values.append(value)
+            inputs_tree[key] = new_values
+
+        if resource_parameters:
+            inputs_tree["__job_resource|__job_resource__select"] = "yes"
+            for key, value in resource_parameters.items():
+                inputs_tree["__job_resource|%s" % key] = value
+
+        # HACK: Flatten single-value lists. Required when using expand_grouping
+        for key, value in inputs_tree.items():
+            if isinstance(value, list) and len(value) == 1:
+                inputs_tree[key] = value[0]
+
+        submit_response = self.__submit_tool(history_id, tool_id=testdef.tool_id, tool_input=inputs_tree)
+        submit_response_object = submit_response.json()
+        try:
+            return Bunch(
+                inputs=inputs_tree,
+                outputs=self.__dictify_outputs(submit_response_object),
+                output_collections=self.__dictify_output_collections(submit_response_object),
+                jobs=submit_response_object['jobs'],
+            )
+        except KeyError:
+            message = "Error creating a job for these tool inputs - %s" % submit_response_object['err_msg']
+            raise RunToolException(message, inputs_tree)
+
+    def _create_collection(self, history_id, collection_def):
+        create_payload = dict(
+            name=collection_def.name,
+            element_identifiers=dumps(self._element_identifiers(collection_def)),
+            collection_type=collection_def.collection_type,
+            history_id=history_id,
+        )
+        return self._post("dataset_collections", data=create_payload).json()["id"]
+
+    def _element_identifiers(self, collection_def):
+        element_identifiers = []
+        for element_dict in collection_def.elements:
+            element_identifier = element_dict["element_identifier"]
+            element_def = element_dict["element_definition"]
+            if isinstance(element_def, TestCollectionDef):
+                subelement_identifiers = self._element_identifiers(element_def)
+                element = dict(
+                    name=element_identifier,
+                    src="new_collection",
+                    collection_type=element_def.collection_type,
+                    element_identifiers=subelement_identifiers
+                )
+            else:
+                element = self.uploads[element_def["value"]].copy()
+                element["name"] = element_identifier
+            element_identifiers.append(element)
+        return element_identifiers
+
+    def __dictify_output_collections(self, submit_response):
+        output_collections_dict = odict()
+        for output_collection in submit_response['output_collections']:
+            output_collections_dict[output_collection.get("output_name")] = output_collection
+        return output_collections_dict
+
+    def __dictify_outputs(self, datasets_object):
+        # Convert outputs list to a dictionary that can be accessed by
+        # output_name so can be more flexiable about ordering of outputs
+        # but also allows fallback to legacy access as list mode.
+        outputs_dict = odict()
+        index = 0
+        for output in datasets_object['outputs']:
+            outputs_dict[index] = outputs_dict[output.get("output_name")] = output
+            index += 1
+        # Adding each item twice (once with index for backward compat),
+        # overiding length to reflect the real number of outputs.
+        outputs_dict.__len__ = lambda: index
+        return outputs_dict
+
+    def output_hid(self, output_data):
+        return output_data['id']
+
+    def delete_history(self, history):
+        return None
+
+    def __job_ready(self, job_id, history_id):
+        if job_id is None:
+            raise ValueError("__job_ready passed empty job_id")
+        job_json = self._get("jobs/%s" % job_id).json()
+        state = job_json['state']
+        try:
+            return self._state_ready(state, error_msg="Job in error state.")
+        except Exception:
+            if VERBOSE_ERRORS:
+                self._summarize_history(history_id)
+            raise
+
+    def _summarize_history(self, history_id):
+        if history_id is None:
+            raise ValueError("_summarize_history passed empty history_id")
+        print("Problem in history with id %s - summary of datasets below." % history_id)
+        try:
+            history_contents = self.__contents(history_id)
+        except Exception:
+            print("*TEST FRAMEWORK FAILED TO FETCH HISTORY DETAILS*")
+
+        for history_content in history_contents:
+
+            dataset = history_content
+
+            print(ERROR_MESSAGE_DATASET_SEP)
+            dataset_id = dataset.get('id', None)
+            print("| %d - %s (HID - NAME) " % (int(dataset['hid']), dataset['name']))
+            if history_content['history_content_type'] == 'dataset_collection':
+                history_contents_json = self._get("histories/%s/contents/dataset_collections/%s" % (history_id, history_content["id"])).json()
+                print("| Dataset Collection: %s" % history_contents_json)
+                continue
+
+            try:
+                dataset_info = self._dataset_info(history_id, dataset_id)
+                print("| Dataset State:")
+                print(self.format_for_summary(dataset_info.get("state"), "Dataset state is unknown."))
+                print("| Dataset Blurb:")
+                print(self.format_for_summary(dataset_info.get("misc_blurb", ""), "Dataset blurb was empty."))
+                print("| Dataset Info:")
+                print(self.format_for_summary(dataset_info.get("misc_info", ""), "Dataset info is empty."))
+                print("| Peek:")
+                print(self.format_for_summary(dataset_info.get("peek", ""), "Peek unavilable."))
+            except Exception:
+                print("| *TEST FRAMEWORK ERROR FETCHING DATASET DETAILS*")
+            try:
+                provenance_info = self._dataset_provenance(history_id, dataset_id)
+                print("| Dataset Job Standard Output:")
+                print(self.format_for_summary(provenance_info.get("stdout", ""), "Standard output was empty."))
+                print("| Dataset Job Standard Error:")
+                print(self.format_for_summary(provenance_info.get("stderr", ""), "Standard error was empty."))
+            except Exception:
+                print("| *TEST FRAMEWORK ERROR FETCHING JOB DETAILS*")
+            print("|")
+        print(ERROR_MESSAGE_DATASET_SEP)
+
+    def format_for_summary(self, blob, empty_message, prefix="|  "):
+        contents = "\n".join(["%s%s" % (prefix, line.strip()) for line in StringIO(blob).readlines() if line.rstrip("\n\r")])
+        return contents or "%s*%s*" % (prefix, empty_message)
+
+    def _dataset_provenance(self, history_id, id):
+        provenance = self._get("histories/%s/contents/%s/provenance" % (history_id, id)).json()
+        return provenance
+
+    def _dataset_info(self, history_id, id):
+        dataset_json = self._get("histories/%s/contents/%s" % (history_id, id)).json()
+        return dataset_json
+
+    def __contents(self, history_id):
+        history_contents_json = self._get("histories/%s/contents" % history_id).json()
+        return history_contents_json
+
+    def _state_ready(self, state_str, error_msg):
+        if state_str == 'ok':
+            return True
+        elif state_str == 'error':
+            raise Exception(error_msg)
+        return False
+
+    def __submit_tool(self, history_id, tool_id, tool_input, extra_data={}, files=None):
+        data = dict(
+            history_id=history_id,
+            tool_id=tool_id,
+            inputs=dumps(tool_input),
+            **extra_data
+        )
+        return self._post("tools", files=files, data=data)
+
+    def ensure_user_with_email(self, email, password=None):
+        admin_key = self.master_api_key
+        all_users = self._get('users', key=admin_key).json()
+        try:
+            test_user = [user for user in all_users if user["email"] == email][0]
+        except IndexError:
+            username = re.sub('[^a-z-]', '--', email.lower())
+            password = password or 'testpass'
+            # If remote user middleware is enabled - this endpoint consumes
+            # ``remote_user_email`` otherwise it requires ``email``, ``password``
+            # and ``username``.
+            data = dict(
+                remote_user_email=email,
+                email=email,
+                password=password,
+                username=username,
+            )
+            test_user = self._post('users', data, key=admin_key).json()
+        return test_user
+
+    def __test_data_path_builder(self, tool_id):
+        return lambda filename: self.test_data_path(tool_id, filename)
+
+    def __dataset_fetcher(self, history_id):
+        def fetcher(hda_id, base_name=None):
+            url = "histories/%s/contents/%s/display?raw=true" % (history_id, hda_id)
+            if base_name:
+                url += "&filename=%s" % base_name
+            return self._get(url).content
+
+        return fetcher
+
+    def _post(self, path, data={}, files=None, key=None, admin=False, anon=False):
+        if not anon:
+            if not key:
+                key = self.api_key if not admin else self.master_api_key
+            data = data.copy()
+            data['key'] = key
+        return requests.post("%s/%s" % (self.api_url, path), data=data, files=files)
+
+    def _delete(self, path, data={}, key=None, admin=False, anon=False):
+        if not anon:
+            if not key:
+                key = self.api_key if not admin else self.master_api_key
+            data = data.copy()
+            data['key'] = key
+        return requests.delete("%s/%s" % (self.api_url, path), params=data)
+
+    def _patch(self, path, data={}, key=None, admin=False, anon=False):
+        if not anon:
+            if not key:
+                key = self.api_key if not admin else self.master_api_key
+            params = dict(key=key)
+            data = data.copy()
+            data['key'] = key
+        else:
+            params = {}
+        return requests.patch("%s/%s" % (self.api_url, path), params=params, data=data)
+
+    def _put(self, path, data={}, key=None, admin=False, anon=False):
+        if not anon:
+            if not key:
+                key = self.api_key if not admin else self.master_api_key
+            params = dict(key=key)
+            data = data.copy()
+            data['key'] = key
+        else:
+            params = {}
+        return requests.put("%s/%s" % (self.api_url, path), params=params, data=data)
+
+    def _get(self, path, data={}, key=None, admin=False, anon=False):
+        if not anon:
+            if not key:
+                key = self.api_key if not admin else self.master_api_key
+            data = data.copy()
+            data['key'] = key
+        if path.startswith("/api"):
+            path = path[len("/api"):]
+        url = "%s/%s" % (self.api_url, path)
+        return requests.get(url, params=data)
+
+
+class RunToolException(Exception):
+
+    def __init__(self, message, inputs=None):
+        super(RunToolException, self).__init__(message)
+        self.inputs = inputs
+
+
+# Galaxy specific methods - rest of this can be used with arbitrary files and such.
+def verify_hid(filename, hda_id, attributes, test_data_path_builder, hid="", dataset_fetcher=None, keep_outputs_dir=False):
+    assert dataset_fetcher is not None
+
+    def verify_extra_files(extra_files):
+        _verify_extra_files_content(extra_files, hda_id, dataset_fetcher=dataset_fetcher, test_data_path_builder=test_data_path_builder, keep_outputs_dir=keep_outputs_dir)
+
+    data = dataset_fetcher(hda_id)
+    item_label = "History item %s" % hid
+    verify(
+        item_label,
+        data,
+        attributes=attributes,
+        filename=filename,
+        get_filename=test_data_path_builder,
+        keep_outputs_dir=keep_outputs_dir,
+        verify_extra_files=verify_extra_files,
+    )
+
+
+def _verify_composite_datatype_file_content(file_name, hda_id, base_name=None, attributes=None, dataset_fetcher=None, test_data_path_builder=None, keep_outputs_dir=False):
+    assert dataset_fetcher is not None
+
+    data = dataset_fetcher(hda_id, base_name)
+    item_label = "History item %s" % hda_id
+    try:
+        verify(
+            item_label,
+            data,
+            attributes=attributes,
+            filename=file_name,
+            get_filename=test_data_path_builder,
+            keep_outputs_dir=keep_outputs_dir,
+        )
+    except AssertionError as err:
+        errmsg = 'Composite file (%s) of %s different than expected, difference:\n' % (base_name, item_label)
+        errmsg += str(err)
+        raise AssertionError(errmsg)
+
+
+def _verify_extra_files_content(extra_files, hda_id, dataset_fetcher, test_data_path_builder, keep_outputs_dir):
+    files_list = []
+    for extra_file_dict in extra_files:
+        extra_file_type = extra_file_dict["type"]
+        extra_file_name = extra_file_dict["name"]
+        extra_file_attributes = extra_file_dict["attributes"]
+        extra_file_value = extra_file_dict["value"]
+
+        if extra_file_type == 'file':
+            files_list.append((extra_file_name, extra_file_value, extra_file_attributes))
+        elif extra_file_type == 'directory':
+            for filename in os.listdir(test_data_path_builder(extra_file_value)):
+                files_list.append((filename, os.path.join(extra_file_value, filename), extra_file_attributes))
+        else:
+            raise ValueError('unknown extra_files type: %s' % extra_file_type)
+    for filename, filepath, attributes in files_list:
+        _verify_composite_datatype_file_content(filepath, hda_id, base_name=filename, attributes=attributes, dataset_fetcher=dataset_fetcher, test_data_path_builder=test_data_path_builder, keep_outputs_dir=keep_outputs_dir)
+
+
+def verify_tool(tool_id, galaxy_interactor, resource_parameters={}, register_job_data=None, test_index=0, tool_version=None):
+    tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version)
+    tool_test_dict = tool_test_dicts[test_index]
+    testdef = ToolTestDescription(tool_test_dict)
+
+    _handle_def_errors(testdef)
+
+    test_history = galaxy_interactor.new_history()
+
+    stage_data_in_history(galaxy_interactor, tool_id, testdef.test_data(), test_history)
+
+    # Once data is ready, run the tool and check the outputs - record API
+    # input, job info, tool run exception, as well as exceptions related to
+    # job output checking and register they with the test plugin so it can
+    # record structured information.
+    tool_inputs = None
+    job_stdio = None
+    job_output_exceptions = None
+    tool_execution_exception = None
+    expected_failure_occurred = False
+    try:
+        try:
+            tool_response = galaxy_interactor.run_tool(testdef, test_history, resource_parameters=resource_parameters)
+            data_list, jobs, tool_inputs = tool_response.outputs, tool_response.jobs, tool_response.inputs
+            data_collection_list = tool_response.output_collections
+        except RunToolException as e:
+            tool_inputs = e.inputs
+            tool_execution_exception = e
+            if not testdef.expect_failure:
+                raise e
+            else:
+                expected_failure_occurred = True
+        except Exception as e:
+            tool_execution_exception = e
+            raise e
+
+        if not expected_failure_occurred:
+            assert data_list or data_collection_list
+
+            try:
+                job_stdio = _verify_outputs(testdef, test_history, jobs, tool_id, data_list, data_collection_list, galaxy_interactor)
+            except JobOutputsError as e:
+                job_stdio = e.job_stdio
+                job_output_exceptions = e.output_exceptions
+                raise e
+            except Exception as e:
+                job_output_exceptions = [e]
+                raise e
+    finally:
+        job_data = {}
+        if tool_inputs is not None:
+            job_data["inputs"] = tool_inputs
+        if job_stdio is not None:
+            job_data["job"] = job_stdio
+        if job_output_exceptions:
+            job_data["output_problems"] = [str(_) for _ in job_output_exceptions]
+        if tool_execution_exception:
+            job_data["execution_problem"] = str(tool_execution_exception)
+        if register_job_data is not None:
+            register_job_data(job_data)
+
+    galaxy_interactor.delete_history(test_history)
+
+
+def _handle_def_errors(testdef):
+    # If the test generation had an error, raise
+    if testdef.error:
+        if testdef.exception:
+            if isinstance(testdef.exception, Exception):
+                raise testdef.exception
+            else:
+                raise Exception(testdef.exception)
+        else:
+            raise Exception("Test parse failure")
+
+
+def _verify_outputs(testdef, history, jobs, tool_id, data_list, data_collection_list, galaxy_interactor):
+    assert len(jobs) == 1, "Test framework logic error, somehow tool test resulted in more than one job."
+    job = jobs[0]
+
+    maxseconds = testdef.maxseconds
+    if testdef.num_outputs is not None:
+        expected = testdef.num_outputs
+        actual = len(data_list)
+        if expected != actual:
+            messaage_template = "Incorrect number of outputs - expected %d, found %s."
+            message = messaage_template % (expected, actual)
+            raise Exception(message)
+    found_exceptions = []
+
+    def register_exception(e):
+        if not found_exceptions:
+            # Only print this stuff out once.
+            for stream in ['stdout', 'stderr']:
+                if stream in job_stdio:
+                    print(_format_stream(job_stdio[stream], stream=stream, format=True), file=sys.stderr)
+        found_exceptions.append(e)
+
+    if testdef.expect_failure:
+        if testdef.outputs:
+            raise Exception("Cannot specify outputs in a test expecting failure.")
+
+    # Wait for the job to complete and register expections if the final
+    # status was not what test was expecting.
+    job_failed = False
+    try:
+        galaxy_interactor.wait_for_job(job['id'], history, maxseconds)
+    except Exception as e:
+        job_failed = True
+        if not testdef.expect_failure:
+            found_exceptions.append(e)
+
+    job_stdio = galaxy_interactor.get_job_stdio(job['id'])
+
+    if not job_failed and testdef.expect_failure:
+        error = AssertionError("Expected job to fail but Galaxy indicated the job successfully completed.")
+        register_exception(error)
+
+    expect_exit_code = testdef.expect_exit_code
+    if expect_exit_code is not None:
+        exit_code = job_stdio["exit_code"]
+        if str(expect_exit_code) != str(exit_code):
+            error = AssertionError("Expected job to complete with exit code %s, found %s" % (expect_exit_code, exit_code))
+            register_exception(error)
+
+    for output_index, output_dict in enumerate(testdef.outputs):
+        # Get the correct hid
+        name = output_dict["name"]
+        outfile = output_dict["value"]
+        attributes = output_dict["attributes"]
+        output_testdef = Bunch(name=name, outfile=outfile, attributes=attributes)
+        try:
+            output_data = data_list[name]
+        except (TypeError, KeyError):
+            # Legacy - fall back on ordered data list access if data_list is
+            # just a list (case with twill variant or if output changes its
+            # name).
+            if hasattr(data_list, "values"):
+                output_data = list(data_list.values())[output_index]
+            else:
+                output_data = data_list[len(data_list) - len(testdef.outputs) + output_index]
+        assert output_data is not None
+        try:
+            galaxy_interactor.verify_output(history, jobs, output_data, output_testdef=output_testdef, tool_id=tool_id, maxseconds=maxseconds)
+        except Exception as e:
+            register_exception(e)
+
+    other_checks = {
+        "command_line": "Command produced by the job",
+        "stdout": "Standard output of the job",
+        "stderr": "Standard error of the job",
+    }
+    for what, description in other_checks.items():
+        if getattr(testdef, what, None) is not None:
+            try:
+                data = job_stdio[what]
+                verify_assertions(data, getattr(testdef, what))
+            except AssertionError as err:
+                errmsg = '%s different than expected\n' % description
+                errmsg += str(err)
+                register_exception(AssertionError(errmsg))
+
+    for output_collection_def in testdef.output_collections:
+        try:
+            name = output_collection_def.name
+            # TODO: data_collection_list is clearly a bad name for dictionary.
+            if name not in data_collection_list:
+                template = "Failed to find output [%s], tool outputs include [%s]"
+                message = template % (name, ",".join(data_collection_list.keys()))
+                raise AssertionError(message)
+
+            # Data collection returned from submission, elements may have been populated after
+            # the job completed so re-hit the API for more information.
+            data_collection_returned = data_collection_list[name]
+            data_collection = galaxy_interactor._get("dataset_collections/%s" % data_collection_returned["id"], data={"instance_type": "history"}).json()
+
+            def get_element(elements, id):
+                for element in elements:
+                    if element["element_identifier"] == id:
+                        return element
+                return False
+
+            expected_collection_type = output_collection_def.collection_type
+            if expected_collection_type:
+                collection_type = data_collection["collection_type"]
+                if expected_collection_type != collection_type:
+                    template = "Expected output collection [%s] to be of type [%s], was of type [%s]."
+                    message = template % (name, expected_collection_type, collection_type)
+                    raise AssertionError(message)
+
+            expected_element_count = output_collection_def.count
+            if expected_element_count:
+                actual_element_count = len(data_collection["elements"])
+                if expected_element_count != actual_element_count:
+                    template = "Expected output collection [%s] to have %s elements, but it had %s."
+                    message = template % (name, expected_element_count, actual_element_count)
+                    raise AssertionError(message)
+
+            def verify_elements(element_objects, element_tests):
+                for element_identifier, (element_outfile, element_attrib) in element_tests.items():
+                    element = get_element(element_objects, element_identifier)
+                    if not element:
+                        template = "Failed to find identifier [%s] for testing, tool generated collection elements [%s]"
+                        message = template % (element_identifier, element_objects)
+                        raise AssertionError(message)
+
+                    element_type = element["element_type"]
+                    if element_type != "dataset_collection":
+                        hda = element["object"]
+                        galaxy_interactor.verify_output_dataset(
+                            history,
+                            hda_id=hda["id"],
+                            outfile=element_outfile,
+                            attributes=element_attrib,
+                            tool_id=tool_id
+                        )
+                    if element_type == "dataset_collection":
+                        elements = element["object"]["elements"]
+                        verify_elements(elements, element_attrib.get("elements", {}))
+
+            verify_elements(data_collection["elements"], output_collection_def.element_tests)
+        except Exception as e:
+            register_exception(e)
+
+    if found_exceptions:
+        raise JobOutputsError(found_exceptions, job_stdio)
+    else:
+        return job_stdio
+
+
+def _format_stream(output, stream, format):
+    output = output or ''
+    if format:
+        msg = "---------------------- >> begin tool %s << -----------------------\n" % stream
+        msg += output + "\n"
+        msg += "----------------------- >> end tool %s << ------------------------\n" % stream
+    else:
+        msg = output
+    return msg
+
+
+class JobOutputsError(AssertionError):
+
+    def __init__(self, output_exceptions, job_stdio):
+        big_message = "\n".join(map(str, output_exceptions))
+        super(JobOutputsError, self).__init__(big_message)
+        self.job_stdio = job_stdio
+        self.output_exceptions = output_exceptions
+
+
+class ToolTestDescription(object):
+    """
+    Encapsulates information about a tool test, and allows creation of a
+    dynamic TestCase class (the unittest framework is very class oriented,
+    doing dynamic tests in this way allows better integration)
+    """
+
+    def __init__(self, processed_test_dict):
+        test_index = processed_test_dict["test_index"]
+        name = processed_test_dict.get('name', 'Test-%d' % (test_index + 1))
+        maxseconds = processed_test_dict.get('maxseconds', DEFAULT_MAX_SECS)
+        if maxseconds is not None:
+            maxseconds = int(maxseconds)
+
+        self.test_index = test_index
+        self.tool_id = processed_test_dict["tool_id"]
+        self.name = name
+        self.maxseconds = maxseconds
+        self.required_files = processed_test_dict.get("required_files", [])
+
+        inputs = processed_test_dict.get("inputs", {})
+        loaded_inputs = {}
+        for key, value in inputs.items():
+            if isinstance(value, dict) and value.get("model_class"):
+                loaded_inputs[key] = TestCollectionDef.from_dict(value)
+            else:
+                loaded_inputs[key] = value
+
+        self.inputs = loaded_inputs
+        self.outputs = processed_test_dict.get("outputs", [])
+        self.num_outputs = processed_test_dict.get("num_outputs", 0)
+
+        self.error = processed_test_dict.get("error", False)
+        self.exception = processed_test_dict.get("exception", None)
+
+        self.output_collections = map(TestCollectionOutputDef.from_dict, processed_test_dict.get("output_collections", []))
+        self.command_line = processed_test_dict.get("command", None)
+        self.stdout = processed_test_dict.get("stdout", None)
+        self.stderr = processed_test_dict.get("stderr", None)
+        self.expect_exit_code = processed_test_dict.get("expect_exit_code", None)
+        self.expect_failure = processed_test_dict.get("expect_failure", False)
+
+    def test_data(self):
+        """
+        Iterator over metadata representing the required files for upload.
+        """
+        return test_data_iter(self.required_files)
+
+    def to_dict(self):
+        inputs_dict = {}
+        for key, value in self.inputs.items():
+            if hasattr(value, "to_dict"):
+                inputs_dict[key] = value.to_dict()
+            else:
+                inputs_dict[key] = value
+
+        return {
+            "inputs": inputs_dict,
+            "outputs": self.outputs,
+            "output_collections": map(lambda o: o.to_dict(), self.output_collections),
+            "num_outputs": self.num_outputs,
+            "command_line": self.command_line,
+            "stdout": self.stdout,
+            "stderr": self.stderr,
+            "expect_exit_code": self.expect_exit_code,
+            "expect_failure": self.expect_failure,
+            "name": self.name,
+            "test_index": self.test_index,
+            "tool_id": self.tool_id,
+            "required_files": self.required_files,
+            "error": self.error,
+            "exception": self.exception,
+        }
+
+
+@nottest
+def test_data_iter(required_files):
+    for fname, extra in required_files:
+        data_dict = dict(
+            fname=fname,
+            metadata=extra.get('metadata', {}),
+            composite_data=extra.get('composite_data', []),
+            ftype=extra.get('ftype', DEFAULT_FTYPE),
+            dbkey=extra.get('dbkey', DEFAULT_DBKEY),
+        )
+        edit_attributes = extra.get('edit_attributes', [])
+
+        # currently only renaming is supported
+        for edit_att in edit_attributes:
+            if edit_att.get('type', None) == 'name':
+                new_name = edit_att.get('value', None)
+                assert new_name, 'You must supply the new dataset name as the value tag of the edit_attributes tag'
+                data_dict['name'] = new_name
+            else:
+                raise Exception('edit_attributes type (%s) is unimplemented' % edit_att.get('type', None))
+
+        yield data_dict

lib/galaxy/tools/verify/script.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+import argparse
+import json
+import sys
+
+from galaxy.tools.verify.interactor import GalaxyInteractorApi, verify_tool
+
+DESCRIPTION = "Script to quickly run a tool test against a running Galaxy instance."
+
+
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv[1:]
+
+    args = _arg_parser().parse_args(argv)
+    galaxy_interactor_kwds = {
+        "galaxy_url": args.galaxy_url,
+        "master_api_key": args.admin_key,
+        "api_key": args.key,
+        "keep_outputs_dir": args.output,
+    }
+    tool_id = args.tool_id
+    test_index = int(args.test_index)
+    tool_version = args.tool_version
+
+    galaxy_interactor = GalaxyInteractorApi(**galaxy_interactor_kwds)
+
+    test_results = []
+
+    def _register_job_data(job_data):
+        test_results.append({
+            'id': tool_id + "-" + str(test_index),
+            'has_data': True,
+            'data': job_data,
+        })
+
+    try:
+        verify_tool(
+            tool_id, galaxy_interactor, test_index=test_index, tool_version=tool_version, register_job_data=_register_job_data
+        )
+    finally:
+        report_obj = {
+            'version': '0.1',
+            'tests': test_results,
+        }
+        output_json = args.output_json
+        if output_json:
+            with open(args.output_json, "w") as f:
+                print(report_obj)
+                json.dump(report_obj, f)
+
+
+def _arg_parser():
+    parser = argparse.ArgumentParser(description=DESCRIPTION)
+    parser.add_argument('-u', '--galaxy-url', default="http://localhost:8080", help='Galaxy URL')
+    parser.add_argument('-k', '--key', default=None, help='Galaxy User API Key')
+    parser.add_argument('-a', '--admin-key', default=None, help='Galaxy Admin API Key')
+    parser.add_argument('-t', '--tool-id', default=None, help='Tool ID')
+    parser.add_argument('--tool-version', default=None, help='Tool Version')
+    parser.add_argument('-i', '--test-index', default=0, help='Tool Test Index (starting at 0)')
+    parser.add_argument('-o', '--output', default=None, help='directory to dump outputs to')
+    parser.add_argument('-j', '--output_json', default=None, help='output metadata json')
+    return parser
+
+
+if __name__ == "__main__":
+    main()

lib/galaxy/tools/verify/test_data.py

@@ -8,6 +8,7 @@ from string import Template
 
 from galaxy.util import (
     asbool,
+    in_directory,
     smart_str
 )
 
@@ -27,32 +28,25 @@ LIST_SEP = re.compile("\s*,\s*")
 
 class TestDataResolver(object):
 
-    def __init__(self, env_var='GALAXY_TEST_FILE_DIR', environ=os.environ):
-        file_dirs = environ.get(env_var, None)
+    def __init__(self, file_dirs=None, env_var='GALAXY_TEST_FILE_DIR', environ=os.environ):
+        if file_dirs is None:
+            file_dirs = environ.get(env_var, None)
+        if file_dirs is None:
+            file_dirs = "test-data,https://github.com/galaxyproject/galaxy-test-data.git"
         if file_dirs:
             self.resolvers = [build_resolver(u, environ) for u in LIST_SEP.split(file_dirs)]
         else:
             self.resolvers = []
 
     def get_filename(self, name):
-        if not self.resolvers:
-            filename = None
-        else:
-            resolver = self.resolvers[0]
+        filename = None
+        for resolver in self.resolvers or []:
+            if not resolver.exists(name):
+                continue
             filename = resolver.path(name)
-            if not resolver.exists(filename):
-                for resolver in self.resolvers[1:]:
-                    if resolver.exists(name):
-                        filename = resolver.path(name)
-            else:
-                # For backward compat. returning first path if none
-                # exist - though I don't know if this function is ever
-                # actually used in a context where one should return
-                # a file even if it doesn't exist (e.g. a prefix or
-                # or something) - I am pretty sure it is not used in
-                # such a fashion in the context of tool tests.
-                filename = resolver.path(name)
-        return os.path.abspath(filename)
+
+        if filename:
+            return os.path.abspath(filename)
 
 
 def build_resolver(uri, environ):
@@ -68,7 +62,8 @@ class FileDataResolver(object):
         self.file_dir = file_dir
 
     def exists(self, filename):
-        return os.path.exists(self.path(filename))
+        path = os.path.abspath(self.path(filename))
+        return os.path.exists(path) and in_directory(path, self.file_dir)
 
     def path(self, filename):
         return os.path.join(self.file_dir, filename)

lib/galaxy/util/path/__init__.py

@@ -6,7 +6,10 @@ import errno
 import imp
 import logging
 from functools import partial
-from grp import getgrgid
+try:
+    from grp import getgrgid
+except ImportError:
+    getgrgid = None
 from itertools import starmap
 from operator import getitem
 from os import (
@@ -28,7 +31,10 @@ from os.path import (
     relpath,
     sep as separator,
 )
-from pwd import getpwuid
+try:
+    from pwd import getpwuid
+except ImportError:
+    getpwuid = None
 
 from six import iteritems, string_types
 from six.moves import filter, map, zip
@@ -165,6 +171,9 @@ def __path_permission_for_user(path, username):
     :type username:     string
     :param username:    a username matching the systems username
     """
+    if getpwuid is None:
+        raise NotImplementedError("This functionality is not implemented for Windows.")
+
     group_id_of_file = stat(path).st_gid
     file_owner = getpwuid(stat(path).st_uid)
     group_members = getgrgid(group_id_of_file).gr_mem

lib/galaxy/util/sanitize_html.py

@@ -1,427 +1,45 @@
 """
-HTML Sanitizer (ripped from feedparser)
+HTML Sanitizer (lists of acceptable_* ripped from feedparser)
 """
+import bleach
 
-import re
-import sgmllib
+_acceptable_elements = ['a', 'abbr', 'acronym', 'address', 'area', 'article',
+        'aside', 'audio', 'b', 'big', 'blockquote', 'br', 'button', 'canvas',
+        'caption', 'center', 'cite', 'code', 'col', 'colgroup', 'command',
+        'datagrid', 'datalist', 'dd', 'del', 'details', 'dfn', 'dialog', 'dir',
+        'div', 'dl', 'dt', 'em', 'event-source', 'fieldset', 'figure',
+        'footer', 'font', 'form', 'header', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
+        'hr', 'i', 'img', 'input', 'ins', 'keygen', 'kbd', 'label', 'legend',
+        'li', 'm', 'map', 'menu', 'meter', 'multicol', 'nav', 'nextid', 'ol',
+        'output', 'optgroup', 'option', 'p', 'pre', 'progress', 'q', 's',
+        'samp', 'section', 'select', 'small', 'sound', 'source', 'spacer',
+        'span', 'strike', 'strong', 'sub', 'sup', 'table', 'tbody', 'td',
+        'textarea', 'time', 'tfoot', 'th', 'thead', 'tr', 'tt', 'u', 'ul',
+        'var', 'video', 'noscript']
 
-from six import unichr
-from six.moves.html_entities import name2codepoint
-
-from galaxy.util import unicodify
-
-_cp1252 = {
-    unichr(128): unichr(8364),  # euro sign
-    unichr(130): unichr(8218),  # single low-9 quotation mark
-    unichr(131): unichr(402),  # latin small letter f with hook
-    unichr(132): unichr(8222),  # double low-9 quotation mark
-    unichr(133): unichr(8230),  # horizontal ellipsis
-    unichr(134): unichr(8224),  # dagger
-    unichr(135): unichr(8225),  # double dagger
-    unichr(136): unichr(710),  # modifier letter circumflex accent
-    unichr(137): unichr(8240),  # per mille sign
-    unichr(138): unichr(352),  # latin capital letter s with caron
-    unichr(139): unichr(8249),  # single left-pointing angle quotation mark
-    unichr(140): unichr(338),  # latin capital ligature oe
-    unichr(142): unichr(381),  # latin capital letter z with caron
-    unichr(145): unichr(8216),  # left single quotation mark
-    unichr(146): unichr(8217),  # right single quotation mark
-    unichr(147): unichr(8220),  # left double quotation mark
-    unichr(148): unichr(8221),  # right double quotation mark
-    unichr(149): unichr(8226),  # bullet
-    unichr(150): unichr(8211),  # en dash
-    unichr(151): unichr(8212),  # em dash
-    unichr(152): unichr(732),  # small tilde
-    unichr(153): unichr(8482),  # trade mark sign
-    unichr(154): unichr(353),  # latin small letter s with caron
-    unichr(155): unichr(8250),  # single right-pointing angle quotation mark
-    unichr(156): unichr(339),  # latin small ligature oe
-    unichr(158): unichr(382),  # latin small letter z with caron
-    unichr(159): unichr(376)}  # latin capital letter y with diaeresis
+_acceptable_attributes = ['abbr', 'accept', 'accept-charset', 'accesskey',
+        'action', 'align', 'alt', 'autocomplete', 'autofocus', 'axis',
+        'background', 'balance', 'bgcolor', 'bgproperties', 'border',
+        'bordercolor', 'bordercolordark', 'bordercolorlight', 'bottompadding',
+        'cellpadding', 'cellspacing', 'ch', 'challenge', 'char', 'charoff',
+        'choff', 'charset', 'checked', 'cite', 'class', 'clear', 'color',
+        'cols', 'colspan', 'compact', 'contenteditable', 'controls', 'coords',
+        'data', 'datafld', 'datapagesize', 'datasrc', 'datetime', 'default',
+        'delay', 'dir', 'disabled', 'draggable', 'dynsrc', 'enctype', 'end',
+        'face', 'for', 'form', 'frame', 'galleryimg', 'gutter', 'headers',
+        'height', 'hidefocus', 'hidden', 'high', 'href', 'hreflang', 'hspace',
+        'icon', 'id', 'inputmode', 'ismap', 'keytype', 'label', 'leftspacing',
+        'lang', 'list', 'longdesc', 'loop', 'loopcount', 'loopend',
+        'loopstart', 'low', 'lowsrc', 'max', 'maxlength', 'media', 'method',
+        'min', 'multiple', 'name', 'nohref', 'noshade', 'nowrap', 'open',
+        'optimum', 'pattern', 'ping', 'point-size', 'prompt', 'pqg',
+        'radiogroup', 'readonly', 'rel', 'repeat-max', 'repeat-min', 'replace',
+        'required', 'rev', 'rightspacing', 'rows', 'rowspan', 'rules', 'scope',
+        'selected', 'shape', 'size', 'span', 'src', 'start', 'step', 'summary',
+        'suppress', 'tabindex', 'target', 'template', 'title', 'toppadding',
+        'type', 'unselectable', 'usemap', 'urn', 'valign', 'value', 'variable',
+        'volume', 'vspace', 'vrml', 'width', 'wrap', 'xml:lang']
 
 
-class _BaseHTMLProcessor(sgmllib.SGMLParser):
-    special = re.compile('''[<>'"]''')
-    bare_ampersand = re.compile("&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)")
-    elements_no_end_tag = ['area', 'base', 'basefont', 'br', 'col', 'frame', 'hr',
-                           'img', 'input', 'isindex', 'link', 'meta', 'param']
-
-    def __init__(self, encoding, type):
-        self.encoding = encoding
-        self.type = type
-        # if _debug: sys.stderr.write('entering BaseHTMLProcessor, encoding=%s\n' % self.encoding)
-        sgmllib.SGMLParser.__init__(self)
-
-    def reset(self):
-        self.pieces = []
-        sgmllib.SGMLParser.reset(self)
-
-    def _shorttag_replace(self, match):
-        tag = match.group(1)
-        if tag in self.elements_no_end_tag:
-            return '<' + tag + ' />'
-        else:
-            return '<' + tag + '></' + tag + '>'
-
-    def parse_starttag(self, i):
-        j = sgmllib.SGMLParser.parse_starttag(self, i)
-        if self.type == 'application/xhtml+xml':
-            if j > 2 and self.rawdata[j - 2:j] == '/>':
-                self.unknown_endtag(self.lasttag)
-        return j
-
-    def feed(self, data):
-        data = re.compile(r'<!((?!DOCTYPE|--|\[))', re.IGNORECASE).sub(r'&lt;!\1', data)
-        data = re.sub(r'<([^<>\s]+?)\s*/>', self._shorttag_replace, data)
-        data = data.replace('&#39;', "'")
-        data = data.replace('&#34;', '"')
-        sgmllib.SGMLParser.feed(self, data)
-        sgmllib.SGMLParser.close(self)
-
-    def normalize_attrs(self, attrs):
-        if not attrs:
-            return attrs
-        # utility method to be called by descendants
-        attrs = dict([(k.lower(), v) for k, v in attrs]).items()
-        attrs = [(k, k in ('rel', 'type') and v.lower() or v) for k, v in attrs]
-        attrs.sort()
-        return attrs
-
-    def unknown_starttag(self, tag, attrs):
-        # called for each start tag
-        # attrs is a list of (attr, value) tuples
-        # e.g. for <pre class='screen'>, tag='pre', attrs=[('class', 'screen')]
-        uattrs = []
-        strattrs = ''
-        if attrs:
-            for key, value in attrs:
-                value = value.replace('>', '&gt;').replace('<', '&lt;').replace('"', '&quot;')
-                value = self.bare_ampersand.sub("&amp;", value)
-                uattrs.append((key, value))
-            strattrs = ''.join([' %s="%s"' % (k, v) for k, v in uattrs])
-        if tag in self.elements_no_end_tag:
-            self.pieces.append('<%s%s />' % (tag, strattrs))
-        else:
-            self.pieces.append('<%s%s>' % (tag, strattrs))
-
-    def unknown_endtag(self, tag):
-        # called for each end tag, e.g. for </pre>, tag will be 'pre'
-        # Reconstruct the original end tag.
-        if tag not in self.elements_no_end_tag:
-            self.pieces.append("</%(tag)s>" % locals())
-
-    def handle_charref(self, ref):
-        # called for each character reference, e.g. for '&#160;', ref will be '160'
-        # Reconstruct the original character reference.
-        if ref.startswith('x'):
-            value = unichr(int(ref[1:], 16))
-        else:
-            value = unichr(int(ref))
-
-        if value in _cp1252.keys():
-            self.pieces.append('&#%s;' % hex(ord(_cp1252[value]))[1:])
-        else:
-            self.pieces.append('&#%(ref)s;' % locals())
-
-    def handle_entityref(self, ref):
-        # called for each entity reference, e.g. for '&copy;', ref will be 'copy'
-        # Reconstruct the original entity reference.
-        if ref in name2codepoint:
-            self.pieces.append('&%(ref)s;' % locals())
-        else:
-            self.pieces.append('&amp;%(ref)s' % locals())
-
-    def handle_data(self, text):
-        # called for each block of plain text, i.e. outside of any tag and
-        # not containing any character or entity references
-        # Store the original text verbatim.
-        self.pieces.append(text)
-
-    def handle_comment(self, text):
-        # called for each HTML comment, e.g. <!-- insert Javascript code here -->
-        # Reconstruct the original comment.
-        self.pieces.append('<!--%(text)s-->' % locals())
-
-    def handle_pi(self, text):
-        # called for each processing instruction, e.g. <?instruction>
-        # Reconstruct original processing instruction.
-        self.pieces.append('<?%(text)s>' % locals())
-
-    def handle_decl(self, text):
-        # called for the DOCTYPE, if present, e.g.
-        # <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
-        #     "http://www.w3.org/TR/html4/loose.dtd">
-        # Reconstruct original DOCTYPE
-        self.pieces.append('<!%(text)s>' % locals())
-
-    _new_declname_match = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9:]*\s*').match
-
-    def _scan_name(self, i, declstartpos):
-        rawdata = self.rawdata
-        n = len(rawdata)
-        if i == n:
-            return None, -1
-        m = self._new_declname_match(rawdata, i)
-        if m:
-            s = m.group()
-            name = s.strip()
-            if (i + len(s)) == n:
-                return None, -1  # end of buffer
-            return name.lower(), m.end()
-        else:
-            self.handle_data(rawdata)
-#            self.updatepos(declstartpos, i)
-            return None, -1
-
-    def convert_charref(self, name):
-        return '&#%s;' % name
-
-    def convert_entityref(self, name):
-        return '&%s;' % name
-
-    def output(self):
-        '''Return processed HTML as a single string'''
-        return ''.join(self.pieces)
-
-
-class _HTMLSanitizer(_BaseHTMLProcessor):
-    acceptable_elements = ['a', 'abbr', 'acronym', 'address', 'area', 'article',
-                           'aside', 'audio', 'b', 'big', 'blockquote', 'br', 'button', 'canvas',
-                           'caption', 'center', 'cite', 'code', 'col', 'colgroup', 'command',
-                           'datagrid', 'datalist', 'dd', 'del', 'details', 'dfn', 'dialog', 'dir',
-                           'div', 'dl', 'dt', 'em', 'event-source', 'fieldset', 'figure', 'footer',
-                           'font', 'form', 'header', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i',
-                           'img', 'input', 'ins', 'keygen', 'kbd', 'label', 'legend', 'li', 'm', 'map',
-                           'menu', 'meter', 'multicol', 'nav', 'nextid', 'ol', 'output', 'optgroup',
-                           'option', 'p', 'pre', 'progress', 'q', 's', 'samp', 'section', 'select',
-                           'small', 'sound', 'source', 'spacer', 'span', 'strike', 'strong', 'sub',
-                           'sup', 'table', 'tbody', 'td', 'textarea', 'time', 'tfoot', 'th', 'thead',
-                           'tr', 'tt', 'u', 'ul', 'var', 'video', 'noscript']
-
-    acceptable_attributes = ['abbr', 'accept', 'accept-charset', 'accesskey',
-                             'action', 'align', 'alt', 'autocomplete', 'autofocus', 'axis',
-                             'background', 'balance', 'bgcolor', 'bgproperties', 'border',
-                             'bordercolor', 'bordercolordark', 'bordercolorlight', 'bottompadding',
-                             'cellpadding', 'cellspacing', 'ch', 'challenge', 'char', 'charoff',
-                             'choff', 'charset', 'checked', 'cite', 'class', 'clear', 'color', 'cols',
-                             'colspan', 'compact', 'contenteditable', 'controls', 'coords', 'data',
-                             'datafld', 'datapagesize', 'datasrc', 'datetime', 'default', 'delay',
-                             'dir', 'disabled', 'draggable', 'dynsrc', 'enctype', 'end', 'face', 'for',
-                             'form', 'frame', 'galleryimg', 'gutter', 'headers', 'height', 'hidefocus',
-                             'hidden', 'high', 'href', 'hreflang', 'hspace', 'icon', 'id', 'inputmode',
-                             'ismap', 'keytype', 'label', 'leftspacing', 'lang', 'list', 'longdesc',
-                             'loop', 'loopcount', 'loopend', 'loopstart', 'low', 'lowsrc', 'max',
-                             'maxlength', 'media', 'method', 'min', 'multiple', 'name', 'nohref',
-                             'noshade', 'nowrap', 'open', 'optimum', 'pattern', 'ping', 'point-size',
-                             'prompt', 'pqg', 'radiogroup', 'readonly', 'rel', 'repeat-max',
-                             'repeat-min', 'replace', 'required', 'rev', 'rightspacing', 'rows',
-                             'rowspan', 'rules', 'scope', 'selected', 'shape', 'size', 'span', 'src',
-                             'start', 'step', 'summary', 'suppress', 'tabindex', 'target', 'template',
-                             'title', 'toppadding', 'type', 'unselectable', 'usemap', 'urn', 'valign',
-                             'value', 'variable', 'volume', 'vspace', 'vrml', 'width', 'wrap',
-                             'xml:lang']
-
-    unacceptable_elements_with_end_tag = ['script', 'applet', 'style']
-
-    acceptable_css_properties = ['azimuth', 'background-color',
-                                 'border-bottom-color', 'border-collapse', 'border-color',
-                                 'border-left-color', 'border-right-color', 'border-top-color', 'clear',
-                                 'color', 'cursor', 'direction', 'display', 'elevation', 'float', 'font',
-                                 'font-family', 'font-size', 'font-style', 'font-variant', 'font-weight',
-                                 'height', 'letter-spacing', 'line-height', 'overflow', 'pause',
-                                 'pause-after', 'pause-before', 'pitch', 'pitch-range', 'richness',
-                                 'speak', 'speak-header', 'speak-numeral', 'speak-punctuation',
-                                 'speech-rate', 'stress', 'text-align', 'text-decoration', 'text-indent',
-                                 'unicode-bidi', 'vertical-align', 'voice-family', 'volume',
-                                 'white-space', 'width']
-
-    # survey of common keywords found in feeds
-    acceptable_css_keywords = ['auto', 'aqua', 'black', 'block', 'blue',
-                               'bold', 'both', 'bottom', 'brown', 'center', 'collapse', 'dashed',
-                               'dotted', 'fuchsia', 'gray', 'green', '!important', 'italic', 'left',
-                               'lime', 'maroon', 'medium', 'none', 'navy', 'normal', 'nowrap', 'olive',
-                               'pointer', 'purple', 'red', 'right', 'solid', 'silver', 'teal', 'top',
-                               'transparent', 'underline', 'white', 'yellow']
-
-    valid_css_values = re.compile('^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|' +
-                                  '\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$')
-
-    mathml_elements = ['annotation', 'annotation-xml', 'maction', 'math',
-                       'merror', 'mfenced', 'mfrac', 'mi', 'mmultiscripts', 'mn', 'mo', 'mover', 'mpadded',
-                       'mphantom', 'mprescripts', 'mroot', 'mrow', 'mspace', 'msqrt', 'mstyle',
-                       'msub', 'msubsup', 'msup', 'mtable', 'mtd', 'mtext', 'mtr', 'munder',
-                       'munderover', 'none', 'semantics']
-
-    mathml_attributes = ['actiontype', 'align', 'columnalign', 'columnalign',
-                         'columnalign', 'close', 'columnlines', 'columnspacing', 'columnspan', 'depth',
-                         'display', 'displaystyle', 'encoding', 'equalcolumns', 'equalrows',
-                         'fence', 'fontstyle', 'fontweight', 'frame', 'height', 'linethickness',
-                         'lspace', 'mathbackground', 'mathcolor', 'mathvariant', 'mathvariant',
-                         'maxsize', 'minsize', 'open', 'other', 'rowalign', 'rowalign', 'rowalign',
-                         'rowlines', 'rowspacing', 'rowspan', 'rspace', 'scriptlevel', 'selection',
-                         'separator', 'separators', 'stretchy', 'width', 'width', 'xlink:href',
-                         'xlink:show', 'xlink:type', 'xmlns', 'xmlns:xlink']
-
-    # svgtiny - foreignObject + linearGradient + radialGradient + stop
-    svg_elements = ['a', 'animate', 'animateColor', 'animateMotion',
-                    'animateTransform', 'circle', 'defs', 'desc', 'ellipse', 'foreignObject',
-                    'font-face', 'font-face-name', 'font-face-src', 'g', 'glyph', 'hkern',
-                    'linearGradient', 'line', 'marker', 'metadata', 'missing-glyph', 'mpath',
-                    'path', 'polygon', 'polyline', 'radialGradient', 'rect', 'set', 'stop',
-                    'svg', 'switch', 'text', 'title', 'tspan', 'use']
-
-    # svgtiny + class + opacity + offset + xmlns + xmlns:xlink
-    svg_attributes = ['accent-height', 'accumulate', 'additive', 'alphabetic',
-                      'arabic-form', 'ascent', 'attributeName', 'attributeType',
-                      'baseProfile', 'bbox', 'begin', 'by', 'calcMode', 'cap-height',
-                      'class', 'color', 'color-rendering', 'content', 'cx', 'cy', 'd', 'dx',
-                      'dy', 'descent', 'display', 'dur', 'end', 'fill', 'fill-opacity',
-                      'fill-rule', 'font-family', 'font-size', 'font-stretch', 'font-style',
-                      'font-variant', 'font-weight', 'from', 'fx', 'fy', 'g1', 'g2',
-                      'glyph-name', 'gradientUnits', 'hanging', 'height', 'horiz-adv-x',
-                      'horiz-origin-x', 'id', 'ideographic', 'k', 'keyPoints', 'keySplines',
-                      'keyTimes', 'lang', 'mathematical', 'marker-end', 'marker-mid',
-                      'marker-start', 'markerHeight', 'markerUnits', 'markerWidth', 'max',
-                      'min', 'name', 'offset', 'opacity', 'orient', 'origin',
-                      'overline-position', 'overline-thickness', 'panose-1', 'path',
-                      'pathLength', 'points', 'preserveAspectRatio', 'r', 'refX', 'refY',
-                      'repeatCount', 'repeatDur', 'requiredExtensions', 'requiredFeatures',
-                      'restart', 'rotate', 'rx', 'ry', 'slope', 'stemh', 'stemv',
-                      'stop-color', 'stop-opacity', 'strikethrough-position',
-                      'strikethrough-thickness', 'stroke', 'stroke-dasharray',
-                      'stroke-dashoffset', 'stroke-linecap', 'stroke-linejoin',
-                      'stroke-miterlimit', 'stroke-opacity', 'stroke-width', 'systemLanguage',
-                      'target', 'text-anchor', 'to', 'transform', 'type', 'u1', 'u2',
-                      'underline-position', 'underline-thickness', 'unicode', 'unicode-range',
-                      'units-per-em', 'values', 'version', 'viewBox', 'visibility', 'width',
-                      'widths', 'x', 'x-height', 'x1', 'x2', 'xlink:actuate', 'xlink:arcrole',
-                      'xlink:href', 'xlink:role', 'xlink:show', 'xlink:title', 'xlink:type',
-                      'xml:base', 'xml:lang', 'xml:space', 'xmlns', 'xmlns:xlink', 'y', 'y1',
-                      'y2', 'zoomAndPan']
-
-    svg_attr_map = None
-    svg_elem_map = None
-
-    acceptable_svg_properties = ['fill', 'fill-opacity', 'fill-rule',
-                                 'stroke', 'stroke-width', 'stroke-linecap', 'stroke-linejoin',
-                                 'stroke-opacity']
-
-    def reset(self):
-        _BaseHTMLProcessor.reset(self)
-        self.unacceptablestack = 0
-        self.mathmlOK = 0
-        self.svgOK = 0
-
-    def unknown_starttag(self, tag, attrs):
-        acceptable_attributes = self.acceptable_attributes
-        keymap = {}
-        if tag not in self.acceptable_elements or self.svgOK:
-            if tag in self.unacceptable_elements_with_end_tag:
-                self.unacceptablestack += 1
-
-            # not otherwise acceptable, perhaps it is MathML or SVG?
-            if tag == 'math' and ('xmlns', 'http://www.w3.org/1998/Math/MathML') in attrs:
-                self.mathmlOK += 1
-            if tag == 'svg' and ('xmlns', 'http://www.w3.org/2000/svg') in attrs:
-                self.svgOK += 1
-
-            # chose acceptable attributes based on tag class, else bail
-            if self.mathmlOK and tag in self.mathml_elements:
-                acceptable_attributes = self.mathml_attributes
-            elif self.svgOK and tag in self.svg_elements:
-                # for most vocabularies, lowercasing is a good idea.  Many
-                # svg elements, however, are camel case
-                if not self.svg_attr_map:
-                    lower = [attr.lower() for attr in self.svg_attributes]
-                    mix = [a for a in self.svg_attributes if a not in lower]
-                    self.svg_attributes = lower
-                    self.svg_attr_map = dict([(a.lower(), a) for a in mix])
-
-                    lower = [attr.lower() for attr in self.svg_elements]
-                    mix = [a for a in self.svg_elements if a not in lower]
-                    self.svg_elements = lower
-                    self.svg_elem_map = dict([(a.lower(), a) for a in mix])
-                acceptable_attributes = self.svg_attributes
-                tag = self.svg_elem_map.get(tag, tag)
-                keymap = self.svg_attr_map
-            elif tag not in self.acceptable_elements:
-                return
-
-        # declare xlink namespace, if needed
-        if self.mathmlOK or self.svgOK:
-            if any(map(lambda n, v: n.startswith('xlink:'), attrs)):
-                if not ('xmlns:xlink', 'http://www.w3.org/1999/xlink') in attrs:
-                    attrs.append(('xmlns:xlink', 'http://www.w3.org/1999/xlink'))
-
-        clean_attrs = []
-        for key, value in self.normalize_attrs(attrs):
-            if key == "href" and value.strip().startswith("javascript"):
-                pass
-            elif key in acceptable_attributes:
-                key = keymap.get(key, key)
-                clean_attrs.append((key, value))
-            elif key == 'style':
-                pass
-                # clean_value = self.sanitize_style(value)
-                # if clean_value: clean_attrs.append((key,clean_value))
-        _BaseHTMLProcessor.unknown_starttag(self, tag, clean_attrs)
-
-    def unknown_endtag(self, tag):
-        if tag not in self.acceptable_elements:
-            if tag in self.unacceptable_elements_with_end_tag:
-                self.unacceptablestack -= 1
-            if self.mathmlOK and tag in self.mathml_elements:
-                if tag == 'math' and self.mathmlOK:
-                    self.mathmlOK -= 1
-            elif self.svgOK and tag in self.svg_elements:
-                tag = self.svg_elem_map.get(tag, tag)
-                if tag == 'svg' and self.svgOK:
-                    self.svgOK -= 1
-            else:
-                return
-        _BaseHTMLProcessor.unknown_endtag(self, tag)
-
-    def handle_pi(self, text):
-        pass
-
-    def handle_decl(self, text):
-        pass
-
-    def handle_data(self, text):
-        if not self.unacceptablestack:
-            _BaseHTMLProcessor.handle_data(self, text)
-
-    def sanitize_style(self, style):
-        # disallow urls
-        style = re.compile('url\s*\(\s*[^\s)]+?\s*\)\s*').sub(' ', style)
-
-        # gauntlet
-        if not re.match("""^([:,;#%.\sa-zA-Z0-9!]|\w-\w|'[\s\w]+'|"[\s\w]+"|\([\d,\s]+\))*$""", style):
-            return ''
-        if not re.match("^(\s*[-\w]+\s*:\s*[^:;]*(;|$))*$", style):
-            return ''
-
-        clean = []
-        for prop, value in re.findall("([-\w]+)\s*:\s*([^:;]*)", style):
-            if not value:
-                continue
-            if prop.lower() in self.acceptable_css_properties:
-                clean.append(prop + ': ' + value + ';')
-            elif prop.split('-')[0].lower() in ['background', 'border', 'margin', 'padding']:
-                for keyword in value.split():
-                    if keyword not in self.acceptable_css_keywords and \
-                            not self.valid_css_values.match(keyword):
-                        break
-                else:
-                    clean.append(prop + ': ' + value + ';')
-            elif self.svgOK and prop.lower() in self.acceptable_svg_properties:
-                clean.append(prop + ': ' + value + ';')
-        return ' '.join(clean)
-
-
-def sanitize_html(htmlSource, encoding="utf-8", type="text/html"):
-    p = _HTMLSanitizer(encoding, type)
-    p.feed(unicodify(htmlSource, encoding))
-    data = p.output()
-    data = data.strip().replace('\r\n', '\n')
-    return data
+def sanitize_html(htmlSource):
+    return bleach.clean(htmlSource, tags=_acceptable_elements, attributes=_acceptable_attributes, strip=True)

lib/galaxy/util/validation.py

@@ -12,13 +12,13 @@ def validate_and_sanitize_basestring(key, val):
     if not isinstance(val, string_types):
         raise exceptions.RequestParameterInvalidException('%s must be a string or unicode: %s'
                                                           % (key, str(type(val))))
-    return sanitize_html(val, 'utf-8', 'text/html')
+    return sanitize_html(val)
 
 
 def validate_and_sanitize_basestring_list(key, val):
     try:
         assert isinstance(val, list)
-        return [sanitize_html(t, 'utf-8', 'text/html') for t in val]
+        return [sanitize_html(t) for t in val]
     except (AssertionError, TypeError):
         raise exceptions.RequestParameterInvalidException('%s must be a list of strings: %s'
                                                           % (key, str(type(val))))
@@ -32,7 +32,7 @@ def validate_boolean(key, val):
 
 
 # TODO:
-# def validate_integer( self, key, val, min, max ):
-# def validate_float( self, key, val, min, max ):
-# def validate_number( self, key, val, min, max ):
-# def validate_genome_build( self, key, val ):
+# def validate_integer(self, key, val, min, max):
+# def validate_float(self, key, val, min, max):
+# def validate_number(self, key, val, min, max):
+# def validate_genome_build(self, key, val):

lib/galaxy/visualization/plugins/resource_parser.py

@@ -29,7 +29,7 @@ class ResourceParser(object):
     new keys (e.g. dataset_id="NNN" -> hda=<HistoryDatasetAssociation>).
     """
     primitive_parsers = {
-        'str'   : lambda param: galaxy.util.sanitize_html.sanitize_html(param, 'utf-8'),
+        'str'   : lambda param: galaxy.util.sanitize_html.sanitize_html(param),
         'bool'  : lambda param: galaxy.util.string_as_bool(param),
         'int'   : int,
         'float' : float,
@@ -217,7 +217,7 @@ class ResourceParser(object):
         # TODO: ideally this would check v. a list of valid dbkeys
         elif param_type == 'dbkey':
             dbkey = query_param
-            parsed_param = galaxy.util.sanitize_html.sanitize_html(dbkey, 'utf-8')
+            parsed_param = galaxy.util.sanitize_html.sanitize_html(dbkey)
 
         return parsed_param
 

lib/galaxy/web/base/controller.py

@@ -517,7 +517,7 @@ class UsesLibraryMixinItems(SharableItemSecurityMixin):
         # PRECONDITION: folder_id has already been altered to remove the folder prefix ('F')
         # TODO: allow name and other, editable ldda attrs?
         if ldda_message:
-            ldda_message = util.sanitize_html.sanitize_html(ldda_message, 'utf-8')
+            ldda_message = sanitize_html(ldda_message)
 
         # check permissions on (all three?) resources: hda, library, folder
         # TODO: do we really need the library??
@@ -1121,7 +1121,7 @@ class UsesVisualizationMixin(UsesLibraryMixinItems):
         else:
             self.create_item_slug(trans.sa_session, visualization)
         if annotation:
-            annotation = sanitize_html(annotation, 'utf-8', 'text/html')
+            annotation = sanitize_html(annotation)
             # TODO: if this is to stay in the mixin, UsesAnnotations should be added to the superclasses
             #   right now this is depending on the classes that include this mixin to have UsesAnnotations
             self.add_item_annotation(trans.sa_session, trans.user, visualization, annotation)

lib/galaxy/webapps/galaxy/api/annotations.py

@@ -8,7 +8,7 @@ from galaxy import (
     managers
 )
 from galaxy.model.item_attrs import UsesAnnotations
-from galaxy.util import sanitize_html
+from galaxy.util.sanitize_html import sanitize_html
 from galaxy.web import _future_expose_api as expose_api
 from galaxy.web.base.controller import (
     BaseAPIController,
@@ -36,7 +36,7 @@ class BaseAnnotationsController(BaseAPIController, UsesStoredWorkflowMixin, Uses
         if item is not None:
             new_annotation = payload.get("text")
             # TODO: sanitize on display not entry
-            new_annotation = sanitize_html.sanitize_html(new_annotation, 'utf-8', 'text/html')
+            new_annotation = sanitize_html(new_annotation)
 
             self.add_item_annotation(trans.sa_session, trans.get_user(), item, new_annotation)
             trans.sa_session.flush()

lib/galaxy/webapps/galaxy/api/folder_contents.py

@@ -275,7 +275,7 @@ class FolderContentsController(BaseAPIController, UsesLibraryMixin, UsesLibraryM
         from_hdca_id = payload.pop('from_hdca_id', None)
         ldda_message = payload.pop('ldda_message', '')
         if ldda_message:
-            ldda_message = util.sanitize_html.sanitize_html(ldda_message, 'utf-8')
+            ldda_message = util.sanitize_html.sanitize_html(ldda_message)
         try:
             if from_hda_id:
                 decoded_hda_id = self.decode_id(from_hda_id)

lib/galaxy/webapps/galaxy/api/page_revisions.py

@@ -66,7 +66,7 @@ class PageRevisionsController(BaseAPIController, SharableItemSecurityMixin, Uses
         else:
             title = page.title
 
-        content = sanitize_html(content, 'utf-8', 'text/html')
+        content = sanitize_html(content)
 
         page_revision = trans.app.model.PageRevision()
         page_revision.title = title

lib/galaxy/webapps/galaxy/api/pages.py

@@ -81,13 +81,13 @@ class PagesController(BaseAPIController, SharableItemSecurityMixin, UsesAnnotati
             raise exceptions.DuplicatedSlugException("Page slug must be unique")
 
         content = payload.get("content", "")
-        content = sanitize_html(content, 'utf-8', 'text/html')
+        content = sanitize_html(content)
 
         # Create the new stored page
         page = trans.app.model.Page()
         page.title = payload['title']
         page.slug = payload['slug']
-        page_annotation = sanitize_html(payload.get("annotation", ""), 'utf-8', 'text/html')
+        page_annotation = sanitize_html(payload.get("annotation", ""))
         self.add_item_annotation(trans.sa_session, trans.get_user(), page, page_annotation)
         page.user = user
         # And the first (empty) page revision

lib/galaxy/webapps/galaxy/api/tools.py

@@ -6,10 +6,13 @@ from six.moves.urllib.parse import unquote_plus
 import galaxy.queue_worker
 from galaxy import exceptions, managers, util, web
 from galaxy.managers.collections_util import dictify_dataset_collection_instance
+from galaxy.util.json import safe_dumps
+from galaxy.util.odict import odict
 from galaxy.visualization.genomes import GenomeRegion
 from galaxy.web import _future_expose_api as expose_api
 from galaxy.web import _future_expose_api_anonymous as expose_api_anonymous
 from galaxy.web import _future_expose_api_anonymous_and_sessionless as expose_api_anonymous_and_sessionless
+from galaxy.web import _future_expose_api_raw_anonymous_and_sessionless as expose_api_raw_anonymous_and_sessionless
 from galaxy.web.base.controller import BaseAPIController
 from galaxy.web.base.controller import UsesVisualizationMixin
 
@@ -98,6 +101,74 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin):
         tool = self._get_tool(id, tool_version=tool_version, user=trans.user)
         return tool.to_json(trans, kwd.get('inputs', kwd))
 
+    @expose_api
+    @web.require_admin
+    def test_data_path(self, trans, id, **kwd):
+        """
+        GET /api/tools/{tool_id}/test_data_path?tool_version={tool_version}
+        """
+        # TODO: eliminate copy and paste with above code.
+        if 'payload' in kwd:
+            kwd = kwd.get('payload')
+        tool_version = kwd.get('tool_version', None)
+        tool = self._get_tool(id, tool_version=tool_version, user=trans.user)
+        path = tool.test_data_path(kwd.get("filename"))
+        if path:
+            return path
+        else:
+            raise exceptions.ObjectNotFound("Specified test data path not found.")
+
+    @expose_api_anonymous_and_sessionless
+    def tests_summary(self, trans, **kwd):
+        """
+        GET /api/tools/tests_summary
+
+        Fetch summary information for each tool and version combination with tool tests
+        defined. This summary information currently includes tool name and a count of
+        the tests.
+
+        Fetch complete test data for each tool with /api/tools/{tool_id}/test_data?tool_version=<tool_version>
+        """
+        test_counts_by_tool = {}
+        for id, tool in self.app.toolbox.tools():
+            tests = tool.tests
+            if tests:
+                if tool.id not in test_counts_by_tool:
+                    test_counts_by_tool[tool.id] = {}
+                available_versions = test_counts_by_tool[tool.id]
+                available_versions[tool.version] = {
+                    "tool_name": tool.name,
+                    "count": len(tests),
+                }
+        return test_counts_by_tool
+
+    @expose_api_raw_anonymous_and_sessionless
+    def test_data(self, trans, id, **kwd):
+        """
+        GET /api/tools/{tool_id}/test_data?tool_version={tool_version}
+
+        This API endpoint is unstable and experimental. In particular the format of the
+        response has not been entirely nailed down (it exposes too many Galaxy
+        internals/Pythonisms in a rough way). If this endpoint is being used from outside
+        of scripts shipped with Galaxy let us know and please be prepared for the response
+        from this API to change its format in some ways.
+        """
+        # TODO: eliminate copy and paste with above code.
+        if 'payload' in kwd:
+            kwd = kwd.get('payload')
+        tool_version = kwd.get('tool_version', None)
+        tool = self._get_tool(id, tool_version=tool_version, user=trans.user)
+
+        # Encode in this method to handle odict objects in tool representation.
+        def json_encodeify(obj):
+            if isinstance(obj, odict):
+                return dict(obj)
+            else:
+                return obj
+
+        result = [t.to_dict() for t in tool.tests]
+        return safe_dumps(result, default=json_encodeify)
+
     @expose_api
     @web.require_admin
     def reload(self, trans, id, **kwd):

lib/galaxy/webapps/galaxy/api/visualizations.py

@@ -176,7 +176,7 @@ class VisualizationsController(BaseAPIController, UsesVisualizationMixin, Sharab
             if key == 'type':
                 if not isinstance(val, string_types):
                     raise ValidationError('%s must be a string or unicode: %s' % (key, str(type(val))))
-                val = util.sanitize_html.sanitize_html(val, 'utf-8')
+                val = util.sanitize_html.sanitize_html(val)
             elif key == 'config':
                 if not isinstance(val, dict):
                     raise ValidationError('%s must be a dictionary: %s' % (key, str(type(val))))
@@ -184,22 +184,22 @@ class VisualizationsController(BaseAPIController, UsesVisualizationMixin, Sharab
             elif key == 'annotation':
                 if not isinstance(val, string_types):
                     raise ValidationError('%s must be a string or unicode: %s' % (key, str(type(val))))
-                val = util.sanitize_html.sanitize_html(val, 'utf-8')
+                val = util.sanitize_html.sanitize_html(val)
 
             # these are keys that actually only be *updated* at the revision level and not here
             #   (they are still valid for create, tho)
             elif key == 'title':
                 if not isinstance(val, string_types):
                     raise ValidationError('%s must be a string or unicode: %s' % (key, str(type(val))))
-                val = util.sanitize_html.sanitize_html(val, 'utf-8')
+                val = util.sanitize_html.sanitize_html(val)
             elif key == 'slug':
                 if not isinstance(val, string_types):
                     raise ValidationError('%s must be a string: %s' % (key, str(type(val))))
-                val = util.sanitize_html.sanitize_html(val, 'utf-8')
+                val = util.sanitize_html.sanitize_html(val)
             elif key == 'dbkey':
                 if not isinstance(val, string_types):
                     raise ValidationError('%s must be a string or unicode: %s' % (key, str(type(val))))
-                val = util.sanitize_html.sanitize_html(val, 'utf-8')
+                val = util.sanitize_html.sanitize_html(val)
 
             elif key not in valid_but_uneditable_keys:
                 continue

lib/galaxy/webapps/galaxy/buildapp.py

@@ -274,6 +274,9 @@ def populate_api_routes(webapp, app):
     webapp.mapper.connect('/api/tools/all_requirements', action='all_requirements', controller="tools")
     webapp.mapper.connect('/api/tools/{id:.+?}/build', action='build', controller="tools")
     webapp.mapper.connect('/api/tools/{id:.+?}/reload', action='reload', controller="tools")
+    webapp.mapper.connect('/api/tools/tests_summary', action='tests_summary', controller="tools")
+    webapp.mapper.connect('/api/tools/{id:.+?}/test_data_path', action='test_data_path', controller="tools")
+    webapp.mapper.connect('/api/tools/{id:.+?}/test_data', action='test_data', controller="tools")
     webapp.mapper.connect('/api/tools/{id:.+?}/diagnostics', action='diagnostics', controller="tools")
     webapp.mapper.connect('/api/tools/{id:.+?}/citations', action='citations', controller="tools")
     webapp.mapper.connect('/api/tools/{id:.+?}/download', action='download', controller="tools")

lib/galaxy/webapps/galaxy/config_schema.yml

@@ -1817,6 +1817,18 @@ mapping:
         desc: |
           Set maximum size of ngrams
 
+      tool_test_data_directories:
+        type: str
+        default: 'test-data'
+        required: false
+        desc: |
+          Set tool test data directory. The test framework sets this value to
+          'test-data,https://github.com/galaxyproject/galaxy-test-data.git' which will
+          cause Galaxy to clone down extra test data on the fly for certain tools
+          distributed with Galaxy but this is likely not appropriate for production systems.
+          Instead one can simply clone that repository directly and specify a path here
+          instead of a Git HTTP repository.
+
       id_secret:
         type: str
         default: USING THE DEFAULT IS NOT SECURE!

lib/galaxy/webapps/galaxy/controllers/dataset.py

@@ -435,7 +435,7 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE
                 data.datatype.after_setting_metadata(data)
                 # Sanitize annotation before adding it.
                 if payload.get('annotation'):
-                    annotation = sanitize_html(payload.get('annotation'), 'utf-8', 'text/html')
+                    annotation = sanitize_html(payload.get('annotation'))
                     self.add_item_annotation(trans.sa_session, trans.get_user(), data, annotation)
                 # if setting metadata previously failed and all required elements have now been set, clear the failed state.
                 if data._state == trans.model.Dataset.states.FAILED_METADATA and not data.missing_meta():
@@ -700,7 +700,7 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE
             web.httpexceptions.HTTPNotFound()
         if dataset and new_annotation:
             # Sanitize annotation before adding it.
-            new_annotation = sanitize_html(new_annotation, 'utf-8', 'text/html')
+            new_annotation = sanitize_html(new_annotation)
             self.add_item_annotation(trans.sa_session, trans.get_user(), dataset, new_annotation)
             trans.sa_session.flush()
             return new_annotation

lib/galaxy/webapps/galaxy/controllers/page.py

@@ -1,16 +1,75 @@
+import re
 from json import loads
 
 from markupsafe import escape
-from sqlalchemy import and_, desc, false, true
-from sqlalchemy.orm import eagerload, undefer
+from six.moves.html_entities import name2codepoint
+from six.moves.html_parser import HTMLParser
+from sqlalchemy import (
+    and_,
+    desc,
+    false,
+    true
+)
+from sqlalchemy.orm import (
+    eagerload,
+    undefer
+)
 
-from galaxy import managers, model, util, web
+from galaxy import (
+    managers,
+    model,
+    util,
+    web
+)
 from galaxy.model.item_attrs import UsesItemRatings
 from galaxy.util import unicodify
-from galaxy.util.sanitize_html import _BaseHTMLProcessor, sanitize_html
-from galaxy.web import error, url_for
-from galaxy.web.base.controller import BaseUIController, SharableMixin, UsesStoredWorkflowMixin, UsesVisualizationMixin
-from galaxy.web.framework.helpers import grids, time_ago
+from galaxy.util.sanitize_html import sanitize_html
+from galaxy.web import (
+    error,
+    url_for
+)
+from galaxy.web.base.controller import (
+    BaseUIController,
+    SharableMixin,
+    UsesStoredWorkflowMixin,
+    UsesVisualizationMixin
+)
+from galaxy.web.framework.helpers import (
+    grids,
+    time_ago
+)
+
+
+# Copied from https://github.com/kurtmckee/feedparser
+_cp1252 = {
+    128: u'\u20ac',  # euro sign
+    130: u'\u201a',  # single low-9 quotation mark
+    131: u'\u0192',  # latin small letter f with hook
+    132: u'\u201e',  # double low-9 quotation mark
+    133: u'\u2026',  # horizontal ellipsis
+    134: u'\u2020',  # dagger
+    135: u'\u2021',  # double dagger
+    136: u'\u02c6',  # modifier letter circumflex accent
+    137: u'\u2030',  # per mille sign
+    138: u'\u0160',  # latin capital letter s with caron
+    139: u'\u2039',  # single left-pointing angle quotation mark
+    140: u'\u0152',  # latin capital ligature oe
+    142: u'\u017d',  # latin capital letter z with caron
+    145: u'\u2018',  # left single quotation mark
+    146: u'\u2019',  # right single quotation mark
+    147: u'\u201c',  # left double quotation mark
+    148: u'\u201d',  # right double quotation mark
+    149: u'\u2022',  # bullet
+    150: u'\u2013',  # en dash
+    151: u'\u2014',  # em dash
+    152: u'\u02dc',  # small tilde
+    153: u'\u2122',  # trade mark sign
+    154: u'\u0161',  # latin small letter s with caron
+    155: u'\u203a',  # single right-pointing angle quotation mark
+    156: u'\u0153',  # latin small ligature oe
+    158: u'\u017e',  # latin small letter z with caron
+    159: u'\u0178',  # latin capital letter y with diaeresis
+}
 
 
 def format_bool(b):
@@ -223,18 +282,52 @@ class VisualizationSelectionGrid(ItemSelectionGrid):
     )
 
 
-class _PageContentProcessor(_BaseHTMLProcessor):
-    """ Processes page content to produce HTML that is suitable for display. For now, processor renders embedded objects. """
+# Adapted from the _BaseHTMLProcessor class of https://github.com/kurtmckee/feedparser
+class _PageContentProcessor(HTMLParser, object):
+    """
+    Processes page content to produce HTML that is suitable for display.
+    For now, processor renders embedded objects.
+    """
+    bare_ampersand = re.compile("&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)")
+    elements_no_end_tag = set([
+        'area', 'base', 'basefont', 'br', 'col', 'command', 'embed', 'frame',
+        'hr', 'img', 'input', 'isindex', 'keygen', 'link', 'meta', 'param',
+        'source', 'track', 'wbr'
+    ])
 
-    def __init__(self, trans, encoding, type, render_embed_html_fn):
-        _BaseHTMLProcessor.__init__(self, encoding, type)
+    def __init__(self, trans, render_embed_html_fn):
+        HTMLParser.__init__(self)
         self.trans = trans
         self.ignore_content = False
         self.num_open_tags_for_ignore = 0
         self.render_embed_html_fn = render_embed_html_fn
 
-    def unknown_starttag(self, tag, attrs):
-        """ Called for each start tag; attrs is a list of (attr, value) tuples. """
+    def reset(self):
+        self.pieces = []
+        HTMLParser.reset(self)
+
+    def _shorttag_replace(self, match):
+        tag = match.group(1)
+        if tag in self.elements_no_end_tag:
+            return '<' + tag + ' />'
+        else:
+            return '<' + tag + '></' + tag + '>'
+
+    def feed(self, data):
+        data = re.compile(r'<!((?!DOCTYPE|--|\[))', re.IGNORECASE).sub(r'&lt;!\1', data)
+        data = re.sub(r'<([^<>\s]+?)\s*/>', self._shorttag_replace, data)
+        data = data.replace('&#39;', "'")
+        data = data.replace('&#34;', '"')
+        HTMLParser.feed(self, data)
+        HTMLParser.close(self)
+
+    def handle_starttag(self, tag, attrs):
+        """
+        Called for each start tag
+
+        attrs is a list of (attr, value) tuples, e.g. for <pre class='screen'>,
+        tag='pre', attrs=[('class', 'screen')]
+        """
 
         # If ignoring content, just increment tag count and ignore.
         if self.ignore_content:
@@ -264,17 +357,25 @@ class _PageContentProcessor(_BaseHTMLProcessor):
             return
 
         # Default behavior: not ignoring and no embedded content.
-        _BaseHTMLProcessor.unknown_starttag(self, tag, attrs)
+        uattrs = []
+        strattrs = ''
+        if attrs:
+            for key, value in attrs:
+                value = value.replace('>', '&gt;').replace('<', '&lt;').replace('"', '&quot;')
+                value = self.bare_ampersand.sub("&amp;", value)
+                uattrs.append((key, value))
+            strattrs = ''.join(' %s="%s"' % (k, v) for k, v in uattrs)
+        if tag in self.elements_no_end_tag:
+            self.pieces.append('<%s%s />' % (tag, strattrs))
+        else:
+            self.pieces.append('<%s%s>' % (tag, strattrs))
 
-    def handle_data(self, text):
-        """ Called for each block of plain text. """
-        if self.ignore_content:
-            return
-        _BaseHTMLProcessor.handle_data(self, text)
-
-    def unknown_endtag(self, tag):
-        """ Called for each end tag. """
+    def handle_endtag(self, tag):
+        """
+        Called for each end tag
 
+        E.g. for </pre>, tag will be 'pre'
+        """
         # If ignoring content, see if current tag is the end of content to ignore.
         if self.ignore_content:
             self.num_open_tags_for_ignore -= 1
@@ -283,8 +384,63 @@ class _PageContentProcessor(_BaseHTMLProcessor):
                 self.ignore_content = False
             return
 
-        # Default behavior:
-        _BaseHTMLProcessor.unknown_endtag(self, tag)
+        # Default behavior: reconstruct the original end tag.
+        if tag not in self.elements_no_end_tag:
+            self.pieces.append("</%s>" % tag)
+
+    def handle_charref(self, ref):
+        # called for each character reference, e.g. for '&#160;', ref will be '160'
+        # Reconstruct the original character reference.
+        ref = ref.lower()
+        if ref.startswith('x'):
+            value = int(ref[1:], 16)
+        else:
+            value = int(ref)
+
+        if value in _cp1252:
+            self.pieces.append('&#%s;' % hex(ord(_cp1252[value]))[1:])
+        else:
+            self.pieces.append('&#%s;' % ref)
+
+    def handle_entityref(self, ref):
+        # called for each entity reference, e.g. for '&copy;', ref will be 'copy'
+        # Reconstruct the original entity reference.
+        if ref in name2codepoint or ref == 'apos':
+            self.pieces.append('&%s;' % ref)
+        else:
+            self.pieces.append('&amp;%s' % ref)
+
+    def handle_data(self, text):
+        """
+        Called for each block of plain text
+
+        Called outside of any tag and not containing any character or entity
+        references. Store the original text verbatim.
+        """
+        if self.ignore_content:
+            return
+        self.pieces.append(text)
+
+    def handle_comment(self, text):
+        # called for each HTML comment, e.g. <!-- insert Javascript code here -->
+        # Reconstruct the original comment.
+        self.pieces.append('<!--%s-->' % text)
+
+    def handle_decl(self, text):
+        # called for the DOCTYPE, if present, e.g.
+        # <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+        #     "http://www.w3.org/TR/html4/loose.dtd">
+        # Reconstruct original DOCTYPE
+        self.pieces.append('<!%s>' % text)
+
+    def handle_pi(self, text):
+        # called for each processing instruction, e.g. <?instruction>
+        # Reconstruct original processing instruction.
+        self.pieces.append('<?%s>' % text)
+
+    def output(self):
+        '''Return processed HTML as a single string'''
+        return ''.join(self.pieces)
 
 
 class PageController(BaseUIController, SharableMixin,
@@ -387,7 +543,7 @@ class PageController(BaseUIController, SharableMixin,
                 p.slug = p_slug
                 p.user = user
                 if p_annotation:
-                    p_annotation = sanitize_html(p_annotation, 'utf-8', 'text/html')
+                    p_annotation = sanitize_html(p_annotation)
                     self.add_item_annotation(trans.sa_session, user, p, p_annotation)
                 # And the first (empty) page revision
                 p_revision = model.PageRevision()
@@ -449,7 +605,7 @@ class PageController(BaseUIController, SharableMixin,
                 p.title = p_title
                 p.slug = p_slug
                 if p_annotation:
-                    p_annotation = sanitize_html(p_annotation, 'utf-8', 'text/html')
+                    p_annotation = sanitize_html(p_annotation)
                     self.add_item_annotation(trans.sa_session, user, p, p_annotation)
                 trans.sa_session.add(p)
                 trans.sa_session.flush()
@@ -550,7 +706,7 @@ class PageController(BaseUIController, SharableMixin,
         assert page.user == trans.user
 
         # Sanitize content
-        content = sanitize_html(content, 'utf-8', 'text/html')
+        content = sanitize_html(content)
 
         # Add a new revision to the page with the provided content.
         page_revision = model.PageRevision()
@@ -567,7 +723,7 @@ class PageController(BaseUIController, SharableMixin,
             item = trans.sa_session.query(item_class).filter_by(id=item_id).first()
             if not item:
                 raise RuntimeError("cannot find annotated item")
-            text = sanitize_html(annotation_dict['text'], 'utf-8', 'text/html')
+            text = sanitize_html(annotation_dict['text'])
 
             # Add/update annotation.
             if item_id and item_class and text:
@@ -615,8 +771,10 @@ class PageController(BaseUIController, SharableMixin,
         self.security_check(trans, page, False, True)
 
         # Process page content.
-        processor = _PageContentProcessor(trans, 'utf-8', 'text/html', self._get_embed_html)
+        processor = _PageContentProcessor(trans, self._get_embed_html)
         processor.feed(page.latest_revision.content)
+        # Output is string, so convert to unicode for display.
+        page_content = unicodify(processor.output(), 'utf-8')
 
         # Get rating data.
         user_item_rating = 0
@@ -628,8 +786,6 @@ class PageController(BaseUIController, SharableMixin,
                 user_item_rating = 0
         ave_item_rating, num_ratings = self.get_ave_item_rating_data(trans.sa_session, page)
 
-        # Output is string, so convert to unicode for display.
-        page_content = unicodify(processor.output(), 'utf-8')
         return trans.fill_template_mako("page/display.mako", item=page,
                                         item_data=page_content,
                                         user_item_rating=user_item_rating,

lib/galaxy/webapps/galaxy/controllers/visualization.py

@@ -620,7 +620,7 @@ class VisualizationController(BaseUIController, SharableMixin, UsesVisualization
                 v.slug = v_slug
                 v.dbkey = v_dbkey
                 if v_annotation:
-                    v_annotation = sanitize_html(v_annotation, 'utf-8', 'text/html')
+                    v_annotation = sanitize_html(v_annotation)
                     self.add_item_annotation(trans.sa_session, trans.get_user(), v, v_annotation)
                 trans.sa_session.add(v)
                 trans.sa_session.flush()

lib/galaxy/webapps/galaxy/controllers/workflow.py

@@ -4,10 +4,10 @@ import base64
 import json
 import logging
 import os
-import sgmllib
 
 import requests
 from markupsafe import escape
+from six.moves.html_parser import HTMLParser
 from six.moves.http_client import HTTPConnection
 from sqlalchemy import and_
 from sqlalchemy.orm import eagerload, joinedload, lazyload, undefer
@@ -153,16 +153,17 @@ class StoredWorkflowAllPublishedGrid(grids.Grid):
             self.model_class.deleted == expression.false())
 
 
-# Simple SGML parser to get all content in a single tag.
-class SingleTagContentsParser(sgmllib.SGMLParser):
+# Simple HTML parser to get all content in a single tag.
+class SingleTagContentsParser(HTMLParser):
 
     def __init__(self, target_tag):
-        sgmllib.SGMLParser.__init__(self)
+        # Cannot use super() because HTMLParser is an old-style class in Python2
+        HTMLParser.__init__(self)
         self.target_tag = target_tag
         self.cur_tag = None
         self.tag_content = ""
 
-    def unknown_starttag(self, tag, attrs):
+    def handle_starttag(self, tag, attrs):
         """ Called for each start tag. """
         self.cur_tag = tag
 
@@ -404,7 +405,7 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi
         stored = self.get_stored_workflow(trans, id)
         if new_annotation:
             # Sanitize annotation before adding it.
-            new_annotation = sanitize_html(new_annotation, 'utf-8', 'text/html')
+            new_annotation = sanitize_html(new_annotation)
             self.add_item_annotation(trans.sa_session, trans.get_user(), stored, new_annotation)
             trans.sa_session.flush()
             return new_annotation
@@ -547,7 +548,7 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi
             workflow.stored_workflow = stored_workflow
             stored_workflow.latest_workflow = workflow
             # Add annotation.
-            workflow_annotation = sanitize_html(workflow_annotation, 'utf-8', 'text/html')
+            workflow_annotation = sanitize_html(workflow_annotation)
             self.add_item_annotation(trans.sa_session, trans.get_user(), stored_workflow, workflow_annotation)
             # Persist
             session = trans.sa_session
@@ -573,7 +574,7 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi
             workflow.stored_workflow = stored_workflow
             stored_workflow.latest_workflow = workflow
             # Add annotation.
-            workflow_annotation = sanitize_html(workflow_annotation, 'utf-8', 'text/html')
+            workflow_annotation = sanitize_html(workflow_annotation)
             self.add_item_annotation(trans.sa_session, trans.get_user(), stored_workflow, workflow_annotation)
 
             # Persist

lib/tool_shed/util/workflow_util.py

@@ -287,7 +287,7 @@ def get_workflow_from_dict(trans, workflow_dict, tools_metadata, repository_id,
         if trans.webapp.name == 'galaxy':
             annotation = step_dict.get('annotation', '')
             if annotation:
-                annotation = sanitize_html(annotation, 'utf-8', 'text/html')
+                annotation = sanitize_html(annotation)
                 new_step_annotation = trans.model.WorkflowStepAnnotationAssociation()
                 new_step_annotation.annotation = annotation
                 new_step_annotation.user = trans.user
@@ -398,7 +398,7 @@ def save_workflow(trans, workflow, workflow_dict=None):
     stored.latest_workflow = workflow
     stored.user = trans.user
     if workflow_dict and workflow_dict.get('annotation', ''):
-        annotation = sanitize_html(workflow_dict['annotation'], 'utf-8', 'text/html')
+        annotation = sanitize_html(workflow_dict['annotation'])
         new_annotation = trans.model.StoredWorkflowAnnotationAssociation()
         new_annotation.annotation = annotation
         new_annotation.user = trans.user

scripts/microbes/harvest_bacteria.py

@@ -12,10 +12,13 @@ import time
 from ftplib import FTP
 
 import requests
+try:
+    from bs4 import BeautifulSoup
+except ImportError:
+    raise Exception("BeautifulSoup4 library not found, please install it, e.g. with 'pip install BeautifulSoup4'")
 from six.moves.urllib.request import urlretrieve
 
-from BeautifulSoup import BeautifulSoup  # noqa: I100, I202
-from util import (
+from util import (  # noqa: I202
     get_bed_from_genbank,
     get_bed_from_GeneMark,
     get_bed_from_GeneMarkHMM,

test/api/test_tools.py

@@ -135,6 +135,54 @@ class ToolsTestCase(api.ApiTestCase):
         self._assert_has_keys(tool_info, "inputs", "outputs", "panel_section_id")
         return tool_info
 
+    @skip_without_tool("composite_output")
+    def test_test_data_filepath_security(self):
+        test_data_response = self._get("tools/%s/test_data_path?filename=../CONTRIBUTORS.md" % "composite_output", admin=True)
+        assert test_data_response.status_code == 404, test_data_response.json()
+
+    @skip_without_tool("composite_output")
+    def test_test_data_admin_security(self):
+        test_data_response = self._get("tools/%s/test_data_path?filename=../CONTRIBUTORS.md" % "composite_output")
+        assert test_data_response.status_code == 403, test_data_response.json()
+
+    @skip_without_tool("composite_output")
+    def test_test_data_composite_output(self):
+        test_data_response = self._get("tools/%s/test_data" % "composite_output")
+        assert test_data_response.status_code == 200
+        test_data = test_data_response.json()
+        assert len(test_data) == 1
+        test_case = test_data[0]
+        self._assert_has_keys(test_case, "inputs", "outputs", "output_collections", "required_files")
+        assert len(test_case["inputs"]) == 1, test_case
+        # input0 = next(iter(test_case["inputs"].values()))
+
+    @skip_without_tool("collection_two_paired")
+    def test_test_data_collection_two_paired(self):
+        test_data_response = self._get("tools/%s/test_data" % "collection_two_paired")
+        assert test_data_response.status_code == 200
+        test_data = test_data_response.json()
+        assert len(test_data) == 2
+        test_case = test_data[0]
+        self._assert_has_keys(test_case, "inputs", "outputs", "output_collections", "required_files")
+        assert len(test_case["inputs"]) == 3, test_case
+
+    @skip_without_tool("collection_nested_test")
+    def test_test_data_collection_nested(self):
+        test_data_response = self._get("tools/%s/test_data" % "collection_nested_test")
+        assert test_data_response.status_code == 200
+        test_data = test_data_response.json()
+        assert len(test_data) == 2
+        test_case = test_data[0]
+        self._assert_has_keys(test_case, "inputs", "outputs", "output_collections", "required_files")
+        assert len(test_case["inputs"]) == 1, test_case
+
+    @skip_without_tool("simple_constructs_y")
+    def test_test_data_yaml_tools(self):
+        test_data_response = self._get("tools/%s/test_data" % "simple_constructs_y")
+        assert test_data_response.status_code == 200
+        test_data = test_data_response.json()
+        assert len(test_data) == 3
+
     def test_unzip_collection(self):
         with self.dataset_populator.test_history() as history_id:
             hdca_id = self.__build_pair(history_id, ["123", "456"])

test/base/api.py

@@ -15,7 +15,7 @@ from .api_util import (
     OTHER_USER,
     TEST_USER,
 )
-from .interactor import GalaxyInteractorApi as BaseInteractor
+from .interactor import TestCaseGalaxyInteractor as BaseInteractor
 from .testcase import FunctionalTestCase
 
 

test/base/driver_util.py

@@ -1,7 +1,6 @@
 """Scripts for drivers of Galaxy functional tests."""
 
 import fcntl
-import json
 import logging
 import os
 import random
@@ -29,6 +28,7 @@ from six.moves.urllib.parse import urlparse
 
 from galaxy.app import UniverseApplication as GalaxyUniverseApplication
 from galaxy.config import LOGGING_CONFIG_DEFAULT
+from galaxy.tools.verify.interactor import GalaxyInteractorApi, verify_tool
 from galaxy.util import asbool, download_to_file
 from galaxy.util.properties import load_app_properties
 from galaxy.web import buildapp
@@ -37,7 +37,6 @@ from .api_util import get_master_api_key, get_user_api_key
 from .instrument import StructuredTestDataPlugin
 from .nose_util import run
 from .test_logging import logging_config_file
-from .tool_shed_util import parse_tool_panel_config
 
 galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir))
 DEFAULT_WEB_HOST = "localhost"
@@ -475,20 +474,6 @@ def cleanup_directory(tempdir):
 
 def setup_shed_tools_for_test(app, tmpdir, testing_migrated_tools, testing_installed_tools):
     """Modify Galaxy app's toolbox for migrated or installed tool tests."""
-    # Store a jsonified dictionary of tool_id : GALAXY_TEST_FILE_DIR pairs.
-    galaxy_tool_shed_test_file = os.path.join(tmpdir, 'shed_tools_dict')
-    shed_tools_dict = {}
-    if testing_migrated_tools:
-        has_test_data, shed_tools_dict = parse_tool_panel_config(MIGRATED_TOOL_PANEL_CONFIG, shed_tools_dict)
-    elif testing_installed_tools:
-        for shed_tool_config in INSTALLED_TOOL_PANEL_CONFIGS:
-            has_test_data, shed_tools_dict = parse_tool_panel_config(shed_tool_config, shed_tools_dict)
-    # Persist the shed_tools_dict to the galaxy_tool_shed_test_file.
-    with open(galaxy_tool_shed_test_file, 'w') as shed_tools_file:
-        shed_tools_file.write(json.dumps(shed_tools_dict))
-    if not os.path.isabs(galaxy_tool_shed_test_file):
-        galaxy_tool_shed_test_file = os.path.join(galaxy_root, galaxy_tool_shed_test_file)
-    os.environ['GALAXY_TOOL_SHED_TEST_FILE'] = galaxy_tool_shed_test_file
     if testing_installed_tools:
         # TODO: Do this without modifying app - that is a pretty violation
         # of Galaxy's abstraction - we shouldn't require app at all let alone
@@ -923,17 +908,20 @@ class GalaxyTestDriver(TestDriver):
         return functional.test_toolbox
 
     def run_tool_test(self, tool_id, index=0, resource_parameters={}):
-        import functional.test_toolbox
-        functional.test_toolbox.toolbox = self.app.toolbox
-        tool = self.app.toolbox.get_tool(tool_id)
-        testdef = tool.tests[index]
-        test_case_cls = functional.test_toolbox.ToolTestCase
-        test_case = test_case_cls(methodName="setUp")  # NO-OP
-        test_case.shed_tool_id = None
-        test_case.master_api_key = get_master_api_key()
-        test_case.user_api_key = get_user_api_key()
-        test_case.setUp()
-        test_case.do_it(testdef, resource_parameters=resource_parameters)
+        host, port, url = target_url_parts()
+        galaxy_interactor_kwds = {
+            "galaxy_url": url,
+            "master_api_key": get_master_api_key(),
+            "api_key": get_user_api_key(),
+            "keep_outputs_dir": None,
+        }
+        galaxy_interactor = GalaxyInteractorApi(**galaxy_interactor_kwds)
+        verify_tool(
+            tool_id=tool_id,
+            test_index=index,
+            galaxy_interactor=galaxy_interactor,
+            resource_parameters=resource_parameters
+        )
 
 
 def drive_test(test_driver_class):
@@ -942,6 +930,24 @@ def drive_test(test_driver_class):
     sys.exit(test_driver.run())
 
 
+def setup_keep_outdir():
+    keep_outdir = os.environ.get('GALAXY_TEST_SAVE', '')
+    if keep_outdir > '':
+        try:
+            os.makedirs(keep_outdir)
+        except Exception:
+            pass
+    return keep_outdir
+
+
+def target_url_parts():
+    host = os.environ.get('GALAXY_TEST_HOST')
+    port = os.environ.get('GALAXY_TEST_PORT')
+    default_url = "http://%s:%s" % (host, port)
+    url = os.environ.get('GALAXY_TEST_EXTERNAL', default_url)
+    return host, port, url
+
+
 __all__ = (
     "copy_database_template",
     "build_logger",
@@ -952,7 +958,9 @@ __all__ = (
     "database_conf",
     "get_webapp_global_conf",
     "nose_config_and_run",
+    "setup_keep_outdir",
     "setup_galaxy_config",
+    "target_url_parts",
     "TestDriver",
     "wait_for_http_server",
 )

test/base/interactor.py

@@ -1,520 +1,14 @@
-from __future__ import print_function
-
-import os
-import re
-import time
-from json import dumps
-from logging import getLogger
-
-from requests import (
-    delete,
-    get,
-    patch,
-    post,
-    put,
-)
-from six import StringIO, text_type
-
-from galaxy import util
-from galaxy.tools.parser.interface import TestCollectionDef
-from galaxy.util.bunch import Bunch
-from galaxy.util.odict import odict
-
-log = getLogger(__name__)
-
-# Off by default because it can pound the database pretty heavily
-# and result in sqlite errors on larger tests or larger numbers of
-# tests.
-VERBOSE_ERRORS = util.asbool(os.environ.get("GALAXY_TEST_VERBOSE_ERRORS", False))
-UPLOAD_ASYNC = util.asbool(os.environ.get("GALAXY_TEST_UPLOAD_ASYNC", True))
-ERROR_MESSAGE_DATASET_SEP = "--------------------------------------"
+from galaxy.tools.verify.interactor import GalaxyInteractorApi
 
 
-def build_interactor(test_case, type="api"):
-    interactor_class = GALAXY_INTERACTORS[type]
-    return interactor_class(test_case)
-
-
-def stage_data_in_history(galaxy_interactor, all_test_data, history, shed_tool_id=None):
-    # Upload any needed files
-    upload_waits = []
-
-    if UPLOAD_ASYNC:
-        for test_data in all_test_data:
-            upload_waits.append(galaxy_interactor.stage_data_async(test_data, history, shed_tool_id))
-        for upload_wait in upload_waits:
-            upload_wait()
-    else:
-        for test_data in all_test_data:
-            upload_wait = galaxy_interactor.stage_data_async(test_data, history, shed_tool_id)
-            upload_wait()
-
-
-class GalaxyInteractorApi(object):
+class TestCaseGalaxyInteractor(GalaxyInteractorApi):
 
     def __init__(self, functional_test_case, test_user=None):
         self.functional_test_case = functional_test_case
-        self.api_url = "%s/api" % functional_test_case.url.rstrip("/")
-        self.master_api_key = functional_test_case.master_api_key
-        self.api_key = self.__get_user_key(functional_test_case.user_api_key, functional_test_case.master_api_key, test_user=test_user)
-        self.uploads = {}
-
-    def verify_output(self, history_id, jobs, output_data, output_testdef, shed_tool_id, maxseconds):
-        outfile = output_testdef.outfile
-        attributes = output_testdef.attributes
-        name = output_testdef.name
-        self.wait_for_jobs(history_id, jobs, maxseconds)
-        hid = self.__output_id(output_data)
-        # TODO: Twill version verifys dataset is 'ok' in here.
-        self.verify_output_dataset(history_id=history_id, hda_id=hid, outfile=outfile, attributes=attributes, shed_tool_id=shed_tool_id)
-
-        primary_datasets = attributes.get('primary_datasets', {})
-        if primary_datasets:
-            job_id = self._dataset_provenance(history_id, hid)["job_id"]
-            outputs = self._get("jobs/%s/outputs" % (job_id)).json()
-
-        for designation, (primary_outfile, primary_attributes) in primary_datasets.items():
-            primary_output = None
-            for output in outputs:
-                if output["name"] == '__new_primary_file_%s|%s__' % (name, designation):
-                    primary_output = output
-                    break
-
-            if not primary_output:
-                msg_template = "Failed to find primary dataset with designation [%s] for output with name [%s]"
-                msg_args = (designation, name)
-                raise Exception(msg_template % msg_args)
-
-            primary_hda_id = primary_output["dataset"]["id"]
-            self.verify_output_dataset(history_id, primary_hda_id, primary_outfile, primary_attributes, shed_tool_id=shed_tool_id)
-
-    def wait_for_jobs(self, history_id, jobs, maxseconds):
-        for job in jobs:
-            self.wait_for_job(job['id'], history_id, maxseconds)
-
-    def verify_output_dataset(self, history_id, hda_id, outfile, attributes, shed_tool_id):
-        fetcher = self.__dataset_fetcher(history_id)
-        self.functional_test_case.verify_hid(
-            outfile,
-            hda_id=hda_id,
-            attributes=attributes,
-            dataset_fetcher=fetcher,
-            shed_tool_id=shed_tool_id
+        super(TestCaseGalaxyInteractor, self).__init__(
+            galaxy_url=functional_test_case.url,
+            master_api_key=functional_test_case.master_api_key,
+            api_key=functional_test_case.user_api_key,
+            test_user=test_user,
+            keep_outputs_dir=getattr(functional_test_case, "keepOutdir", None),
         )
-        self._verify_metadata(history_id, hda_id, attributes)
-
-    def _verify_metadata(self, history_id, hid, attributes):
-        """Check dataset metadata.
-
-        ftype on output maps to `file_ext` on the hda's API description, `name`, `info`,
-        `dbkey` and `tags` all map to the API description directly. Other metadata attributes
-        are assumed to be datatype-specific and mapped with a prefix of `metadata_`.
-        """
-        metadata = attributes.get('metadata', {}).copy()
-        for key, value in metadata.copy().items():
-            if key not in ['name', 'info', 'tags']:
-                new_key = "metadata_%s" % key
-                metadata[new_key] = metadata[key]
-                del metadata[key]
-            elif key == "info":
-                metadata["misc_info"] = metadata["info"]
-                del metadata["info"]
-        expected_file_type = attributes.get('ftype', None)
-        if expected_file_type:
-            metadata["file_ext"] = expected_file_type
-
-        if metadata:
-            time.sleep(5)
-            dataset = self._get("histories/%s/contents/%s" % (history_id, hid)).json()
-            for key, value in metadata.items():
-                try:
-                    dataset_value = dataset.get(key, None)
-
-                    def compare(val, expected):
-                        if text_type(val) != text_type(expected):
-                            msg = "Dataset metadata verification for [%s] failed, expected [%s] but found [%s]. Dataset API value was [%s]."
-                            msg_params = (key, value, dataset_value, dataset)
-                            msg = msg % msg_params
-                            raise Exception(msg)
-
-                    if isinstance(dataset_value, list):
-                        value = text_type(value).split(",")
-                        if len(value) != len(dataset_value):
-                            msg = "Dataset metadata verification for [%s] failed, expected [%s] but found [%s], lists differ in length. Dataset API value was [%s]."
-                            msg_params = (key, value, dataset_value, dataset)
-                            msg = msg % msg_params
-                            raise Exception(msg)
-                        for val, expected in zip(dataset_value, value):
-                            compare(val, expected)
-                    else:
-                        compare(dataset_value, value)
-                except KeyError:
-                    msg = "Failed to verify dataset metadata, metadata key [%s] was not found." % key
-                    raise Exception(msg)
-
-    def wait_for_job(self, job_id, history_id, maxseconds):
-        self.functional_test_case.wait_for(lambda: not self.__job_ready(job_id, history_id), maxseconds=maxseconds)
-
-    def get_job_stdio(self, job_id):
-        job_stdio = self.__get_job_stdio(job_id).json()
-        return job_stdio
-
-    def __get_job(self, job_id):
-        return self._get('jobs/%s' % job_id)
-
-    def __get_job_stdio(self, job_id):
-        return self._get('jobs/%s?full=true' % job_id)
-
-    def new_history(self):
-        history_json = self._post("histories", {"name": "test_history"}).json()
-        return history_json['id']
-
-    def __output_id(self, output_data):
-        # Allow data structure coming out of tools API - {id: <id>, output_name: <name>, etc...}
-        # or simple id as comes out of workflow API.
-        try:
-            output_id = output_data.get('id')
-        except AttributeError:
-            output_id = output_data
-        return output_id
-
-    def stage_data_async(self, test_data, history_id, shed_tool_id, async=True):
-        fname = test_data['fname']
-        tool_input = {
-            "file_type": test_data['ftype'],
-            "dbkey": test_data['dbkey'],
-        }
-        for elem in test_data.get('metadata', []):
-            tool_input["files_metadata|%s" % elem.get('name')] = elem.get('value')
-
-        composite_data = test_data['composite_data']
-        if composite_data:
-            files = {}
-            for i, composite_file in enumerate(composite_data):
-                file_name = self.functional_test_case.get_filename(composite_file.get('value'), shed_tool_id=shed_tool_id)
-                files["files_%s|file_data" % i] = open(file_name, 'rb')
-                tool_input.update({
-                    "files_%d|type" % i: "upload_dataset",
-                })
-            name = test_data['name']
-        else:
-            file_name = self.functional_test_case.get_filename(fname, shed_tool_id=shed_tool_id)
-            name = test_data.get('name', None)
-            if not name:
-                name = os.path.basename(file_name)
-
-            tool_input.update({
-                "files_0|NAME": name,
-                "files_0|type": "upload_dataset",
-            })
-            files = {
-                "files_0|file_data": open(file_name, 'rb')
-            }
-        submit_response_object = self.__submit_tool(history_id, "upload1", tool_input, extra_data={"type": "upload_dataset"}, files=files)
-        submit_response = submit_response_object.json()
-        try:
-            dataset = submit_response["outputs"][0]
-        except KeyError:
-            raise Exception(submit_response)
-        # raise Exception(str(dataset))
-        hid = dataset['id']
-        self.uploads[os.path.basename(fname)] = self.uploads[fname] = self.uploads[name] = {"src": "hda", "id": hid}
-        return self.__wait_for_history(history_id)
-
-    def run_tool(self, testdef, history_id, resource_parameters={}):
-        # We need to handle the case where we've uploaded a valid compressed file since the upload
-        # tool will have uncompressed it on the fly.
-
-        inputs_tree = testdef.inputs.copy()
-        for key, value in inputs_tree.items():
-            values = [value] if not isinstance(value, list) else value
-            new_values = []
-            for value in values:
-                if isinstance(value, TestCollectionDef):
-                    hdca_id = self._create_collection(history_id, value)
-                    new_values = [dict(src="hdca", id=hdca_id)]
-                elif value in self.uploads:
-                    new_values.append(self.uploads[value])
-                else:
-                    new_values.append(value)
-            inputs_tree[key] = new_values
-
-        if resource_parameters:
-            inputs_tree["__job_resource|__job_resource__select"] = "yes"
-            for key, value in resource_parameters.items():
-                inputs_tree["__job_resource|%s" % key] = value
-
-        # HACK: Flatten single-value lists. Required when using expand_grouping
-        for key, value in inputs_tree.items():
-            if isinstance(value, list) and len(value) == 1:
-                inputs_tree[key] = value[0]
-
-        submit_response = self.__submit_tool(history_id, tool_id=testdef.tool.id, tool_input=inputs_tree)
-        submit_response_object = submit_response.json()
-        try:
-            return Bunch(
-                inputs=inputs_tree,
-                outputs=self.__dictify_outputs(submit_response_object),
-                output_collections=self.__dictify_output_collections(submit_response_object),
-                jobs=submit_response_object['jobs'],
-            )
-        except KeyError:
-            message = "Error creating a job for these tool inputs - %s" % submit_response_object['err_msg']
-            raise RunToolException(message, inputs_tree)
-
-    def _create_collection(self, history_id, collection_def):
-        create_payload = dict(
-            name=collection_def.name,
-            element_identifiers=dumps(self._element_identifiers(collection_def)),
-            collection_type=collection_def.collection_type,
-            history_id=history_id,
-        )
-        return self._post("dataset_collections", data=create_payload).json()["id"]
-
-    def _element_identifiers(self, collection_def):
-        element_identifiers = []
-        for (element_identifier, element) in collection_def.elements:
-            if isinstance(element, TestCollectionDef):
-                subelement_identifiers = self._element_identifiers(element)
-                element = dict(
-                    name=element_identifier,
-                    src="new_collection",
-                    collection_type=element.collection_type,
-                    element_identifiers=subelement_identifiers
-                )
-            else:
-                element_name = element[0]
-                element = self.uploads[element[1]].copy()
-                element["name"] = element_name
-            element_identifiers.append(element)
-        return element_identifiers
-
-    def __dictify_output_collections(self, submit_response):
-        output_collections_dict = odict()
-        for output_collection in submit_response['output_collections']:
-            output_collections_dict[output_collection.get("output_name")] = output_collection
-        return output_collections_dict
-
-    def __dictify_outputs(self, datasets_object):
-        # Convert outputs list to a dictionary that can be accessed by
-        # output_name so can be more flexiable about ordering of outputs
-        # but also allows fallback to legacy access as list mode.
-        outputs_dict = odict()
-        index = 0
-        for output in datasets_object['outputs']:
-            outputs_dict[index] = outputs_dict[output.get("output_name")] = output
-            index += 1
-        # Adding each item twice (once with index for backward compat),
-        # overiding length to reflect the real number of outputs.
-        outputs_dict.__len__ = lambda: index
-        return outputs_dict
-
-    def output_hid(self, output_data):
-        return output_data['id']
-
-    def delete_history(self, history):
-        return None
-
-    def __wait_for_history(self, history_id):
-        def wait():
-            while not self.__history_ready(history_id):
-                pass
-        return wait
-
-    def __job_ready(self, job_id, history_id):
-        if job_id is None:
-            raise ValueError("__job_ready passed empty job_id")
-        job_json = self._get("jobs/%s" % job_id).json()
-        state = job_json['state']
-        try:
-            return self._state_ready(state, error_msg="Job in error state.")
-        except Exception:
-            if VERBOSE_ERRORS:
-                self._summarize_history(history_id)
-            raise
-
-    def __history_ready(self, history_id):
-        if history_id is None:
-            raise ValueError("__history_ready passed empty history_id")
-        history_json = self._get("histories/%s" % history_id).json()
-        state = history_json['state']
-        try:
-            return self._state_ready(state, error_msg="History in error state.")
-        except Exception:
-            if VERBOSE_ERRORS:
-                self._summarize_history(history_id)
-            raise
-
-    def _summarize_history(self, history_id):
-        if history_id is None:
-            raise ValueError("_summarize_history passed empty history_id")
-        print("Problem in history with id %s - summary of datasets below." % history_id)
-        try:
-            history_contents = self.__contents(history_id)
-        except Exception:
-            print("*TEST FRAMEWORK FAILED TO FETCH HISTORY DETAILS*")
-
-        for history_content in history_contents:
-
-            dataset = history_content
-
-            print(ERROR_MESSAGE_DATASET_SEP)
-            dataset_id = dataset.get('id', None)
-            print("| %d - %s (HID - NAME) " % (int(dataset['hid']), dataset['name']))
-            if history_content['history_content_type'] == 'dataset_collection':
-                history_contents_json = self._get("histories/%s/contents/dataset_collections/%s" % (history_id, history_content["id"])).json()
-                print("| Dataset Collection: %s" % history_contents_json)
-                continue
-
-            try:
-                dataset_info = self._dataset_info(history_id, dataset_id)
-                print("| Dataset State:")
-                print(self.format_for_summary(dataset_info.get("state"), "Dataset state is unknown."))
-                print("| Dataset Blurb:")
-                print(self.format_for_summary(dataset_info.get("misc_blurb", ""), "Dataset blurb was empty."))
-                print("| Dataset Info:")
-                print(self.format_for_summary(dataset_info.get("misc_info", ""), "Dataset info is empty."))
-                print("| Peek:")
-                print(self.format_for_summary(dataset_info.get("peek", ""), "Peek unavilable."))
-            except Exception:
-                print("| *TEST FRAMEWORK ERROR FETCHING DATASET DETAILS*")
-            try:
-                provenance_info = self._dataset_provenance(history_id, dataset_id)
-                print("| Dataset Job Standard Output:")
-                print(self.format_for_summary(provenance_info.get("stdout", ""), "Standard output was empty."))
-                print("| Dataset Job Standard Error:")
-                print(self.format_for_summary(provenance_info.get("stderr", ""), "Standard error was empty."))
-            except Exception:
-                print("| *TEST FRAMEWORK ERROR FETCHING JOB DETAILS*")
-            print("|")
-        print(ERROR_MESSAGE_DATASET_SEP)
-
-    def format_for_summary(self, blob, empty_message, prefix="|  "):
-        contents = "\n".join(["%s%s" % (prefix, line.strip()) for line in StringIO(blob).readlines() if line.rstrip("\n\r")])
-        return contents or "%s*%s*" % (prefix, empty_message)
-
-    def _dataset_provenance(self, history_id, id):
-        provenance = self._get("histories/%s/contents/%s/provenance" % (history_id, id)).json()
-        return provenance
-
-    def _dataset_info(self, history_id, id):
-        dataset_json = self._get("histories/%s/contents/%s" % (history_id, id)).json()
-        return dataset_json
-
-    def __contents(self, history_id):
-        history_contents_json = self._get("histories/%s/contents" % history_id).json()
-        return history_contents_json
-
-    def _state_ready(self, state_str, error_msg):
-        if state_str == 'ok':
-            return True
-        elif state_str == 'error':
-            raise Exception(error_msg)
-        return False
-
-    def __submit_tool(self, history_id, tool_id, tool_input, extra_data={}, files=None):
-        data = dict(
-            history_id=history_id,
-            tool_id=tool_id,
-            inputs=dumps(tool_input),
-            **extra_data
-        )
-        return self._post("tools", files=files, data=data)
-
-    def ensure_user_with_email(self, email, password=None):
-        admin_key = self.master_api_key
-        all_users = self._get('users', key=admin_key).json()
-        try:
-            test_user = [user for user in all_users if user["email"] == email][0]
-        except IndexError:
-            username = re.sub('[^a-z-]', '--', email.lower())
-            password = password or 'testpass'
-            # If remote user middleware is enabled - this endpoint consumes
-            # ``remote_user_email`` otherwise it requires ``email``, ``password``
-            # and ``username``.
-            data = dict(
-                remote_user_email=email,
-                email=email,
-                password=password,
-                username=username,
-            )
-            test_user = self._post('users', data, key=admin_key).json()
-        return test_user
-
-    def __get_user_key(self, user_key, admin_key, test_user=None):
-        if not test_user:
-            test_user = "test@bx.psu.edu"
-        if user_key:
-            return user_key
-        test_user = self.ensure_user_with_email(test_user)
-        return self._post("users/%s/api_key" % test_user['id'], key=admin_key).json()
-
-    def __dataset_fetcher(self, history_id):
-        def fetcher(hda_id, base_name=None):
-            url = "histories/%s/contents/%s/display?raw=true" % (history_id, hda_id)
-            if base_name:
-                url += "&filename=%s" % base_name
-            return self._get(url).content
-
-        return fetcher
-
-    def _post(self, path, data={}, files=None, key=None, admin=False, anon=False):
-        if not anon:
-            if not key:
-                key = self.api_key if not admin else self.master_api_key
-            data = data.copy()
-            data['key'] = key
-        return post("%s/%s" % (self.api_url, path), data=data, files=files)
-
-    def _delete(self, path, data={}, key=None, admin=False, anon=False):
-        if not anon:
-            if not key:
-                key = self.api_key if not admin else self.master_api_key
-            data = data.copy()
-            data['key'] = key
-        return delete("%s/%s" % (self.api_url, path), params=data)
-
-    def _patch(self, path, data={}, key=None, admin=False, anon=False):
-        if not anon:
-            if not key:
-                key = self.api_key if not admin else self.master_api_key
-            params = dict(key=key)
-            data = data.copy()
-            data['key'] = key
-        else:
-            params = {}
-        return patch("%s/%s" % (self.api_url, path), params=params, data=data)
-
-    def _put(self, path, data={}, key=None, admin=False, anon=False):
-        if not anon:
-            if not key:
-                key = self.api_key if not admin else self.master_api_key
-            params = dict(key=key)
-            data = data.copy()
-            data['key'] = key
-        else:
-            params = {}
-        return put("%s/%s" % (self.api_url, path), params=params, data=data)
-
-    def _get(self, path, data={}, key=None, admin=False, anon=False):
-        if not anon:
-            if not key:
-                key = self.api_key if not admin else self.master_api_key
-            data = data.copy()
-            data['key'] = key
-        if path.startswith("/api"):
-            path = path[len("/api"):]
-        url = "%s/%s" % (self.api_url, path)
-        return get(url, params=data)
-
-
-class RunToolException(Exception):
-
-    def __init__(self, message, inputs=None):
-        super(RunToolException, self).__init__(message)
-        self.inputs = inputs
-
-
-GALAXY_INTERACTORS = {
-    'api': GalaxyInteractorApi,
-}

test/base/testcase.py

@@ -2,19 +2,14 @@ from __future__ import print_function
 
 import logging
 import os
-import time
 import unittest
-from json import loads
 
-from galaxy.tools.verify import verify
 from galaxy.tools.verify.test_data import TestDataResolver
 from galaxy.web import security
-from .driver_util import GalaxyTestDriver
+from .driver_util import GalaxyTestDriver, setup_keep_outdir, target_url_parts
 
 log = logging.getLogger(__name__)
 
-DEFAULT_TOOL_TEST_WAIT = os.environ.get("GALAXY_TEST_DEFAULT_WAIT", 86400)
-
 
 class FunctionalTestCase(unittest.TestCase):
 
@@ -22,25 +17,9 @@ class FunctionalTestCase(unittest.TestCase):
         # Security helper
         self.security = security.SecurityHelper(id_secret='changethisinproductiontoo')
         self.history_id = os.environ.get('GALAXY_TEST_HISTORY_ID', None)
-        self.host = os.environ.get('GALAXY_TEST_HOST')
-        self.port = os.environ.get('GALAXY_TEST_PORT')
-        default_url = "http://%s:%s" % (self.host, self.port)
-        self.url = os.environ.get('GALAXY_TEST_EXTERNAL', default_url)
+        self.host, self.port, self.url = target_url_parts()
         self.test_data_resolver = TestDataResolver()
-        tool_shed_test_file = os.environ.get('GALAXY_TOOL_SHED_TEST_FILE', None)
-        if tool_shed_test_file:
-            f = open(tool_shed_test_file, 'r')
-            text = f.read()
-            f.close()
-            self.shed_tools_dict = loads(text)
-        else:
-            self.shed_tools_dict = {}
-        self.keepOutdir = os.environ.get('GALAXY_TEST_SAVE', '')
-        if self.keepOutdir > '':
-            try:
-                os.makedirs(self.keepOutdir)
-            except Exception:
-                pass
+        self.keepOutdir = setup_keep_outdir()
 
     @classmethod
     def setUpClass(cls):
@@ -57,91 +36,6 @@ class FunctionalTestCase(unittest.TestCase):
         if cls._test_driver:
             cls._test_driver.tear_down()
 
-    def get_filename(self, filename, shed_tool_id=None):
-        # For tool tests override get_filename to point at an installed tool if shed_tool_id is set.
-        if shed_tool_id and getattr(self, "shed_tools_dict", None):
-            file_dir = self.shed_tools_dict[shed_tool_id]
-            if file_dir:
-                return os.path.abspath(os.path.join(file_dir, filename))
+    def get_filename(self, filename):
+        # No longer used by tool tests - drop if isn't used else where.
         return self.test_data_resolver.get_filename(filename)
-
-    # TODO: Make this more generic, shouldn't be related to tool stuff I guess.
-    def wait_for(self, func, **kwd):
-        sleep_amount = 0.2
-        slept = 0
-        walltime_exceeded = kwd.get("maxseconds", None)
-        if walltime_exceeded is None:
-            walltime_exceeded = DEFAULT_TOOL_TEST_WAIT
-
-        exceeded = True
-        while slept <= walltime_exceeded:
-            result = func()
-            if result:
-                time.sleep(sleep_amount)
-                slept += sleep_amount
-                sleep_amount *= 2
-            else:
-                exceeded = False
-                break
-
-        if exceeded:
-            message = 'Tool test run exceeded walltime [total %s, max %s], terminating.' % (slept, walltime_exceeded)
-            log.info(message)
-            raise AssertionError(message)
-
-    # TODO: Move verify_xxx into GalaxyInteractor or some relevant mixin.
-    def verify_hid(self, filename, hda_id, attributes, shed_tool_id, hid="", dataset_fetcher=None):
-        assert dataset_fetcher is not None
-
-        def get_filename(test_filename):
-            return self.get_filename(test_filename, shed_tool_id=shed_tool_id)
-
-        def verify_extra_files(extra_files):
-            self._verify_extra_files_content(extra_files, hda_id, shed_tool_id=shed_tool_id, dataset_fetcher=dataset_fetcher)
-
-        data = dataset_fetcher(hda_id)
-        item_label = "History item %s" % hid
-        verify(
-            item_label,
-            data,
-            attributes=attributes,
-            filename=filename,
-            get_filename=get_filename,
-            keep_outputs_dir=self.keepOutdir,
-            verify_extra_files=verify_extra_files,
-        )
-
-    def _verify_composite_datatype_file_content(self, file_name, hda_id, base_name=None, attributes=None, dataset_fetcher=None, shed_tool_id=None):
-        assert dataset_fetcher is not None
-
-        def get_filename(test_filename):
-            return self.get_filename(test_filename, shed_tool_id=shed_tool_id)
-
-        data = dataset_fetcher(hda_id, base_name)
-        item_label = "History item %s" % hda_id
-        try:
-            verify(
-                item_label,
-                data,
-                attributes=attributes,
-                filename=file_name,
-                get_filename=get_filename,
-                keep_outputs_dir=self.keepOutdir,
-            )
-        except AssertionError as err:
-            errmsg = 'Composite file (%s) of %s different than expected, difference:\n' % (base_name, item_label)
-            errmsg += str(err)
-            raise AssertionError(errmsg)
-
-    def _verify_extra_files_content(self, extra_files, hda_id, dataset_fetcher, shed_tool_id=None):
-        files_list = []
-        for extra_type, extra_value, extra_name, extra_attributes in extra_files:
-            if extra_type == 'file':
-                files_list.append((extra_name, extra_value, extra_attributes))
-            elif extra_type == 'directory':
-                for filename in os.listdir(self.get_filename(extra_value, shed_tool_id=shed_tool_id)):
-                    files_list.append((filename, os.path.join(extra_value, filename), extra_attributes))
-            else:
-                raise ValueError('unknown extra_files type: %s' % extra_type)
-        for filename, filepath, attributes in files_list:
-            self._verify_composite_datatype_file_content(filepath, hda_id, base_name=filename, attributes=attributes, dataset_fetcher=dataset_fetcher, shed_tool_id=shed_tool_id)

test/base/tool_shed_util.py

@@ -1,86 +0,0 @@
-import logging
-import os
-
-from galaxy.util import galaxy_root_path, parse_xml
-
-log = logging.getLogger(__name__)
-
-# Set a 10 minute timeout for repository installation.
-repository_installation_timeout = 600
-
-
-def get_installed_repository_info(elem, last_galaxy_test_file_dir, last_tested_repository_name, last_tested_changeset_revision, tool_path):
-    """
-    Return the GALAXY_TEST_FILE_DIR, the containing repository name and the
-    change set revision for the tool elem. This only happens when testing
-    tools installed from the tool shed.
-    """
-    tool_config_path = elem.get('file')
-    installed_tool_path_items = tool_config_path.split('/repos/')
-    sans_shed = installed_tool_path_items[1]
-    path_items = sans_shed.split('/')
-    repository_owner = path_items[0]
-    repository_name = path_items[1]
-    changeset_revision = path_items[2]
-    if repository_name != last_tested_repository_name or changeset_revision != last_tested_changeset_revision:
-        # Locate the test-data directory.
-        installed_tool_path = os.path.join(installed_tool_path_items[0], 'repos', repository_owner, repository_name, changeset_revision)
-        for root, dirs, files in os.walk(os.path.join(tool_path, installed_tool_path)):
-            if '.' in dirs:
-                dirs.remove('.hg')
-            if 'test-data' in dirs:
-                return os.path.join(root, 'test-data'), repository_name, changeset_revision
-        return None, repository_name, changeset_revision
-    return last_galaxy_test_file_dir, last_tested_repository_name, last_tested_changeset_revision
-
-
-def parse_tool_panel_config(config, shed_tools_dict):
-    """
-    Parse a shed-related tool panel config to generate the shed_tools_dict. This only happens when testing tools installed from the tool shed.
-    """
-    last_galaxy_test_file_dir = None
-    last_tested_repository_name = None
-    last_tested_changeset_revision = None
-    tool_path = None
-    has_test_data = False
-    tree = parse_xml(config)
-    root = tree.getroot()
-    tool_path = root.get('tool_path')
-    for elem in root:
-        if elem.tag == 'tool':
-            galaxy_test_file_dir, \
-                last_tested_repository_name, \
-                last_tested_changeset_revision = get_installed_repository_info(elem,
-                                                                               last_galaxy_test_file_dir,
-                                                                               last_tested_repository_name,
-                                                                               last_tested_changeset_revision,
-                                                                               tool_path)
-            if galaxy_test_file_dir:
-                if not has_test_data:
-                    has_test_data = True
-                if galaxy_test_file_dir != last_galaxy_test_file_dir:
-                    if not os.path.isabs(galaxy_test_file_dir):
-                        galaxy_test_file_dir = os.path.join(galaxy_root_path, galaxy_test_file_dir)
-                guid = elem.get('guid')
-                shed_tools_dict[guid] = galaxy_test_file_dir
-                last_galaxy_test_file_dir = galaxy_test_file_dir
-        elif elem.tag == 'section':
-            for section_elem in elem:
-                if section_elem.tag == 'tool':
-                    galaxy_test_file_dir, \
-                        last_tested_repository_name, \
-                        last_tested_changeset_revision = get_installed_repository_info(section_elem,
-                                                                                       last_galaxy_test_file_dir,
-                                                                                       last_tested_repository_name,
-                                                                                       last_tested_changeset_revision,
-                                                                                       tool_path)
-                    if galaxy_test_file_dir:
-                        if not has_test_data:
-                            has_test_data = True
-                        if galaxy_test_file_dir != last_galaxy_test_file_dir:
-                            if not os.path.isabs(galaxy_test_file_dir):
-                                galaxy_test_file_dir = os.path.join(galaxy_root_path, galaxy_test_file_dir)
-                        guid = section_elem.get('guid')
-                        shed_tools_dict[guid] = galaxy_test_file_dir
-                        last_galaxy_test_file_dir = galaxy_test_file_dir
-    return has_test_data, shed_tools_dict

test/functional/test_data_managers.py

@@ -4,8 +4,7 @@ import os.path
 import shutil
 import tempfile
 
-from base.interactor import stage_data_in_history
-
+from galaxy.tools.verify.interactor import stage_data_in_history
 from .test_toolbox import ToolTestCase
 
 log = logging.getLogger(__name__)
@@ -19,7 +18,7 @@ class DataManagerToolTestCase(ToolTestCase):
         """
         Run through a tool test case.
         """
-        shed_tool_id = self.shed_tool_id
+        tool_id = self.tool_id
 
         self._handle_test_def_errors(testdef)
 
@@ -27,7 +26,7 @@ class DataManagerToolTestCase(ToolTestCase):
 
         test_history = galaxy_interactor.new_history()  # history where inputs will be put, if any
 
-        stage_data_in_history(galaxy_interactor, testdef.test_data(), test_history, shed_tool_id)
+        stage_data_in_history(galaxy_interactor, tool_id, testdef.test_data(), test_history)
 
         galaxy_interactor.run_tool(testdef, test_history)  # test_history will have inputs only, outputs are placed in the specialized data manager history
 
@@ -43,7 +42,7 @@ class DataManagerToolTestCase(ToolTestCase):
 
         self.assertTrue(data_list)
 
-        self._verify_outputs(testdef, data_manager_history, shed_tool_id, data_list, galaxy_interactor)
+        self._verify_outputs(testdef, data_manager_history, tool_id, data_list, galaxy_interactor)
 
         self.switch_history(id=self.security.encode_id(test_history.id))
 
@@ -89,7 +88,7 @@ def build_tests(tmp_dir=None, testing_shed_tools=False, master_api_key=None, use
             log.warning("No Tool has been specified for Data Manager: %s", data_manager_id)
         if tool.tests:
             # fixme data_manager.tool_shed_repository_info_dict should be filled when is toolshed based
-            shed_tool_id = None if not testing_shed_tools else tool.id
+            tool_id = tool.id
             # Create a new subclass of ToolTestCase, dynamically adding methods
             # named test_tool_XXX that run each test defined in the tool config.
             name = "TestForDataManagerTool_" + data_manager_id.replace(' ', '_')
@@ -103,7 +102,7 @@ def build_tests(tmp_dir=None, testing_shed_tools=False, master_api_key=None, use
                 test_method = make_test_method(testdef)
                 test_method.__doc__ = "%s ( %s ) > %s" % (tool.name, tool.id, testdef.name)
                 namespace['test_tool_%06d' % j] = test_method
-                namespace['shed_tool_id'] = shed_tool_id
+                namespace['tool_id'] = tool_id
                 namespace['master_api_key'] = master_api_key
                 namespace['user_api_key'] = user_api_key
             # The new.classobj function returns a new class object, with name name, derived

test/functional/test_toolbox.py

@@ -2,7 +2,6 @@ from __future__ import print_function
 
 import logging
 import new
-import sys
 
 try:
     from nose.tools import nottest
@@ -10,11 +9,9 @@ except ImportError:
     def nottest(x):
         return x
 
-from base.instrument import register_job_data
-from base.interactor import build_interactor, RunToolException, stage_data_in_history
+from base.driver_util import setup_keep_outdir, target_url_parts
 from galaxy.tools import DataManagerTool  # noqa: I201
-from galaxy.tools.verify.asserts import verify_assertions
-from galaxy.util import bunch
+from galaxy.tools.verify.interactor import GalaxyInteractorApi, verify_tool  # noqa: I201
 from .twilltestcase import TwillTestCase
 
 log = logging.getLogger(__name__)
@@ -32,251 +29,15 @@ class ToolTestCase(TwillTestCase):
     subclass DataManagerToolTestCase requires the use of Twill still.
     """
 
-    def do_it(self, testdef, resource_parameters={}):
+    def do_it(self, tool_id=None, tool_version=None, test_index=0, resource_parameters={}):
         """
         Run through a tool test case.
         """
-        shed_tool_id = self.shed_tool_id
+        if tool_id is None:
+            tool_id = self.tool_id
+        assert tool_id
 
-        self._handle_test_def_errors(testdef)
-
-        galaxy_interactor = self._galaxy_interactor(testdef)
-
-        test_history = galaxy_interactor.new_history()
-
-        stage_data_in_history(galaxy_interactor, testdef.test_data(), test_history, shed_tool_id)
-
-        # Once data is ready, run the tool and check the outputs - record API
-        # input, job info, tool run exception, as well as exceptions related to
-        # job output checking and register they with the test plugin so it can
-        # record structured information.
-        tool_inputs = None
-        job_stdio = None
-        job_output_exceptions = None
-        tool_execution_exception = None
-        expected_failure_occurred = False
-        try:
-            try:
-                tool_response = galaxy_interactor.run_tool(testdef, test_history, resource_parameters=resource_parameters)
-                data_list, jobs, tool_inputs = tool_response.outputs, tool_response.jobs, tool_response.inputs
-                data_collection_list = tool_response.output_collections
-            except RunToolException as e:
-                tool_inputs = e.inputs
-                tool_execution_exception = e
-                if not testdef.expect_failure:
-                    raise e
-                else:
-                    expected_failure_occurred = True
-            except Exception as e:
-                tool_execution_exception = e
-                raise e
-
-            if not expected_failure_occurred:
-                self.assertTrue(data_list or data_collection_list)
-
-                try:
-                    job_stdio = self._verify_outputs(testdef, test_history, jobs, shed_tool_id, data_list, data_collection_list, galaxy_interactor)
-                except JobOutputsError as e:
-                    job_stdio = e.job_stdio
-                    job_output_exceptions = e.output_exceptions
-                    raise e
-                except Exception as e:
-                    job_output_exceptions = [e]
-                    raise e
-        finally:
-            job_data = {}
-            if tool_inputs is not None:
-                job_data["inputs"] = tool_inputs
-            if job_stdio is not None:
-                job_data["job"] = job_stdio
-            if job_output_exceptions:
-                job_data["output_problems"] = [str(_) for _ in job_output_exceptions]
-            if tool_execution_exception:
-                job_data["execution_problem"] = str(tool_execution_exception)
-            register_job_data(job_data)
-
-        galaxy_interactor.delete_history(test_history)
-
-    def _galaxy_interactor(self, testdef):
-        return build_interactor(self, testdef.interactor)
-
-    def _handle_test_def_errors(self, testdef):
-        # If the test generation had an error, raise
-        if testdef.error:
-            if testdef.exception:
-                raise testdef.exception
-            else:
-                raise Exception("Test parse failure")
-
-    def _verify_outputs(self, testdef, history, jobs, shed_tool_id, data_list, data_collection_list, galaxy_interactor):
-        assert len(jobs) == 1, "Test framework logic error, somehow tool test resulted in more than one job."
-        job = jobs[0]
-
-        maxseconds = testdef.maxseconds
-        if testdef.num_outputs is not None:
-            expected = testdef.num_outputs
-            actual = len(data_list)
-            if expected != actual:
-                messaage_template = "Incorrect number of outputs - expected %d, found %s."
-                message = messaage_template % (expected, actual)
-                raise Exception(message)
-        found_exceptions = []
-
-        def register_exception(e):
-            if not found_exceptions:
-                # Only print this stuff out once.
-                for stream in ['stdout', 'stderr']:
-                    if stream in job_stdio:
-                        print(self._format_stream(job_stdio[stream], stream=stream, format=True), file=sys.stderr)
-            found_exceptions.append(e)
-
-        if testdef.expect_failure:
-            if testdef.outputs:
-                raise Exception("Cannot specify outputs in a test expecting failure.")
-
-        # Wait for the job to complete and register expections if the final
-        # status was not what test was expecting.
-        job_failed = False
-        try:
-            galaxy_interactor.wait_for_job(job['id'], history, maxseconds)
-        except Exception as e:
-            job_failed = True
-            if not testdef.expect_failure:
-                found_exceptions.append(e)
-
-        job_stdio = galaxy_interactor.get_job_stdio(job['id'])
-
-        if not job_failed and testdef.expect_failure:
-            error = AssertionError("Expected job to fail but Galaxy indicated the job successfully completed.")
-            register_exception(error)
-
-        expect_exit_code = testdef.expect_exit_code
-        if expect_exit_code is not None:
-            exit_code = job_stdio["exit_code"]
-            if str(expect_exit_code) != str(exit_code):
-                error = AssertionError("Expected job to complete with exit code %s, found %s" % (expect_exit_code, exit_code))
-                register_exception(error)
-
-        for output_index, output_tuple in enumerate(testdef.outputs):
-            # Get the correct hid
-            name, outfile, attributes = output_tuple
-            output_testdef = bunch.Bunch(name=name, outfile=outfile, attributes=attributes)
-            try:
-                output_data = data_list[name]
-            except (TypeError, KeyError):
-                # Legacy - fall back on ordered data list access if data_list is
-                # just a list (case with twill variant or if output changes its
-                # name).
-                if hasattr(data_list, "values"):
-                    output_data = list(data_list.values())[output_index]
-                else:
-                    output_data = data_list[len(data_list) - len(testdef.outputs) + output_index]
-            self.assertTrue(output_data is not None)
-            try:
-                galaxy_interactor.verify_output(history, jobs, output_data, output_testdef=output_testdef, shed_tool_id=shed_tool_id, maxseconds=maxseconds)
-            except Exception as e:
-                register_exception(e)
-
-        other_checks = {
-            "command_line": "Command produced by the job",
-            "stdout": "Standard output of the job",
-            "stderr": "Standard error of the job",
-        }
-        for what, description in other_checks.items():
-            if getattr(testdef, what, None) is not None:
-                try:
-                    data = job_stdio[what]
-                    verify_assertions(data, getattr(testdef, what))
-                except AssertionError as err:
-                    errmsg = '%s different than expected\n' % description
-                    errmsg += str(err)
-                    register_exception(AssertionError(errmsg))
-
-        for output_collection_def in testdef.output_collections:
-            try:
-                name = output_collection_def.name
-                # TODO: data_collection_list is clearly a bad name for dictionary.
-                if name not in data_collection_list:
-                    template = "Failed to find output [%s], tool outputs include [%s]"
-                    message = template % (name, ",".join(data_collection_list.keys()))
-                    raise AssertionError(message)
-
-                # Data collection returned from submission, elements may have been populated after
-                # the job completed so re-hit the API for more information.
-                data_collection_returned = data_collection_list[name]
-                data_collection = galaxy_interactor._get("dataset_collections/%s" % data_collection_returned["id"], data={"instance_type": "history"}).json()
-
-                def get_element(elements, id):
-                    for element in elements:
-                        if element["element_identifier"] == id:
-                            return element
-                    return False
-
-                expected_collection_type = output_collection_def.collection_type
-                if expected_collection_type:
-                    collection_type = data_collection["collection_type"]
-                    if expected_collection_type != collection_type:
-                        template = "Expected output collection [%s] to be of type [%s], was of type [%s]."
-                        message = template % (name, expected_collection_type, collection_type)
-                        raise AssertionError(message)
-
-                expected_element_count = output_collection_def.count
-                if expected_element_count:
-                    actual_element_count = len(data_collection["elements"])
-                    if expected_element_count != actual_element_count:
-                        template = "Expected output collection [%s] to have %s elements, but it had %s."
-                        message = template % (name, expected_element_count, actual_element_count)
-                        raise AssertionError(message)
-
-                def verify_elements(element_objects, element_tests):
-                    for element_identifier, (element_outfile, element_attrib) in element_tests.items():
-                        element = get_element(element_objects, element_identifier)
-                        if not element:
-                            template = "Failed to find identifier [%s] for testing, tool generated collection elements [%s]"
-                            message = template % (element_identifier, element_objects)
-                            raise AssertionError(message)
-
-                        element_type = element["element_type"]
-                        if element_type != "dataset_collection":
-                            hda = element["object"]
-                            galaxy_interactor.verify_output_dataset(
-                                history,
-                                hda_id=hda["id"],
-                                outfile=element_outfile,
-                                attributes=element_attrib,
-                                shed_tool_id=shed_tool_id
-                            )
-                        if element_type == "dataset_collection":
-                            elements = element["object"]["elements"]
-                            verify_elements(elements, element_attrib.get("elements", {}))
-
-                verify_elements(data_collection["elements"], output_collection_def.element_tests)
-            except Exception as e:
-                register_exception(e)
-
-        if found_exceptions:
-            raise JobOutputsError(found_exceptions, job_stdio)
-        else:
-            return job_stdio
-
-    def _format_stream(self, output, stream, format):
-        output = output or ''
-        if format:
-            msg = "---------------------- >> begin tool %s << -----------------------\n" % stream
-            msg += output + "\n"
-            msg += "----------------------- >> end tool %s << ------------------------\n" % stream
-        else:
-            msg = output
-        return msg
-
-
-class JobOutputsError(AssertionError):
-
-    def __init__(self, output_exceptions, job_stdio):
-        big_message = "\n".join(map(str, output_exceptions))
-        super(JobOutputsError, self).__init__(big_message)
-        self.job_stdio = job_stdio
-        self.output_exceptions = output_exceptions
+        verify_tool(tool_id, self.galaxy_interactor, resource_parameters=resource_parameters, test_index=test_index, tool_version=tool_version)
 
 
 @nottest
@@ -286,8 +47,17 @@ def build_tests(app=None, testing_shed_tools=False, master_api_key=None, user_ap
     classes for all of its tests and put them into this modules globals() so
     they can be discovered by nose.
     """
-    if app is None:
-        return
+    # galaxy_interactor = None
+    # if app is None:
+    host, port, url = target_url_parts()
+    keep_outputs_dir = setup_keep_outdir()
+    galaxy_interactor_kwds = {
+        "galaxy_url": url,
+        "master_api_key": master_api_key,
+        "api_key": user_api_key,
+        "keep_outputs_dir": keep_outputs_dir,
+    }
+    galaxy_interactor = GalaxyInteractorApi(**galaxy_interactor_kwds)
 
     # Push all the toolbox tests to module level
     G = globals()
@@ -296,35 +66,40 @@ def build_tests(app=None, testing_shed_tools=False, master_api_key=None, user_ap
     for key, val in G.items():
         if key.startswith('TestForTool_'):
             del G[key]
-    for i, tool_id in enumerate(app.toolbox.tools_by_id):
-        tool = app.toolbox.get_tool(tool_id)
-        if isinstance(tool, TOOL_TYPES_NO_TEST):
-            # We do not test certain types of tools (e.g. Data Manager tools) as part of ToolTestCase
-            continue
-        if tool.tests:
-            shed_tool_id = None if not testing_shed_tools else tool.id
-            # Create a new subclass of ToolTestCase, dynamically adding methods
-            # named test_tool_XXX that run each test defined in the tool config.
-            name = "TestForTool_" + tool.id.replace(' ', '_')
-            baseclasses = (ToolTestCase, )
-            namespace = dict()
-            for j, testdef in enumerate(tool.tests):
-                test_function_name = 'test_tool_%06d' % j
 
-                def make_test_method(td):
+    tests_summary = galaxy_interactor.get_tests_summary()
+    for tool_id, tool_summary in tests_summary.items():
+        # Create a new subclass of ToolTestCase, dynamically adding methods
+        # named test_tool_XXX that run each test defined in the tool config.
+        name = "TestForTool_" + tool_id.replace(' ', '_')
+        baseclasses = (ToolTestCase, )
+        namespace = dict()
+
+        all_versions_test_count = 0
+
+        for tool_version, version_summary in tool_summary.items():
+            count = version_summary["count"]
+            for i in range(count):
+                test_function_name = 'test_tool_%06d' % all_versions_test_count
+
+                def make_test_method(tool_version, test_index):
                     def test_tool(self):
-                        self.do_it(td)
+                        self.do_it(tool_version=tool_version, test_index=test_index)
                     test_tool.__name__ = test_function_name
 
                     return test_tool
 
-                test_method = make_test_method(testdef)
-                test_method.__doc__ = "%s ( %s ) > %s" % (tool.name, tool.id, testdef.name)
+                test_method = make_test_method(tool_version, i)
+                test_method.__doc__ = "( %s ) > Test-%d" % (tool_id, all_versions_test_count + 1)
                 namespace[test_function_name] = test_method
-                namespace['shed_tool_id'] = shed_tool_id
+                namespace['tool_id'] = tool_id
+                namespace["galaxy_interactor"] = galaxy_interactor
                 namespace['master_api_key'] = master_api_key
                 namespace['user_api_key'] = user_api_key
+
+                all_versions_test_count += 1
+
             # The new.classobj function returns a new class object, with name name, derived
             # from baseclasses (which should be a tuple of classes) and with namespace dict.
-            new_class_obj = new.classobj(name, baseclasses, namespace)
+            new_class_obj = new.classobj(str(name), baseclasses, namespace)
             G[name] = new_class_obj

test/functional/twilltestcase.py

@@ -15,7 +15,7 @@ from six import string_types, StringIO
 from six.moves.urllib.parse import urlencode, urlparse
 from twill.other_packages._mechanize_dist import ClientForm
 
-from base.testcase import FunctionalTestCase  # noqa: I100,I202
+from base.testcase import FunctionalTestCase  # noqa: I100,I201,I202
 from galaxy.util import unicodify  # noqa: I201
 
 # Force twill to log to a buffer -- FIXME: Should this go to stdout and be captured by nose?

test/integration/test_data_manager_table_reload.py

@@ -1,38 +1,12 @@
-import os
 import random
 import string
-import tempfile
 
 from base import integration_util
-from base.populators import DatasetPopulator, DEFAULT_TIMEOUT
+from base.populators import DatasetPopulator
 from nose.plugins.skip import SkipTest
 
-# Needs a longer timeout because of the conda_auto_install.
-DATA_MANAGER_JOB_TIMEOUT = DEFAULT_TIMEOUT * 3
+from .uses_shed import CONDA_AUTO_INSTALL_JOB_TIMEOUT, UsesShed
 
-SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
-TOOL_SHEDS_CONF = os.path.join(SCRIPT_DIRECTORY, "tool_sheds_conf.xml")
-
-SHED_TOOL_CONF = string.Template("""<?xml version="1.0"?>
-<toolbox tool_path="$shed_tools_path">
-</toolbox>""")
-
-SHED_DATA_MANAGER_CONF = """<?xml version="1.0"?>
-<data_managers>
-</data_managers>"""
-
-SHED_DATA_TABLES = """<?xml version="1.0"?>
-<tables>
-</tables>"""
-
-CREATE_DBKEY_PAYLOAD = {'tool_shed_url': 'https://toolshed.g2.bx.psu.edu',
-                        'name': 'data_manager_fetch_genome_dbkeys_all_fasta',
-                        'owner': 'devteam',
-                        'changeset_revision': 'b1bc53e9bbc5'}
-SAM_FASTA_PAYLOAD = {'tool_shed_url': 'https://toolshed.g2.bx.psu.edu',
-                     'name': 'data_manager_sam_fasta_index_builder',
-                     'owner': 'devteam',
-                     'changeset_revision': '1865e693d8b2'}
 FETCH_TOOL_ID = 'toolshed.g2.bx.psu.edu/repos/devteam/data_manager_fetch_genome_dbkeys_all_fasta/data_manager_fetch_genome_all_fasta_dbkey/0.0.2'
 FETCH_GENOME_DBKEYS_ALL_FASTA_INPUT = {"dbkey_source|dbkey_source_selector": "new",
                                        "dbkey_source|dbkey": "NC_001617.1",
@@ -46,7 +20,7 @@ SAM_FASTA_ID = "toolshed.g2.bx.psu.edu/repos/devteam/data_manager_sam_fasta_inde
 SAM_FASTA_INPUT = {"all_fasta_source": "NC_001617.1", "sequence_name": "", "sequence_id": ""}
 
 
-class DataManagerIntegrationTestCase(integration_util.IntegrationTestCase):
+class DataManagerIntegrationTestCase(integration_util.IntegrationTestCase, UsesShed):
 
     """Test data manager installation and table reload through the API"""
 
@@ -62,66 +36,30 @@ class DataManagerIntegrationTestCase(integration_util.IntegrationTestCase):
             import watchdog  # noqa: F401
         except ImportError:
             raise SkipTest("watchdog library is not available")
-        cls.username = cls.get_secure_ascii_digits()
-        cls.conda_tmp_prefix = tempfile.mkdtemp()
-        cls.shed_tools_dir = tempfile.mkdtemp()
-        cls.shed_tool_data_dir = tempfile.mkdtemp()
-        cls._test_driver.temp_directories.extend([cls.conda_tmp_prefix, cls.shed_tool_data_dir, cls.shed_tools_dir])
-        config["conda_auto_init"] = True
-        config["conda_auto_install"] = True
-        config["conda_prefix"] = os.path.join(cls.conda_tmp_prefix, 'conda')
-        config["tool_sheds_config_file"] = TOOL_SHEDS_CONF
-        config["tool_config_file"] = os.path.join(cls.shed_tools_dir, 'shed_tool_conf.xml')
-        config["shed_data_manager_config_file"] = os.path.join(cls.shed_tool_data_dir, 'shed_data_manager_config_file')
-        config["shed_tool_data_table_config"] = os.path.join(cls.shed_tool_data_dir, 'shed_data_table_conf.xml')
-        config["shed_tool_data_path"] = cls.shed_tool_data_dir
+        cls.configure_shed_and_conda(config)
         config["tool_data_path"] = cls.shed_tool_data_dir
         config["watch_tool_data_dir"] = True
+        cls.username = cls.get_secure_ascii_digits()
         config["admin_users"] = "%s@galaxy.org" % cls.username
-        with open(config["tool_config_file"], 'w') as tool_conf_file:
-            tool_conf_file.write(SHED_TOOL_CONF.substitute(shed_tools_path=cls.shed_tools_dir))
-        with open(config["shed_data_manager_config_file"], 'w') as shed_data_config:
-            shed_data_config.write(SHED_DATA_MANAGER_CONF)
-        with open(config["shed_tool_data_table_config"], 'w') as shed_data_table_config:
-            shed_data_table_config.write(SHED_DATA_TABLES)
 
     def test_data_manager_installation_table_reload(self):
         """
         Test that we can install data managers, create a new dbkey, and use that dbkey in a downstream data manager.
         """
-        create_response = self._post('/tool_shed_repositories/new/install_repository_revision', data=CREATE_DBKEY_PAYLOAD, admin=True)
-        self._assert_status_code_is(create_response, 200)
-        create_response = self._post('/tool_shed_repositories/new/install_repository_revision', data=SAM_FASTA_PAYLOAD, admin=True)
-        self._assert_status_code_is(create_response, 200)
-
+        self.install_repository("devteam", "data_manager_fetch_genome_dbkeys_all_fasta", "b1bc53e9bbc5")
+        self.install_repository("devteam", "data_manager_sam_fasta_index_builder", "1865e693d8b2")
         with self._different_user(email="%s@galaxy.org" % self.username):
             with self.dataset_populator.test_history() as history_id:
                 run_response = self.dataset_populator.run_tool(tool_id=FETCH_TOOL_ID,
                                                                inputs=FETCH_GENOME_DBKEYS_ALL_FASTA_INPUT,
                                                                history_id=history_id,
                                                                assert_ok=False)
-                self.dataset_populator.wait_for_tool_run(history_id=history_id, run_response=run_response, timeout=DATA_MANAGER_JOB_TIMEOUT)
+                self.dataset_populator.wait_for_tool_run(history_id=history_id, run_response=run_response, timeout=CONDA_AUTO_INSTALL_JOB_TIMEOUT)
                 run_response = self.dataset_populator.run_tool(tool_id=SAM_FASTA_ID,
                                                                inputs=SAM_FASTA_INPUT,
                                                                history_id=history_id,
                                                                assert_ok=False)
-                self.dataset_populator.wait_for_tool_run(history_id=history_id, run_response=run_response, timeout=DATA_MANAGER_JOB_TIMEOUT)
-
-    def create_local_user(self):
-        """Creates a local user and returns the user id."""
-        password = self.get_secure_ascii_digits()
-        payload = {'username': self.username,
-                   'password': password,
-                   'email': "%s@galaxy.org" % self.username}
-        create_response = self._post('/users', data=payload, admin=True)
-        self._assert_status_code_is(create_response, 200)
-        response = create_response.json()
-        return response['id']
-
-    def create_api_key_for_user(self, user_id):
-        create_response = self._post("/users/%s/api_key" % user_id, data={}, admin=True)
-        self._assert_status_code_is(create_response, 200)
-        return create_response.json()
+                self.dataset_populator.wait_for_tool_run(history_id=history_id, run_response=run_response, timeout=CONDA_AUTO_INSTALL_JOB_TIMEOUT)
 
     @classmethod
     def get_secure_ascii_digits(cls, n=12):

test/integration/test_shed_tool_tests.py

@@ -0,0 +1,18 @@
+from base import integration_util
+
+from .uses_shed import UsesShed
+
+
+class ToolShedToolTestIntegrationTestCase(integration_util.IntegrationTestCase, UsesShed):
+
+    """Test data manager installation and table reload through the API"""
+
+    framework_tool_and_types = True
+
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        cls.configure_shed_and_conda(config)
+
+    def test_tool_test(self):
+        self.install_repository("devteam", "fastqc", "ff9530579d1f")
+        self._run_tool_test("toolshed.g2.bx.psu.edu/repos/devteam/fastqc/fastqc/0.71")

test/integration/uses_shed.py

@@ -0,0 +1,59 @@
+import os
+import string
+import tempfile
+
+from base.driver_util import FRAMEWORK_UPLOAD_TOOL_CONF
+from base.populators import DEFAULT_TIMEOUT
+
+# Needs a longer timeout because of the conda_auto_install.
+CONDA_AUTO_INSTALL_JOB_TIMEOUT = DEFAULT_TIMEOUT * 3
+
+SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
+TOOL_SHEDS_CONF = os.path.join(SCRIPT_DIRECTORY, "tool_sheds_conf.xml")
+
+SHED_TOOL_CONF = string.Template("""<?xml version="1.0"?>
+<toolbox tool_path="$shed_tools_path">
+</toolbox>""")
+
+SHED_DATA_MANAGER_CONF = """<?xml version="1.0"?>
+<data_managers>
+</data_managers>"""
+
+SHED_DATA_TABLES = """<?xml version="1.0"?>
+<tables>
+</tables>"""
+
+
+class UsesShed(object):
+
+    @classmethod
+    def configure_shed_and_conda(cls, config):
+        cls.conda_tmp_prefix = tempfile.mkdtemp()
+        cls.shed_tools_dir = tempfile.mkdtemp()
+        cls.shed_tool_data_dir = tempfile.mkdtemp()
+        cls._test_driver.temp_directories.extend([cls.conda_tmp_prefix, cls.shed_tool_data_dir, cls.shed_tools_dir])
+        config["conda_auto_init"] = True
+        config["conda_auto_install"] = True
+        config["conda_prefix"] = os.path.join(cls.conda_tmp_prefix, 'conda')
+        config["tool_sheds_config_file"] = TOOL_SHEDS_CONF
+        shed_tool_config = os.path.join(cls.shed_tools_dir, 'shed_tool_conf.xml')
+        config["tool_config_file"] = "%s,%s" % (FRAMEWORK_UPLOAD_TOOL_CONF, shed_tool_config)
+        config["shed_data_manager_config_file"] = os.path.join(cls.shed_tool_data_dir, 'shed_data_manager_config_file')
+        config["shed_tool_data_table_config"] = os.path.join(cls.shed_tool_data_dir, 'shed_data_table_conf.xml')
+        config["shed_tool_data_path"] = cls.shed_tool_data_dir
+        with open(shed_tool_config, 'w') as tool_conf_file:
+            tool_conf_file.write(SHED_TOOL_CONF.substitute(shed_tools_path=cls.shed_tools_dir))
+        with open(config["shed_data_manager_config_file"], 'w') as shed_data_config:
+            shed_data_config.write(SHED_DATA_MANAGER_CONF)
+        with open(config["shed_tool_data_table_config"], 'w') as shed_data_table_config:
+            shed_data_table_config.write(SHED_DATA_TABLES)
+
+    def install_repository(self, owner, name, changeset):
+        payload = {
+            'tool_shed_url': 'https://toolshed.g2.bx.psu.edu',
+            'name': name,
+            'owner': owner,
+            'changeset_revision': changeset
+        }
+        create_response = self._post('/tool_shed_repositories/new/install_repository_revision', data=payload, admin=True)
+        self._assert_status_code_is(create_response, 200)

test/shed_functional/base/twilltestcase.py

@@ -15,13 +15,15 @@ from six.moves.urllib.parse import quote_plus
 import galaxy.model.tool_shed_install as galaxy_model
 import galaxy.util
 import galaxy.webapps.tool_shed.util.hgweb_config
-from base.tool_shed_util import repository_installation_timeout  # noqa: I100,I201
-from functional.twilltestcase import TwillTestCase  # noqa: I100
+from functional.twilltestcase import TwillTestCase  # noqa: I100,I201
 from galaxy.web import security  # noqa: I201
 from tool_shed.util import hg_util, xml_util
 from tool_shed.util.encoding_util import tool_shed_encode
 from . import common, test_db_util
 
+# Set a 10 minute timeout for repository installation.
+repository_installation_timeout = 600
+
 log = logging.getLogger(__name__)
 
 

test/unit/tools/test_parsing.py

@@ -201,17 +201,17 @@ class XmlLoaderTestCase(BaseLoaderTestCase):
         assert len(tests) == 2
         test_dict = tests[0]
         inputs = test_dict["inputs"]
-        assert len(inputs) == 1
+        assert len(inputs) == 1, test_dict
         input1 = inputs[0]
-        assert input1[0] == "foo"
-        assert input1[1] == "5"
+        assert input1["name"] == "foo", input1
+        assert input1["value"] == "5"
 
         outputs = test_dict["outputs"]
         assert len(outputs) == 1
         output1 = outputs[0]
-        assert output1[0] == 'out1'
-        assert output1[1] == 'moo.txt'
-        attributes1 = output1[2]
+        assert output1["name"] == 'out1'
+        assert output1["value"] == 'moo.txt'
+        attributes1 = output1["attributes"]
         assert attributes1["compare"] == "diff"
         assert attributes1["lines_diff"] == 0
 
@@ -219,9 +219,9 @@ class XmlLoaderTestCase(BaseLoaderTestCase):
         outputs = test2["outputs"]
         assert len(outputs) == 1
         output2 = outputs[0]
-        assert output2[0] == 'out1'
-        assert output2[1] is None
-        attributes1 = output2[2]
+        assert output2["name"] == 'out1'
+        assert output2["value"] is None
+        attributes1 = output2["attributes"]
         assert attributes1["compare"] == "sim_size"
         assert attributes1["lines_diff"] == 4
 
@@ -350,15 +350,15 @@ class YamlLoaderTestCase(BaseLoaderTestCase):
         inputs = test_dict["inputs"]
         assert len(inputs) == 1
         input1 = inputs[0]
-        assert input1[0] == "foo"
-        assert input1[1] == 5
+        assert input1["name"] == "foo"
+        assert input1["value"] == 5
 
         outputs = test_dict["outputs"]
         assert len(outputs) == 1
         output1 = outputs[0]
-        assert output1[0] == 'out1'
-        assert output1[1] == 'moo.txt'
-        attributes1 = output1[2]
+        assert output1["name"] == 'out1'
+        assert output1["value"] == 'moo.txt'
+        attributes1 = output1["attributes"]
         assert attributes1["compare"] == "diff"
         assert attributes1["lines_diff"] == 0
 
@@ -366,9 +366,9 @@ class YamlLoaderTestCase(BaseLoaderTestCase):
         outputs = test2["outputs"]
         assert len(outputs) == 1
         output2 = outputs[0]
-        assert output2[0] == 'out1'
-        assert output2[1] is None
-        attributes1 = output2[2]
+        assert output2["name"] == 'out1'
+        assert output2["value"] is None
+        attributes1 = output2["attributes"]
         assert attributes1["compare"] == "sim_size"
         assert attributes1["lines_diff"] == 4
 
@@ -448,3 +448,14 @@ class SpecialToolLoaderTestCase(BaseLoaderTestCase):
         action = self._tool_source.parse_action_module()
         assert action[0] == "galaxy.tools.actions.history_imp_exp"
         assert action[1] == "ExportHistoryToolAction"
+
+
+class CollectionTestCase(BaseLoaderTestCase):
+    source_file_name = os.path.join(os.getcwd(), "test/functional/tools/collection_two_paired.xml")
+    source_contents = None
+
+    def test_tests(self):
+        tests_dict = self._tool_source.parse_tests_to_dict()
+        tests = tests_dict["tests"]
+        assert len(tests) == 2
+        assert len(tests[0]["inputs"]) == 3, tests[0]

tools/genomespace/genomespace_exporter.xml

@@ -1,10 +1,10 @@
 <?xml version="1.0"?>
-<tool name="GenomeSpace Exporter" id="genomespace_exporter" version="0.0.5">
+<tool name="GenomeSpace Exporter" id="genomespace_exporter" require_login="True" version="0.0.6">
     <description> - send data to GenomeSpace</description>
     <environment_variables>
         <environment_variable name="GS_TOKEN">
-            #set $token = $genomespace_browser.split('^')[1] or $__user__.preferences.get('genomespace_token', None)
-            #assert $token, Exception('Invalid token. You must be logged into GenomeSpace through OpenID or select a valid folder via the GenomeSpace browse dialog.')
+            #set $token = $__user__.preferences.get('genomespace_token', None)
+            #assert $token, Exception('Invalid token. You must be logged into GenomeSpace through OpenID.')
             $token
         </environment_variable>
     </environment_variables>
@@ -31,11 +31,9 @@
         <data format="auto" name="output_log" />
     </outputs>
     <help>
-This Tool allows you to export data to GenomeSpace. Click the Browse button to select a file to export. The tool will automatically
-fetch your GenomeSpace token when you select a file from the export dialog.
+This Tool allows you to export data to GenomeSpace. Click the Browse button to select a file to export.
 
-Alternatively, you can be logged in using your GenomeSpace OpenID, and the tool will use this ID if no token is entered.
-This allows you to simply paste the filepath into the location box and leave the token empty. 
+To use this tool, you must have logged in using your GenomeSpace OpenID.
 You can associate your OpenID credentials under the User Preferences panel.
 Click here_ to refresh your GenomeSpace token.
 

tools/genomespace/genomespace_importer.xml

@@ -1,13 +1,18 @@
 <?xml version="1.0"?>
-<tool name="GenomeSpace Importer" id="genomespace_importer" require_login="False" version="0.0.5">
+<tool name="GenomeSpace Importer" id="genomespace_importer" require_login="True" version="0.0.6">
     <description> - receive data from GenomeSpace</description>
+    <environment_variables>
+        <environment_variable name="GS_TOKEN">
+            #set $token = $__user__.preferences.get('genomespace_token', None)
+            #assert $token, Exception('Invalid token. You must be logged into GenomeSpace through OpenID.')
+            $token
+        </environment_variable>
+    </environment_variables>
     <command><![CDATA[
         python $__tool_directory__/genomespace_importer.py
         #set $input_file = $URL.split('^')[0] if '^' in $URL else $URL
-        #set $token = $URL.split('^')[1] if '^' in $URL and $URL.split('^')[1] else $__user__.preferences.get('genomespace_token', None)
 
         #assert $input_file, Exception('You must select a valid input file.')
-        #assert $token, Exception('Invalid token. You must be logged into GenomeSpace through OpenID.')
         --json_parameter_file '${output_file1}'
         --galaxy_root $__root_dir__
         --data_conf $__datatypes_config__
@@ -21,11 +26,9 @@
         <data format="auto" name="output_file1" />
     </outputs>
     <help>
-This tool allows you to import data from GenomeSpace. Click the Browse button to select a file to import. The tool will automatically
-fetch your GenomeSpace token when you select a file from the import dialog.
+This tool allows you to import data from GenomeSpace. Click the Browse button to select a file to import.
 
-Alternatively, you can be logged in using your GenomeSpace OpenID, and the tool will use this ID if no token is entered.
-This allows you to simply paste the filepath into the location box and leave the token empty. 
+To use this tool, you must have logged in using your GenomeSpace OpenID. 
 You can associate your OpenID credentials under the User Preferences panel.
 Click here_ to refresh your GenomeSpace token.
   

tools/genomespace/genomespace_push.xml

@@ -1,10 +1,10 @@
 <?xml version="1.0"?>
-<tool name="GenomeSpace Push" id="genomespace_push"  tool_type="data_source" force_history_refresh="True" hidden="True" display_interface="False" require_login="True" version="0.0.1">
+<tool name="GenomeSpace Push" id="genomespace_push"  tool_type="data_source" force_history_refresh="True" hidden="True" display_interface="False" require_login="True" version="0.0.6">
     <description> - Push data from GenomeSpace to Galaxy</description>
     <environment_variables>
         <environment_variable name="GS_TOKEN">
-            #set $token = $__user__.preferences.get( 'genomespace_token', None )
-            #assert $token, Exception('Invalid token. You must be logged into GenomeSpace through OpenID or select a valid file via the GenomeSpace browse dialog.' )
+            #set $token = $__user__.preferences.get('genomespace_token', None)
+            #assert $token, Exception('Invalid token. You must be logged into GenomeSpace through OpenID.')
             $token
         </environment_variable>
     </environment_variables>