Merge pull request #1398 from jmchilton/toolbox_overhaul

ToolBox Overhaul

config/tool_conf.xml.sample

@@ -1,5 +1,5 @@
 <?xml version='1.0' encoding='utf-8'?>
-<toolbox>
+<toolbox monitor="true">
   <section id="getext" name="Get Data">
     <tool file="data_source/upload.xml" />
     <tool file="data_source/ucsc_tablebrowser.xml" />

lib/galaxy/config.py

@@ -13,6 +13,7 @@ import socket
 import string
 import sys
 import tempfile
+import threading
 from datetime import timedelta
 from galaxy.exceptions import ConfigurationError
 from galaxy.util import listify
@@ -744,18 +745,31 @@ class ConfiguresGalaxyMixin:
     def _configure_genome_builds( self, data_table_name="__dbkeys__", load_old_style=True ):
         self.genome_builds = GenomeBuilds( self, data_table_name=data_table_name, load_old_style=load_old_style )
 
-    def _configure_toolbox( self ):
+    def reload_toolbox(self):
         # Initialize the tools, making sure the list of tool configs includes the reserved migrated_tools_conf.xml file.
+
         tool_configs = self.config.tool_configs
         if self.config.migrated_tools_config not in tool_configs:
             tool_configs.append( self.config.migrated_tools_config )
 
+        from galaxy import tools
+        with self._toolbox_lock:
+            old_toolbox = self.toolbox
+            self.toolbox = tools.ToolBox( tool_configs, self.config.tool_path, self )
+            self.reindex_tool_search()
+            if old_toolbox:
+                old_toolbox.shutdown()
+
+    def _configure_toolbox( self ):
         from galaxy.managers.citations import CitationsManager
         self.citations_manager = CitationsManager( self )
 
-        from galaxy import tools
-        self.toolbox = tools.ToolBox( tool_configs, self.config.tool_path, self )
-        self.reindex_tool_search()
+        from galaxy.tools.toolbox.cache import ToolCache
+        self.tool_cache = ToolCache()
+
+        self._toolbox_lock = threading.Lock()
+        self.toolbox = None
+        self.reload_toolbox()
 
         from galaxy.tools.deps import containers
         galaxy_root_dir = os.path.abspath(self.config.root)

lib/galaxy/tools/__init__.py

@@ -7,6 +7,9 @@ import glob
 import json
 import logging
 import os
+import re
+import tarfile
+import tempfile
 import threading
 import urllib
 from datetime import datetime
@@ -1979,6 +1982,108 @@ class Tool( object, Dictifiable ):
         """
         return output_collect.collect_dynamic_collections( self, output, **kwds )
 
+    def to_archive(self):
+        tool = self.tool
+        tarball_files = []
+        temp_files = []
+        tool_xml = open( os.path.abspath( tool.config_file ), 'r' ).read()
+        # Retrieve tool help images and rewrite the tool's xml into a temporary file with the path
+        # modified to be relative to the repository root.
+        image_found = False
+        if tool.help is not None:
+            tool_help = tool.help._source
+            # Check each line of the rendered tool help for an image tag that points to a location under static/
+            for help_line in tool_help.split( '\n' ):
+                image_regex = re.compile( 'img alt="[^"]+" src="\${static_path}/([^"]+)"' )
+                matches = re.search( image_regex, help_line )
+                if matches is not None:
+                    tool_help_image = matches.group(1)
+                    tarball_path = tool_help_image
+                    filesystem_path = os.path.abspath( os.path.join( self.app.config.root, 'static', tool_help_image ) )
+                    if os.path.exists( filesystem_path ):
+                        tarball_files.append( ( filesystem_path, tarball_path ) )
+                        image_found = True
+                        tool_xml = tool_xml.replace( '${static_path}/%s' % tarball_path, tarball_path )
+        # If one or more tool help images were found, add the modified tool XML to the tarball instead of the original.
+        if image_found:
+            fd, new_tool_config = tempfile.mkstemp( suffix='.xml' )
+            os.close( fd )
+            open( new_tool_config, 'w' ).write( tool_xml )
+            tool_tup = ( os.path.abspath( new_tool_config ), os.path.split( tool.config_file )[-1]  )
+            temp_files.append( os.path.abspath( new_tool_config ) )
+        else:
+            tool_tup = ( os.path.abspath( tool.config_file ), os.path.split( tool.config_file )[-1]  )
+        tarball_files.append( tool_tup )
+        # TODO: This feels hacky.
+        tool_command = tool.command.strip().split()[0]
+        tool_path = os.path.dirname( os.path.abspath( tool.config_file ) )
+        # Add the tool XML to the tuple that will be used to populate the tarball.
+        if os.path.exists( os.path.join( tool_path, tool_command ) ):
+            tarball_files.append( ( os.path.join( tool_path, tool_command ), tool_command ) )
+        # Find and add macros and code files.
+        for external_file in tool.get_externally_referenced_paths( os.path.abspath( tool.config_file ) ):
+            external_file_abspath = os.path.abspath( os.path.join( tool_path, external_file ) )
+            tarball_files.append( ( external_file_abspath, external_file ) )
+        if os.path.exists( os.path.join( tool_path, "Dockerfile" ) ):
+            tarball_files.append( ( os.path.join( tool_path, "Dockerfile" ), "Dockerfile" ) )
+        # Find tests, and check them for test data.
+        tests = tool.tests
+        if tests is not None:
+            for test in tests:
+                # Add input file tuples to the list.
+                for input in test.inputs:
+                    for input_value in test.inputs[ input ]:
+                        input_path = os.path.abspath( os.path.join( 'test-data', input_value ) )
+                        if os.path.exists( input_path ):
+                            td_tup = ( input_path, os.path.join( 'test-data', input_value ) )
+                            tarball_files.append( td_tup )
+                # And add output file tuples to the list.
+                for label, filename, _ in test.outputs:
+                    output_filepath = os.path.abspath( os.path.join( 'test-data', filename ) )
+                    if os.path.exists( output_filepath ):
+                        td_tup = ( output_filepath, os.path.join( 'test-data', filename ) )
+                        tarball_files.append( td_tup )
+        for param in tool.input_params:
+            # Check for tool data table definitions.
+            if hasattr( param, 'options' ):
+                if hasattr( param.options, 'tool_data_table' ):
+                    data_table = param.options.tool_data_table
+                    if hasattr( data_table, 'filenames' ):
+                        data_table_definitions = []
+                        for data_table_filename in data_table.filenames:
+                            # FIXME: from_shed_config seems to always be False.
+                            if not data_table.filenames[ data_table_filename ][ 'from_shed_config' ]:
+                                tar_file = data_table.filenames[ data_table_filename ][ 'filename' ] + '.sample'
+                                sample_file = os.path.join( data_table.filenames[ data_table_filename ][ 'tool_data_path' ],
+                                                            tar_file )
+                                # Use the .sample file, if one exists. If not, skip this data table.
+                                if os.path.exists( sample_file ):
+                                    tarfile_path, tarfile_name = os.path.split( tar_file )
+                                    tarfile_path = os.path.join( 'tool-data', tarfile_name )
+                                    tarball_files.append( ( sample_file, tarfile_path ) )
+                                data_table_definitions.append( data_table.xml_string )
+                        if len( data_table_definitions ) > 0:
+                            # Put the data table definition XML in a temporary file.
+                            table_definition = '<?xml version="1.0" encoding="utf-8"?>\n<tables>\n    %s</tables>'
+                            table_definition = table_definition % '\n'.join( data_table_definitions )
+                            fd, table_conf = tempfile.mkstemp()
+                            os.close( fd )
+                            open( table_conf, 'w' ).write( table_definition )
+                            tarball_files.append( ( table_conf, os.path.join( 'tool-data', 'tool_data_table_conf.xml.sample' ) ) )
+                            temp_files.append( table_conf )
+        # Create the tarball.
+        fd, tarball_archive = tempfile.mkstemp( suffix='.tgz' )
+        os.close( fd )
+        tarball = tarfile.open( name=tarball_archive, mode='w:gz' )
+        # Add the files from the previously generated list.
+        for fspath, tarpath in tarball_files:
+            tarball.add( fspath, arcname=tarpath )
+        tarball.close()
+        # Delete any temporary files that were generated.
+        for temp_file in temp_files:
+            os.remove( temp_file )
+        return tarball_archive
+
     def to_dict( self, trans, link_details=False, io_details=False ):
         """ Returns dict of tool. """
 

lib/galaxy/tools/cwl/__init__.py

@@ -0,0 +1,18 @@
+from .parser import tool_proxy
+from .runtime_actions import handle_outputs
+from .representation import to_cwl_job, to_galaxy_parameters
+
+from .cwltool_deps import (
+    needs_shell_quoting,
+    shellescape,
+)
+
+
+__all__ = [
+    'tool_proxy',
+    'handle_outputs',
+    'to_cwl_job',
+    'to_galaxy_parameters',
+    'needs_shell_quoting',
+    'shellescape',
+]

lib/galaxy/tools/cwl/cwltool_deps.py

@@ -0,0 +1,48 @@
+""" This module contains logic for dealing with cwltool as an optional
+dependency for Galaxy and/or applications which use Galaxy as a library.
+"""
+
+try:
+    import requests
+except ImportError:
+    requests = None
+
+try:
+    from cwltool import (
+        main,
+        workflow,
+        job,
+    )
+except ImportError:
+    main = None
+    workflow = None
+    job = None
+
+try:
+    import shellescape
+except ImportError:
+    shellescape = None
+
+if job is not None:
+    needs_shell_quoting = job.needs_shell_quoting
+else:
+    needs_shell_quoting = None
+
+
+def ensure_cwltool_available():
+    if main is None or workflow is None or shellescape is None:
+        message = "This feature requires cwltool and dependencies to be available, they are not."
+        if requests is None:
+            message += " Library 'requests' unavailable."
+        if shellescape is None:
+            message += " Library 'shellescape' unavailable."
+        raise ImportError(message)
+
+
+__all__ = [
+    'main',
+    'workflow',
+    'ensure_cwltool_available',
+    'shellescape',
+    'needs_shell_quoting',
+]

lib/galaxy/tools/cwl/parser.py

@@ -0,0 +1,392 @@
+""" This module provides proxy objects around objects from the common
+workflow language reference implementation library cwltool. These proxies
+adapt cwltool to Galaxy features and abstract the library away from the rest
+of the framework.
+"""
+from __future__ import absolute_import
+from abc import ABCMeta, abstractmethod
+from galaxy.util.odict import odict
+import json
+import os
+
+from .cwltool_deps import (
+    main,
+    workflow,
+    ensure_cwltool_available,
+)
+
+from galaxy.util.bunch import Bunch
+
+JOB_JSON_FILE = ".cwl_job.json"
+
+
+def tool_proxy(tool_path):
+    """ Provide a proxy object to cwltool data structures to just
+    grab relevant data.
+    """
+    ensure_cwltool_available()
+    tool = to_cwl_tool_object(tool_path)
+    return tool
+
+
+def load_job_proxy(job_directory):
+    ensure_cwltool_available()
+    job_objects_path = os.path.join(job_directory, JOB_JSON_FILE)
+    job_objects = json.load(open(job_objects_path, "r"))
+    tool_path = job_objects["tool_path"]
+    job_inputs = job_objects["job_inputs"]
+    cwl_tool = tool_proxy(tool_path)
+    cwl_job = cwl_tool.job_proxy(job_inputs, job_directory=job_directory)
+    return cwl_job
+
+
+def to_cwl_tool_object(tool_path):
+    if workflow is None:
+        raise Exception("Using CWL tools requires cwltool module.")
+    proxy_class = None
+    cwl_tool = None
+    make_tool = workflow.defaultMakeTool
+    cwl_tool = main.load_tool(tool_path, False, False, make_tool, False)
+    proxy_class = Draft2ToolProxy
+    if proxy_class is None:
+        raise Exception("Unsupported CWL object encountered.")
+    proxy = proxy_class(cwl_tool, tool_path)
+    return proxy
+
+
+class ToolProxy( object ):
+    __metaclass__ = ABCMeta
+
+    def __init__(self, tool, tool_path):
+        self._tool = tool
+        self._tool_path = tool_path
+
+    def job_proxy(self, input_dict, job_directory="."):
+        """ Build a cwltool.job.Job describing computation using a input_json
+        Galaxy will generate mapping the Galaxy description of the inputs into
+        a cwltool compatible variant.
+        """
+        return JobProxy(self, input_dict, job_directory=job_directory)
+
+    @abstractmethod
+    def input_instances(self):
+        """ Return InputInstance objects describing mapping to Galaxy inputs. """
+
+    @abstractmethod
+    def output_instances(self):
+        """ Return OutputInstance objects describing mapping to Galaxy inputs. """
+
+    @abstractmethod
+    def docker_identifier(self):
+        """ Return docker identifier for embedding in tool description. """
+
+    @abstractmethod
+    def description(self):
+        """ Return description to tool. """
+
+
+class Draft2ToolProxy(ToolProxy):
+
+    def description(self):
+        # Feels like I should be getting some abstract namespaced thing
+        # not actually description, is this correct?
+        return self._tool.tool.get('description')
+
+    def input_instances(self):
+        return self._find_inputs(self._tool.inputs_record_schema)
+
+    def _find_inputs(self, schema):
+        schema_type = schema["type"]
+        if isinstance(schema_type, list):
+            raise Exception("Union types not yet implemented.")
+        elif isinstance(schema_type, dict):
+            return self._find_inputs(schema_type)
+        else:
+            if schema_type in self._tool.schemaDefs:
+                schema = self._tool.schemaDefs[schema_type]
+
+            if schema["type"] == "record":
+                return map(_simple_field_to_input, schema["fields"])
+
+    def output_instances(self):
+        outputs_schema = self._tool.outputs_record_schema
+        return self._find_outputs(outputs_schema)
+
+    def _find_outputs(self, schema):
+        rval = []
+        if not rval and schema["type"] == "record":
+            for output in schema["fields"]:
+                # TODO: Handle non-files differently? Make them
+                # JSON maybe?.
+                # output_type = output.get("type", None)
+                # if output_type != "File":
+                #     template = "Unhandled output type [%s] encountered."
+                #     raise Exception(template % output_type)
+                rval.append(_simple_field_to_output(output))
+
+        return rval
+
+    def docker_identifier(self):
+        tool = self._tool.tool
+        reqs_and_hints = tool.get("requirements", []) + tool.get("hints", [])
+        for hint in reqs_and_hints:
+            if hint["class"] == "DockerRequirement":
+                if "dockerImageId" in hint:
+                    return hint["dockerImageId"]
+                else:
+                    return hint["dockerPull"]
+        return None
+
+
+class JobProxy(object):
+
+    def __init__(self, tool_proxy, input_dict, job_directory):
+        self._tool_proxy = tool_proxy
+        self._input_dict = input_dict
+        self._job_directory = job_directory
+
+    def cwl_job(self):
+        return self._tool_proxy._tool.job(
+            self._input_dict,
+            self._job_directory,
+            self._output_callback,
+            use_container=False
+        ).next()
+
+    def _output_callback(self, out):
+        pass
+
+    def save_job(self):
+        job_file = JobProxy._job_file(self._job_directory)
+        job_objects = {
+            "tool_path": os.path.abspath(self._tool_proxy._tool_path),
+            "job_inputs": self._input_dict,
+        }
+        json.dump(job_objects, open(job_file, "w"))
+
+    # @staticmethod
+    # def load_job(tool_proxy, job_directory):
+    #     job_file = JobProxy._job_file(job_directory)
+    #     input_dict = json.load(open(job_file, "r"))
+    #     return JobProxy(tool_proxy, input_dict, job_directory)
+
+    @staticmethod
+    def _job_file(job_directory):
+        return os.path.join(job_directory, JOB_JSON_FILE)
+
+
+def _simple_field_union(field):
+    field_type = _field_to_field_type(field)
+    non_null_type = None
+    if len(field_type) == 2:
+        if "null" == field_type[0]:
+            non_null_type = field_type[1]
+        elif "null" == field_type[1]:
+            non_null_type = field_type[1]
+
+    if non_null_type is None:
+        raise Exception("General union types not yet implemented (simple).")
+
+    name, label, description = _field_metadata(field)
+
+    case_name = "_cwl__type_"
+    case_label = "Specify %s" % label
+
+    options = [
+        {"value": "null", "label": "No", "selected": True},
+        {"value": "%s" % non_null_type, "label": "Yes", "selected": False},
+    ]
+
+    case_input = SelectInputInstance(
+        name=case_name,
+        label=case_label,
+        description=False,
+        options=options,
+    )
+
+    non_null_type_kwds = _simple_field_to_input_type_kwds(field, field_type=non_null_type)
+    non_null_name = "_cwl__value_"
+    non_null_label = label
+    non_null_description = description
+    non_null_input = InputInstance(
+        non_null_name,
+        non_null_label,
+        non_null_description,
+        **non_null_type_kwds
+    )
+    options = [
+        ("null", []),
+        (non_null_type, [non_null_input]),
+    ]
+    return ConditionalInstance(name, case_input, options)
+
+
+def _simple_field_to_input(field):
+    field_type = _field_to_field_type(field)
+    if isinstance(field_type, list):
+        # Length must be greater than 1...
+        return _simple_field_union(field)
+
+    name, label, description = _field_metadata(field)
+
+    type_kwds = _simple_field_to_input_type_kwds(field)
+    return InputInstance(name, label, description, **type_kwds)
+
+
+def _simple_field_to_input_type_kwds(field, field_type=None):
+    simple_map_type_map = {
+        "File": INPUT_TYPE.DATA,
+        "int": INPUT_TYPE.INTEGER,
+        "string": INPUT_TYPE.TEXT,
+        "boolean": INPUT_TYPE.BOOLEAN,
+    }
+
+    if field_type is None:
+        field_type = _field_to_field_type(field)
+
+    if field_type in simple_map_type_map.keys():
+        input_type = simple_map_type_map[field_type]
+        return {"input_type": input_type, "array": False}
+    elif field_type == "array":
+        if isinstance(field["type"], dict):
+            array_type = field["type"]["items"]
+        else:
+            array_type = field["items"]
+        if array_type in simple_map_type_map.keys():
+            input_type = simple_map_type_map[array_type]
+        return {"input_type": input_type, "array": True}
+    else:
+        raise Exception("Unhandled simple field type encountered - [%s]." % field_type)
+
+
+def _field_to_field_type(field):
+    field_type = field["type"]
+    if isinstance(field_type, dict):
+        field_type = field_type["type"]
+    if isinstance(field_type, list):
+        field_type_length = len(field_type)
+        if field_type_length == 0:
+            raise Exception("Zero-length type list encountered, invalid CWL?")
+        elif len(field_type) == 1:
+            field_type = field_type[0]
+
+    return field_type
+
+
+def _field_metadata(field):
+    name = field["name"]
+    label = field.get("label", None)
+    description = field.get("description", None)
+    return name, label, description
+
+
+def _simple_field_to_output(field):
+    name = field["name"]
+    output_instance = OutputInstance(name, output_type=OUTPUT_TYPE.GLOB)
+    return output_instance
+
+
+INPUT_TYPE = Bunch(
+    DATA="data",
+    INTEGER="integer",
+    TEXT="text",
+    BOOLEAN="boolean",
+    SELECT="select",
+    CONDITIONAL="conditional",
+)
+
+
+class ConditionalInstance(object):
+
+    def __init__(self, name, case, whens):
+        self.input_type = INPUT_TYPE.CONDITIONAL
+        self.name = name
+        self.case = case
+        self.whens = whens
+
+    def to_dict(self):
+
+        as_dict = dict(
+            name=self.name,
+            type=INPUT_TYPE.CONDITIONAL,
+            test=self.case.to_dict(),
+            when=odict(),
+        )
+        for value, block in self.whens:
+            as_dict["when"][value] = map(lambda i: i.to_dict(), block)
+
+        return as_dict
+
+
+class SelectInputInstance(object):
+
+    def __init__(self, name, label, description, options):
+        self.input_type = INPUT_TYPE.SELECT
+        self.name = name
+        self.label = label
+        self.description = description
+        self.options = options
+
+    def to_dict(self):
+        # TODO: serialize options...
+        as_dict = dict(
+            name=self.name,
+            label=self.label or self.name,
+            help=self.description,
+            type=self.input_type,
+            options=self.options,
+        )
+        return as_dict
+
+
+class InputInstance(object):
+
+    def __init__(self, name, label, description, input_type, array=False):
+        self.input_type = input_type
+        self.name = name
+        self.label = label
+        self.description = description
+        self.required = True
+        self.array = array
+
+    def to_dict(self, itemwise=True):
+        if itemwise and self.array:
+            as_dict = dict(
+                type="repeat",
+                name="%s_repeat" % self.name,
+                title="%s" % self.name,
+                blocks=[
+                    self.to_dict(itemwise=False)
+                ]
+            )
+        else:
+            as_dict = dict(
+                name=self.name,
+                label=self.label or self.name,
+                help=self.description,
+                type=self.input_type,
+                optional=not self.required,
+            )
+            if self.input_type == INPUT_TYPE.INTEGER:
+                as_dict["value"] = "0"
+        return as_dict
+
+
+OUTPUT_TYPE = Bunch(
+    GLOB="glob",
+    STDOUT="stdout",
+)
+
+
+class OutputInstance(object):
+
+    def __init__(self, name, output_type, path=None):
+        self.name = name
+        self.output_type = output_type
+        self.path = path
+
+
+__all__ = [
+    'tool_proxy',
+    'load_job_proxy',
+]

lib/galaxy/tools/cwl/representation.py

@@ -0,0 +1,100 @@
+""" This module is responsible for converting between Galaxy's tool
+input description and the CWL description for a job json. """
+
+import logging
+
+from galaxy.util import string_as_bool
+
+log = logging.getLogger(__name__)
+
+NOT_PRESENT = object()
+
+
+def to_cwl_job(tool, param_dict):
+    """ tool is Galaxy's representation of the tool and param_dict is the
+    parameter dictionary with wrapped values.
+    """
+    inputs = tool.inputs
+    input_json = {}
+
+    def simple_value(input, param_dict_value):
+        if input.type == "data":
+            return {"path": str(param_dict_value), "class": "File"}
+        elif input.type == "integer":
+            return int(str(param_dict_value))
+        elif input.type == "boolean":
+            return string_as_bool(param_dict_value)
+        else:
+            return str(param_dict_value)
+
+    for input_name, input in inputs.iteritems():
+        if input.type == "repeat":
+            only_input = input.inputs.values()[0]
+            array_value = []
+            for instance in param_dict[input_name]:
+                array_value.append(simple_value(only_input, instance[input_name[:-len("_repeat")]]))
+            input_json[input_name[:-len("_repeat")]] = array_value
+        elif input.type == "conditional":
+            assert input_name in param_dict, "No value for %s in %s" % (input_name, param_dict)
+            current_case = param_dict[input_name]["_cwl__type_"]
+            if current_case != "null":
+                # TODO: make sure trans is not needed here...
+                case_index = input.get_current_case( current_case, None )
+                case_input = input.cases[ case_index ].inputs[ "_cwl__value_"]
+                case_value = param_dict[input_name]["_cwl__value_"]
+                input_json[input_name] = simple_value(case_input, case_value)
+        else:
+            input_json[input_name] = simple_value(input, param_dict[input_name])
+
+    input_json["allocatedResources"] = {
+        "cpu": "$GALAXY_SLOTS",
+    }
+    return input_json
+
+
+def to_galaxy_parameters(tool, as_dict):
+    """ Tool is Galaxy's representation of the tool and as_dict is a Galaxified
+    representation of the input json (no paths, HDA references for instance).
+    """
+    inputs = tool.inputs
+    galaxy_request = {}
+
+    def from_simple_value(input, param_dict_value):
+        return param_dict_value
+
+    for input_name, input in inputs.iteritems():
+        as_dict_value = as_dict.get(input_name, NOT_PRESENT)
+
+        if input.type == "repeat":
+            if input_name not in as_dict:
+                continue
+
+            only_input = input.inputs.values()[0]
+            for index, value in enumerate(as_dict_value):
+                key = "%s_repeat_0|%s" % (input_name, only_input.name)
+                galaxy_value = from_simple_value(only_input, value)
+                galaxy_request[key] = galaxy_value
+        elif input.type == "conditional":
+            # TODO: less crazy handling of defaults...
+            if as_dict_value is NOT_PRESENT:
+                cwl_type = "null"
+            elif isinstance(as_dict_value, bool):
+                cwl_type = "boolean"
+            elif isinstance(as_dict_value, int):
+                cwl_type = "integer"
+            else:
+                cwl_type = "string"
+            galaxy_request["%s|_cwl__type_" % input_name] = cwl_type
+            if cwl_type != "null":
+                current_case_index = input.get_current_case(cwl_type, None)
+                current_case_input = input.cases[ current_case_index ].inputs[ "_cwl__value_" ]
+                galaxy_value = from_simple_value(current_case_input, as_dict_value)
+                galaxy_request["%s|_cwl__value_" % input_name] = galaxy_value
+        elif as_dict_value is NOT_PRESENT:
+            continue
+        else:
+            galaxy_value = from_simple_value(input, as_dict_value)
+            galaxy_request[input_name] = galaxy_value
+
+    log.info("Converted galaxy_request is %s" % galaxy_request)
+    return galaxy_request

lib/galaxy/tools/cwl/runtime_actions.py

@@ -0,0 +1,29 @@
+import os
+import shutil
+
+from .parser import (
+    JOB_JSON_FILE,
+    load_job_proxy,
+)
+
+
+def handle_outputs(job_directory=None):
+    # Relocate dynamically collected files to pre-determined locations
+    # registered with ToolOutput objects via from_work_dir handling.
+    if job_directory is None:
+        job_directory = os.getcwd()
+    cwl_job_file = os.path.join(job_directory, JOB_JSON_FILE)
+    if not os.path.exists(cwl_job_file):
+        # Not a CWL job, just continue
+        return
+    job_proxy = load_job_proxy(job_directory)
+    cwl_job = job_proxy.cwl_job()
+    outputs = cwl_job.collect_outputs(job_directory)
+    for output_name, output in outputs.iteritems():
+        target_path = os.path.join(job_directory, "__cwl_output_%s" % output_name)
+        shutil.move(output["path"], target_path)
+
+
+__all__ = [
+    'handle_outputs',
+]

lib/galaxy/tools/loader_directory.py

@@ -19,11 +19,36 @@ LOAD_FAILURE_ERROR = "Failed to load tool with path %s."
 TOOL_LOAD_ERROR = object()
 TOOL_REGEX = re.compile(r"<tool\s")
 
+YAML_EXTENSIONS = [".yaml", ".yml", ".json"]
+CWL_EXTENSIONS = YAML_EXTENSIONS + [".cwl"]
+
 
 def load_exception_handler(path, exc_info):
     log.warn(LOAD_FAILURE_ERROR % path, exc_info=exc_info)
 
 
+def find_possible_tools_from_path(
+    path,
+    recursive=False,
+    enable_beta_formats=False,
+):
+    possible_tool_files = []
+    for possible_tool_file in __find_tool_files(
+        path, recursive=recursive,
+        enable_beta_formats=enable_beta_formats
+    ):
+        try:
+            does_look_like_a_tool = looks_like_a_tool(possible_tool_file)
+        except IOError:
+            # Some problem reading the tool file, skip.
+            continue
+
+        if does_look_like_a_tool:
+            possible_tool_files.append(possible_tool_file)
+
+    return possible_tool_files
+
+
 def load_tool_elements_from_path(
     path,
     load_exception_handler=load_exception_handler,
@@ -31,21 +56,18 @@ def load_tool_elements_from_path(
     register_load_errors=False,
 ):
     tool_elements = []
-    for file in __find_tool_files(path, recursive=recursive):
+    for possible_tool_file in find_possible_tools_from_path(
+        path,
+        recursive=recursive,
+        enable_beta_formats=False,
+    ):
         try:
-            does_look_like_a_tool = looks_like_a_tool(file)
-        except IOError:
-            # Some problem reading the tool file, skip.
-            continue
-
-        if does_look_like_a_tool:
-            try:
-                tool_elements.append((file, loader.load_tool(file)))
-            except Exception:
-                exc_info = sys.exc_info()
-                load_exception_handler(file, exc_info)
-                if register_load_errors:
-                    tool_elements.append((file, TOOL_LOAD_ERROR))
+            tool_elements.append((file, loader.load_tool(file)))
+        except Exception:
+            exc_info = sys.exc_info()
+            load_exception_handler(file, exc_info)
+            if register_load_errors:
+                tool_elements.append((file, TOOL_LOAD_ERROR))
     return tool_elements
 
 
@@ -104,7 +126,7 @@ def looks_like_a_tool_xml(path):
 
 
 def looks_like_a_tool_yaml(path):
-    if not path.endswith(".yml") and not path.endswith(".json"):
+    if not _has_extension(path, YAML_EXTENSIONS):
         return False
 
     with open(path, "r") as f:
@@ -120,7 +142,25 @@ def looks_like_a_tool_yaml(path):
     return file_class == "GalaxyTool"
 
 
-def __find_tool_files(path, recursive):
+def looks_like_a_tool_cwl(path):
+    if _has_extension(path, CWL_EXTENSIONS):
+        return False
+
+    with open(path, "r") as f:
+        try:
+            as_dict = yaml.safe_load(f)
+        except Exception:
+            return False
+
+    if not isinstance(as_dict, dict):
+        return False
+
+    file_class = as_dict.get("class", None)
+    file_cwl_version = as_dict.get("cwlVersion", None)
+    return file_class == "CommandLineTool" and file_cwl_version
+
+
+def __find_tool_files(path, recursive, enable_beta_formats):
     is_file = not os.path.isdir(path)
     if not os.path.exists(path):
         raise Exception(PATH_DOES_NOT_EXIST_ERROR)
@@ -129,13 +169,23 @@ def __find_tool_files(path, recursive):
     elif is_file:
         return [os.path.abspath(path)]
     else:
-        if not recursive:
-            files = glob.glob(path + "/*.xml")
+        if enable_beta_formats:
+            if not recursive:
+                files = glob.glob(path + "/*")
+            else:
+                files = _find_files(path, "*")
         else:
-            files = _find_files(path, "*.xml")
+            if not recursive:
+                files = glob.glob(path + "/*.xml")
+            else:
+                files = _find_files(path, "*.xml")
         return map(os.path.abspath, files)
 
 
+def _has_extension(path, extensions):
+    return any(map(lambda e: path.endswith(e), extensions))
+
+
 def _find_files(directory, pattern='*'):
     if not os.path.exists(directory):
         raise ValueError("Directory not found {}".format(directory))
@@ -151,4 +201,5 @@ def _find_files(directory, pattern='*'):
 
 BETA_TOOL_CHECKERS = {
     'yaml': looks_like_a_tool_yaml,
+    'cwl': looks_like_a_tool_cwl,
 }

lib/galaxy/tools/parser/cwl.py

@@ -0,0 +1,145 @@
+import logging
+import os
+
+from .interface import ToolSource
+from .interface import PagesSource
+from .interface import PageSource
+from .interface import ToolStdioExitCode
+from .yaml import YamlInputSource
+
+from .output_actions import ToolOutputActionGroup
+from .output_objects import ToolOutput
+
+from galaxy.tools.deps import requirements
+from galaxy.tools.cwl import tool_proxy
+
+from galaxy.util.odict import odict
+
+log = logging.getLogger(__name__)
+
+
+class CwlToolSource(ToolSource):
+
+    def __init__(self, tool_file):
+        self._cwl_tool_file = tool_file
+        self._id, _ = os.path.splitext(os.path.basename(tool_file))
+        self._tool_proxy = tool_proxy(tool_file)
+
+    @property
+    def tool_proxy(self):
+        return self._tool_proxy
+
+    def parse_tool_type(self):
+        return 'cwl'
+
+    def parse_id(self):
+        log.warn("TOOL ID is %s" % self._id)
+        return self._id
+
+    def parse_name(self):
+        return self._id
+
+    def parse_command(self):
+        return "$__cwl_command"
+
+    def parse_environment_variables(self):
+        environment_variables = []
+        # TODO: Is this even possible from here, should instead this be moved
+        # into the job.
+
+        # for environment_variable_el in environment_variables_el.findall("environment_variable"):
+        #    definition = {
+        #        "name": environment_variable_el.get("name"),
+        #        "template": environment_variable_el.text,
+        #    }
+        #    environment_variables.append(
+        #        definition
+        #    )
+
+        return environment_variables
+
+    def parse_help(self):
+        return ""
+
+    def parse_stdio(self):
+        # TODO: remove duplication with YAML
+        from galaxy.jobs.error_level import StdioErrorLevel
+
+        # New format - starting out just using exit code.
+        exit_code_lower = ToolStdioExitCode()
+        exit_code_lower.range_start = float("-inf")
+        exit_code_lower.range_end = -1
+        exit_code_lower.error_level = StdioErrorLevel.FATAL
+        exit_code_high = ToolStdioExitCode()
+        exit_code_high.range_start = 1
+        exit_code_high.range_end = float("inf")
+        exit_code_lower.error_level = StdioErrorLevel.FATAL
+        return [exit_code_lower, exit_code_high], []
+
+    def parse_interpreter(self):
+        return None
+
+    def parse_version(self):
+        return "0.0.1"
+
+    def parse_description(self):
+        return self._tool_proxy.description() or ""
+
+    def parse_input_pages(self):
+        page_source = CwlPageSource(self._tool_proxy)
+        return PagesSource([page_source])
+
+    def parse_outputs(self, tool):
+        output_instances = self._tool_proxy.output_instances()
+        outputs = odict()
+        output_defs = []
+        for output_instance in output_instances:
+            output_defs.append(self._parse_output(tool, output_instance))
+        # TODO: parse outputs collections
+        for output_def in output_defs:
+            outputs[output_def.name] = output_def
+        return outputs, odict()
+
+    def _parse_output(self, tool, output_instance):
+        name = output_instance.name
+        # TODO: handle filters, actions, change_format
+        output = ToolOutput( name )
+        output.format = "_sniff_"
+        output.change_format = []
+        output.format_source = None
+        output.metadata_source = ""
+        output.parent = None
+        output.label = None
+        output.count = None
+        output.filters = []
+        output.tool = tool
+        output.from_work_dir = "__cwl_output_%s" % name
+        output.hidden = ""
+        output.dataset_collectors = []
+        output.actions = ToolOutputActionGroup( output, None )
+        return output
+
+    def parse_requirements_and_containers(self):
+        containers = []
+        docker_identifier = self._tool_proxy.docker_identifier()
+        if docker_identifier:
+            containers.append({"type": "docker",
+                               "identifier": docker_identifier})
+        return requirements.parse_requirements_from_dict(dict(
+            requirements=[],  # TODO: enable via extensions
+            containers=containers,
+        ))
+
+
+class CwlPageSource(PageSource):
+
+    def __init__(self, tool_proxy):
+        cwl_instances = tool_proxy.input_instances()
+        self._input_list = map(self._to_input_source, cwl_instances)
+
+    def _to_input_source(self, input_instance):
+        as_dict = input_instance.to_dict()
+        return YamlInputSource(as_dict)
+
+    def parse_input_sources(self):
+        return self._input_list

lib/galaxy/tools/parser/factory.py

@@ -5,6 +5,7 @@ import yaml
 from .yaml import YamlToolSource
 from .xml import XmlToolSource
 from .xml import XmlInputSource
+from .cwl import CwlToolSource
 from .interface import InputSource
 
 
@@ -27,6 +28,9 @@ def get_tool_source(config_file, enable_beta_formats=True):
         with open(config_file, "r") as f:
             as_dict = ordered_load(f)
             return YamlToolSource(as_dict)
+    elif config_file.endswith(".json") or config_file.endswith(".cwl"):
+        log.info("Loading CWL tool - this is experimental - tool likely will not function in future at least in same way.")
+        return CwlToolSource(config_file)
     else:
         tree = load_tool_xml(config_file)
         root = tree.getroot()

lib/galaxy/tools/toolbox/base.py

@@ -1,14 +1,13 @@
 import logging
 import os
-import re
 import string
-import tarfile
-import tempfile
 
 from markupsafe import escape
 from six.moves.urllib.parse import urlparse
 from six import iteritems
 
+from galaxy.exceptions import ObjectNotFound
+
 from galaxy.util.dictifiable import Dictifiable
 
 from galaxy.util.odict import odict
@@ -17,6 +16,7 @@ from galaxy.util import parse_xml
 from galaxy.util import string_as_bool
 from galaxy.util.bunch import Bunch
 
+from .parser import get_toolbox_parser, ensure_tool_conf_item
 from .panel import ToolPanelElements
 from .panel import ToolSectionLabel
 from .panel import ToolSection
@@ -27,7 +27,8 @@ from .lineages import LineageMap
 from .tags import tool_tag_manager
 
 from .filters import FilterFactory
-from .watcher import get_watcher
+from .watcher import get_tool_watcher
+from .watcher import get_tool_conf_watcher
 
 # Extra tool dependency not used by AbstractToolBox but by
 # BaseGalaxyToolBox
@@ -71,7 +72,8 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
         # (e.g., shed_tool_conf.xml) files include the tool_path attribute within the <toolbox> tag.
         self._tool_root_dir = tool_root_dir
         self.app = app
-        self._tool_watcher = get_watcher( self, app.config )
+        self._tool_watcher = get_tool_watcher( self, app.config )
+        self._tool_conf_watcher = get_tool_conf_watcher( lambda: app.reload_toolbox() )
         self._filter_factory = FilterFactory( self )
         self._tool_tag_manager = tool_tag_manager( app )
         self._init_tools_from_configs( config_filenames )
@@ -121,9 +123,8 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
 
         """
         log.info( "Parsing the tool configuration %s" % config_filename )
-        tree = parse_xml( config_filename )
-        root = tree.getroot()
-        tool_path = root.get( 'tool_path' )
+        tool_conf_source = get_toolbox_parser(config_filename)
+        tool_path = tool_conf_source.parse_tool_path()
         if tool_path:
             # We're parsing a shed_tool_conf file since we have a tool_path attribute.
             parsing_shed_tool_conf = True
@@ -134,16 +135,16 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
         tool_path = self.__resolve_tool_path(tool_path, config_filename)
         # Only load the panel_dict under certain conditions.
         load_panel_dict = not self._integrated_tool_panel_config_has_contents
-        for _, elem in enumerate( root ):
+        for item in tool_conf_source.parse_items():
             index = self._index
             self._index += 1
             if parsing_shed_tool_conf:
-                config_elems.append( elem )
+                config_elems.append( item.elem )
             self.load_item(
-                elem,
+                item,
                 tool_path=tool_path,
                 load_panel_dict=load_panel_dict,
-                guid=elem.get( 'guid' ),
+                guid=item.get( 'guid' ),
                 index=index,
                 internal=True
             )
@@ -154,8 +155,12 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
                                         config_elems=config_elems )
             self._dynamic_tool_confs.append( shed_tool_conf_dict )
 
-    def load_item( self, elem, tool_path, panel_dict=None, integrated_panel_dict=None, load_panel_dict=True, guid=None, index=None, internal=False ):
-        item_type = elem.tag
+        if tool_conf_source.parse_monitor():
+            self._tool_conf_watcher.watch_file(config_filename)
+
+    def load_item( self, item, tool_path, panel_dict=None, integrated_panel_dict=None, load_panel_dict=True, guid=None, index=None, internal=False ):
+        item = ensure_tool_conf_item(item)
+        item_type = item.type
         if item_type not in ['tool', 'section'] and not internal:
             # External calls from tool shed code cannot load labels or tool
             # directories.
@@ -166,15 +171,15 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
         if integrated_panel_dict is None:
             integrated_panel_dict = self._integrated_tool_panel
         if item_type == 'tool':
-            self._load_tool_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, tool_path=tool_path, load_panel_dict=load_panel_dict, guid=guid, index=index )
+            self._load_tool_tag_set( item, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, tool_path=tool_path, load_panel_dict=load_panel_dict, guid=guid, index=index )
         elif item_type == 'workflow':
-            self._load_workflow_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index )
+            self._load_workflow_tag_set( item, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index )
         elif item_type == 'section':
-            self._load_section_tag_set( elem, tool_path=tool_path, load_panel_dict=load_panel_dict, index=index )
+            self._load_section_tag_set( item, tool_path=tool_path, load_panel_dict=load_panel_dict, index=index )
         elif item_type == 'label':
-            self._load_label_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index )
+            self._load_label_tag_set( item, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index )
         elif item_type == 'tool_dir':
-            self._load_tooldir_tag_set( elem, panel_dict, tool_path, integrated_panel_dict, load_panel_dict=load_panel_dict )
+            self._load_tooldir_tag_set( item, panel_dict, tool_path, integrated_panel_dict, load_panel_dict=load_panel_dict )
 
     def get_shed_config_dict_by_filename( self, filename, default=None ):
         for shed_config_dict in self._dynamic_tool_confs:
@@ -533,9 +538,9 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
         else:
             remove_from_dict( self._tool_panel, self._integrated_tool_panel )
 
-    def _load_tool_tag_set( self, elem, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None, index=None ):
+    def _load_tool_tag_set( self, item, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None, index=None ):
         try:
-            path = elem.get( "file" )
+            path = item.get( "file" )
             repository_id = None
 
             tool_shed_repository = None
@@ -543,13 +548,13 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
             if guid is not None:
                 # The tool is contained in an installed tool shed repository, so load
                 # the tool only if the repository has not been marked deleted.
-                tool_shed = elem.find( "tool_shed" ).text
-                repository_name = elem.find( "repository_name" ).text
-                repository_owner = elem.find( "repository_owner" ).text
-                installed_changeset_revision_elem = elem.find( "installed_changeset_revision" )
+                tool_shed = item.elem.find( "tool_shed" ).text
+                repository_name = item.elem.find( "repository_name" ).text
+                repository_owner = item.elem.find( "repository_owner" ).text
+                installed_changeset_revision_elem = item.elem.find( "installed_changeset_revision" )
                 if installed_changeset_revision_elem is None:
                     # Backward compatibility issue - the tag used to be named 'changeset_revision'.
-                    installed_changeset_revision_elem = elem.find( "changeset_revision" )
+                    installed_changeset_revision_elem = item.elem.find( "changeset_revision" )
                 installed_changeset_revision = installed_changeset_revision_elem.text
                 tool_shed_repository = self._get_tool_shed_repository( tool_shed,
                                                                        repository_name,
@@ -563,7 +568,7 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
                 # Else there is not yet a tool_shed_repository record, we're in the process of installing
                 # a new repository, so any included tools can be loaded into the tool panel.
             tool = self.load_tool( os.path.join( tool_path, path ), guid=guid, repository_id=repository_id )
-            if string_as_bool(elem.get( 'hidden', False )):
+            if string_as_bool(item.get( 'hidden', False )):
                 tool.hidden = True
             key = 'tool_%s' % str( tool.id )
             if can_load_into_panel_dict:
@@ -573,19 +578,21 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
                     tool.repository_owner = repository_owner
                     tool.installed_changeset_revision = installed_changeset_revision
                     tool.guid = guid
-                    tool.version = elem.find( "version" ).text
+                    tool.version = item.elem.find( "version" ).text
                 # Make sure the tool has a tool_version.
                 tool_lineage = self._lineage_map.register( tool, tool_shed_repository=tool_shed_repository )
                 # Load the tool's lineage ids.
                 tool.lineage = tool_lineage
-                self._tool_tag_manager.handle_tags( tool.id, elem )
+                if item.has_elem:
+                    self._tool_tag_manager.handle_tags( tool.id, item.elem )
                 self.__add_tool( tool, load_panel_dict, panel_dict )
             # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file.
             integrated_panel_dict.update_or_append( index, key, tool )
             # If labels were specified in the toolbox config, attach them to
             # the tool.
-            if "labels" in elem.attrib:
-                tool.labels = [ label.strip() for label in elem.attrib["labels"].split( "," ) ]
+            labels = item.labels
+            if labels is not None:
+                tool.labels = labels
         except IOError:
             log.error( "Error reading tool configuration file from path: %s." % path )
         except Exception:
@@ -609,10 +616,10 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
         if load_panel_dict:
             self.__add_tool_to_tool_panel( tool, panel_dict, section=isinstance( panel_dict, ToolSection ) )
 
-    def _load_workflow_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ):
+    def _load_workflow_tag_set( self, item, panel_dict, integrated_panel_dict, load_panel_dict, index=None ):
         try:
             # TODO: should id be encoded?
-            workflow_id = elem.get( 'id' )
+            workflow_id = item.get( 'id' )
             workflow = self._load_workflow( workflow_id )
             self._workflows_by_id[ workflow_id ] = workflow
             key = 'workflow_' + workflow_id
@@ -623,35 +630,35 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
         except:
             log.exception( "Error loading workflow: %s" % workflow_id )
 
-    def _load_label_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ):
-        label = ToolSectionLabel( elem )
+    def _load_label_tag_set( self, item, panel_dict, integrated_panel_dict, load_panel_dict, index=None ):
+        label = ToolSectionLabel( item )
         key = 'label_' + label.id
         if load_panel_dict:
             panel_dict[ key ] = label
         integrated_panel_dict.update_or_append( index, key, label )
 
-    def _load_section_tag_set( self, elem, tool_path, load_panel_dict, index=None ):
-        key = elem.get( "id" )
+    def _load_section_tag_set( self, item, tool_path, load_panel_dict, index=None ):
+        key = item.get( "id" )
         if key in self._tool_panel:
             section = self._tool_panel[ key ]
             elems = section.elems
         else:
-            section = ToolSection( elem )
+            section = ToolSection( item )
             elems = section.elems
         if key in self._integrated_tool_panel:
             integrated_section = self._integrated_tool_panel[ key ]
             integrated_elems = integrated_section.elems
         else:
-            integrated_section = ToolSection( elem )
+            integrated_section = ToolSection( item )
             integrated_elems = integrated_section.elems
-        for sub_index, sub_elem in enumerate( elem ):
+        for sub_index, sub_item in enumerate( item.items ):
             self.load_item(
-                sub_elem,
+                sub_item,
                 tool_path=tool_path,
                 panel_dict=elems,
                 integrated_panel_dict=integrated_elems,
                 load_panel_dict=load_panel_dict,
-                guid=sub_elem.get( 'guid' ),
+                guid=sub_item.get( 'guid' ),
                 index=sub_index,
                 internal=True,
             )
@@ -668,9 +675,9 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
         # Always load sections into the integrated_tool_panel.
         self._integrated_tool_panel.update_or_append( index, key, integrated_section )
 
-    def _load_tooldir_tag_set(self, sub_elem, elems, tool_path, integrated_elems, load_panel_dict):
-        directory = os.path.join( tool_path, sub_elem.attrib.get("dir") )
-        recursive = string_as_bool( sub_elem.attrib.get("recursive", True) )
+    def _load_tooldir_tag_set(self, item, elems, tool_path, integrated_elems, load_panel_dict):
+        directory = os.path.join( tool_path, item.get("dir") )
+        recursive = string_as_bool( item.get("recursive", True) )
         self.__watch_directory( directory, elems, integrated_elems, load_panel_dict, recursive, force_watch=True )
 
     def __watch_directory( self, directory, elems, integrated_elems, load_panel_dict, recursive, force_watch=False ):
@@ -702,10 +709,15 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
         if tool_loaded or force_watch:
             self._tool_watcher.watch_directory( directory, quick_load )
 
-    def load_tool( self, config_file, guid=None, repository_id=None, **kwds ):
+    def load_tool( self, config_file, guid=None, repository_id=None, use_cached=True, **kwds ):
         """Load a single tool from the file named by `config_file` and return an instance of `Tool`."""
         # Parse XML configuration file and get the root element
-        tool = self.create_tool( config_file=config_file, repository_id=repository_id, guid=guid, **kwds )
+        tool_cache = getattr( self.app, 'tool_cache', None )
+        tool = use_cached and tool_cache and tool_cache.get_tool( config_file )
+        if not tool:
+            tool = self.create_tool( config_file=config_file, repository_id=repository_id, guid=guid, **kwds )
+            if tool_cache:
+                self.app.tool_cache.cache_tool(config_file, tool)
         tool_id = tool.id
         if not tool_id.startswith("__"):
             # do not monitor special tools written to tmp directory - no reason
@@ -750,109 +762,10 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
         """
         # Make sure the tool is actually loaded.
         if tool_id not in self._tools_by_id:
-            return None, False, "No tool with id %s" % escape( tool_id )
+            raise ObjectNotFound("No tool found with id '%s'." % escape( tool_id ))
         else:
             tool = self._tools_by_id[ tool_id ]
-            tarball_files = []
-            temp_files = []
-            tool_xml = open( os.path.abspath( tool.config_file ), 'r' ).read()
-            # Retrieve tool help images and rewrite the tool's xml into a temporary file with the path
-            # modified to be relative to the repository root.
-            image_found = False
-            if tool.help is not None:
-                tool_help = tool.help._source
-                # Check each line of the rendered tool help for an image tag that points to a location under static/
-                for help_line in tool_help.split( '\n' ):
-                    image_regex = re.compile( 'img alt="[^"]+" src="\${static_path}/([^"]+)"' )
-                    matches = re.search( image_regex, help_line )
-                    if matches is not None:
-                        tool_help_image = matches.group(1)
-                        tarball_path = tool_help_image
-                        filesystem_path = os.path.abspath( os.path.join( trans.app.config.root, 'static', tool_help_image ) )
-                        if os.path.exists( filesystem_path ):
-                            tarball_files.append( ( filesystem_path, tarball_path ) )
-                            image_found = True
-                            tool_xml = tool_xml.replace( '${static_path}/%s' % tarball_path, tarball_path )
-            # If one or more tool help images were found, add the modified tool XML to the tarball instead of the original.
-            if image_found:
-                fd, new_tool_config = tempfile.mkstemp( suffix='.xml' )
-                os.close( fd )
-                open( new_tool_config, 'w' ).write( tool_xml )
-                tool_tup = ( os.path.abspath( new_tool_config ), os.path.split( tool.config_file )[-1]  )
-                temp_files.append( os.path.abspath( new_tool_config ) )
-            else:
-                tool_tup = ( os.path.abspath( tool.config_file ), os.path.split( tool.config_file )[-1]  )
-            tarball_files.append( tool_tup )
-            # TODO: This feels hacky.
-            tool_command = tool.command.strip().split()[0]
-            tool_path = os.path.dirname( os.path.abspath( tool.config_file ) )
-            # Add the tool XML to the tuple that will be used to populate the tarball.
-            if os.path.exists( os.path.join( tool_path, tool_command ) ):
-                tarball_files.append( ( os.path.join( tool_path, tool_command ), tool_command ) )
-            # Find and add macros and code files.
-            for external_file in tool.get_externally_referenced_paths( os.path.abspath( tool.config_file ) ):
-                external_file_abspath = os.path.abspath( os.path.join( tool_path, external_file ) )
-                tarball_files.append( ( external_file_abspath, external_file ) )
-            if os.path.exists( os.path.join( tool_path, "Dockerfile" ) ):
-                tarball_files.append( ( os.path.join( tool_path, "Dockerfile" ), "Dockerfile" ) )
-            # Find tests, and check them for test data.
-            tests = tool.tests
-            if tests is not None:
-                for test in tests:
-                    # Add input file tuples to the list.
-                    for input in test.inputs:
-                        for input_value in test.inputs[ input ]:
-                            input_path = os.path.abspath( os.path.join( 'test-data', input_value ) )
-                            if os.path.exists( input_path ):
-                                td_tup = ( input_path, os.path.join( 'test-data', input_value ) )
-                                tarball_files.append( td_tup )
-                    # And add output file tuples to the list.
-                    for label, filename, _ in test.outputs:
-                        output_filepath = os.path.abspath( os.path.join( 'test-data', filename ) )
-                        if os.path.exists( output_filepath ):
-                            td_tup = ( output_filepath, os.path.join( 'test-data', filename ) )
-                            tarball_files.append( td_tup )
-            for param in tool.input_params:
-                # Check for tool data table definitions.
-                if hasattr( param, 'options' ):
-                    if hasattr( param.options, 'tool_data_table' ):
-                        data_table = param.options.tool_data_table
-                        if hasattr( data_table, 'filenames' ):
-                            data_table_definitions = []
-                            for data_table_filename in data_table.filenames:
-                                # FIXME: from_shed_config seems to always be False.
-                                if not data_table.filenames[ data_table_filename ][ 'from_shed_config' ]:
-                                    tar_file = data_table.filenames[ data_table_filename ][ 'filename' ] + '.sample'
-                                    sample_file = os.path.join( data_table.filenames[ data_table_filename ][ 'tool_data_path' ],
-                                                                tar_file )
-                                    # Use the .sample file, if one exists. If not, skip this data table.
-                                    if os.path.exists( sample_file ):
-                                        tarfile_path, tarfile_name = os.path.split( tar_file )
-                                        tarfile_path = os.path.join( 'tool-data', tarfile_name )
-                                        tarball_files.append( ( sample_file, tarfile_path ) )
-                                    data_table_definitions.append( data_table.xml_string )
-                            if len( data_table_definitions ) > 0:
-                                # Put the data table definition XML in a temporary file.
-                                table_definition = '<?xml version="1.0" encoding="utf-8"?>\n<tables>\n    %s</tables>'
-                                table_definition = table_definition % '\n'.join( data_table_definitions )
-                                fd, table_conf = tempfile.mkstemp()
-                                os.close( fd )
-                                open( table_conf, 'w' ).write( table_definition )
-                                tarball_files.append( ( table_conf, os.path.join( 'tool-data', 'tool_data_table_conf.xml.sample' ) ) )
-                                temp_files.append( table_conf )
-            # Create the tarball.
-            fd, tarball_archive = tempfile.mkstemp( suffix='.tgz' )
-            os.close( fd )
-            tarball = tarfile.open( name=tarball_archive, mode='w:gz' )
-            # Add the files from the previously generated list.
-            for fspath, tarpath in tarball_files:
-                tarball.add( fspath, arcname=tarpath )
-            tarball.close()
-            # Delete any temporary files that were generated.
-            for temp_file in temp_files:
-                os.remove( temp_file )
-            return tarball_archive, True, None
-        return None, False, "An unknown error occurred."
+            return tool.to_archive()
 
     def reload_tool_by_id( self, tool_id ):
         """
@@ -908,6 +821,9 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
         else:
             tool = self._tools_by_id[ tool_id ]
             del self._tools_by_id[ tool_id ]
+            tool_cache = getattr( self.app, 'tool_cache', None )
+            if tool_cache:
+                tool_cache.expire_tool( tool_id )
             if remove_from_panel:
                 tool_key = 'tool_' + tool_id
                 for key, val in self._tool_panel.items():
@@ -995,6 +911,21 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
 
         return rval
 
+    def shutdown(self):
+        exception = None
+        try:
+            self._tool_watcher.shutdown()
+        except Exception as e:
+            exception = e
+
+        try:
+            self._tool_conf_watcher.shutdown()
+        except Exception as e:
+            exception = exception or e
+
+        if exception:
+            raise exception
+
     def _lineage_in_panel( self, panel_dict, tool=None, tool_lineage=None ):
         """ If tool with same lineage already in panel (or section) - find
         and return it. Otherwise return None.

lib/galaxy/tools/toolbox/cache.py

@@ -0,0 +1,26 @@
+
+
+class ToolCache(object):
+    """ Cache tool defintions to allow quickly reloading the whole
+    toolbox.
+    """
+
+    def __init__(self):
+        self._tools_by_path = {}
+        self._tool_paths_by_id = {}
+
+    def get_tool(self, config_filename):
+        """ Get the tool from the cache if the tool is up to date.
+        """
+        return self._tools_by_path.get(config_filename, None)
+
+    def expire_tool(self, tool_id):
+        if tool_id in self._tool_paths_by_id:
+            config_filename = self._tool_paths_by_id[tool_id]
+            del self._tool_paths_by_id[tool_id]
+            del self._tools_by_path[config_filename]
+
+    def cache_tool(self, config_filename, tool):
+        tool_id = str( tool.id )
+        self._tool_paths_by_id[tool_id] = config_filename
+        self._tools_by_path[config_filename] = tool

lib/galaxy/tools/toolbox/panel.py

@@ -3,6 +3,7 @@ from abc import abstractmethod
 from galaxy.util.odict import odict
 from galaxy.util import bunch
 from galaxy.util.dictifiable import Dictifiable
+from .parser import ensure_tool_conf_item
 
 from six import iteritems
 
@@ -49,13 +50,13 @@ class ToolSection( Dictifiable, HasPanelItems, object ):
 
     dict_collection_visible_keys = ( 'id', 'name', 'version' )
 
-    def __init__( self, elem=None ):
+    def __init__( self, item=None ):
         """ Build a ToolSection from an ElementTree element or a dictionary.
         """
-        f = lambda elem, val: elem is not None and elem.get( val ) or ''
-        self.name = f( elem, 'name' )
-        self.id = f( elem, 'id' )
-        self.version = f( elem, 'version' )
+        f = lambda item, val: item is not None and item.get( val ) or ''
+        self.name = f( item, 'name' )
+        self.id = f( item, 'id' )
+        self.version = f( item, 'version' )
         self.elems = ToolPanelElements()
 
     def copy( self ):
@@ -93,13 +94,14 @@ class ToolSectionLabel( Dictifiable, object ):
 
     dict_collection_visible_keys = ( 'id', 'text', 'version' )
 
-    def __init__( self, elem ):
+    def __init__( self, item ):
         """ Build a ToolSectionLabel from an ElementTree element or a
         dictionary.
         """
-        self.text = elem.get( "text" )
-        self.id = elem.get( "id" )
-        self.version = elem.get( "version" ) or ''
+        item = ensure_tool_conf_item(item)
+        self.text = item.get( "text" )
+        self.id = item.get( "id" )
+        self.version = item.get( "version" ) or ''
 
     def to_dict( self, **kwds ):
         return super( ToolSectionLabel, self ).to_dict()

lib/galaxy/tools/toolbox/parser.py

@@ -0,0 +1,138 @@
+from abc import ABCMeta
+from abc import abstractmethod
+
+from galaxy.util import parse_xml, string_as_bool
+import yaml
+
+DEFAULT_MONITOR = False
+
+
+class ToolConfSource(object):
+    """ This interface represents an abstract source to parse tool
+    information from.
+    """
+    __metaclass__ = ABCMeta
+
+    @abstractmethod
+    def parse_items(self):
+        """ Return a list of ToolConfItem
+        """
+
+    @abstractmethod
+    def parse_tool_path(self):
+        """ Return tool_path for tools in this toolbox.
+        """
+
+    def parse_monitor(self):
+        """ Monitor the toolbox configuration source for changes and
+        reload. """
+        return DEFAULT_MONITOR
+
+
+class XmlToolConfSource(ToolConfSource):
+
+    def __init__(self, config_filename):
+        tree = parse_xml(config_filename)
+        self.root = tree.getroot()
+
+    def parse_tool_path(self):
+        return self.root.get('tool_path')
+
+    def parse_items(self):
+        return map(ensure_tool_conf_item, self.root.getchildren())
+
+    def parse_monitor(self):
+        return string_as_bool(self.root.get('monitor', DEFAULT_MONITOR))
+
+
+class YamlToolConfSource(ToolConfSource):
+
+    def __init__(self, config_filename):
+        with open(config_filename, "r") as f:
+            as_dict = yaml.load(f)
+        self.as_dict = as_dict
+
+    def parse_tool_path(self):
+        return self.as_dict.get('tool_path')
+
+    def parse_items(self):
+        return map(ToolConfItem.from_dict, self.as_dict.get('items'))
+
+    def parse_monitor(self):
+        return self.as_dict.get('monitor', DEFAULT_MONITOR)
+
+
+class ToolConfItem(object):
+    """ This interface represents an abstract source to parse tool
+    information from.
+    """
+
+    def __init__(self, type, attributes, elem=None):
+        self.type = type
+        self.attributes = attributes
+        self._elem = elem
+
+    @classmethod
+    def from_dict(cls, _as_dict):
+        as_dict = _as_dict.copy()
+        type = as_dict.get('type')
+        del as_dict['type']
+        attributes = as_dict
+        if type == 'section':
+            items = map(cls.from_dict, as_dict['items'])
+            del as_dict['items']
+            item = ToolConfSection(attributes, items)
+        else:
+            item = ToolConfItem(type, attributes)
+        return item
+
+    def get(self, key, default=None):
+        return self.attributes.get(key, default)
+
+    @property
+    def has_elem(self):
+        return self._elem is not None
+
+    @property
+    def elem(self):
+        if self._elem is None:
+            raise Exception("item.elem called on toolbox element from non-XML source")
+        return self._elem
+
+    @property
+    def labels(self):
+        labels = None
+        if "labels" in self.attributes:
+            labels = [ label.strip() for label in self.attributes["labels"].split( "," ) ]
+        return labels
+
+
+class ToolConfSection(ToolConfItem):
+
+    def __init__(self, attributes, items, elem=None):
+        super(ToolConfSection, self).__init__('section', attributes, elem)
+        self.items = items
+
+
+def ensure_tool_conf_item(xml_or_item):
+    if xml_or_item is None:
+        return None
+    elif isinstance(xml_or_item, ToolConfItem):
+        return xml_or_item
+    else:
+        elem = xml_or_item
+        type = elem.tag
+        attributes = elem.attrib
+        if type != "section":
+            return ToolConfItem(type, attributes, elem)
+        else:
+            items = map(ensure_tool_conf_item, elem.getchildren())
+            return ToolConfSection(attributes, items, elem=elem)
+
+
+def get_toolbox_parser(config_filename):
+    is_yaml = any(map(lambda e: config_filename.endswith(e), [".yml", ".yaml", ".json"]))
+    if is_yaml:
+        return YamlToolConfSource(config_filename)
+    else:
+        return XmlToolConfSource(config_filename)

lib/galaxy/tools/toolbox/watcher.py

@@ -1,4 +1,8 @@
+import logging
 import os.path
+import threading
+import time
+
 try:
     from watchdog.events import FileSystemEventHandler
     from watchdog.observers import Observer
@@ -9,34 +13,131 @@ except ImportError:
     PollingObserver = object
     can_watch = False
 
-import logging
 log = logging.getLogger( __name__ )
 
 
-def get_watcher(toolbox, config):
-    watch_tools_val = str( getattr(config, "watch_tools", False) ).lower()
-    if watch_tools_val in ( 'true', 'yes', 'on' ):
-        return ToolWatcher(toolbox)
-    elif watch_tools_val == "auto":
-        try:
-            return ToolWatcher(toolbox)
-        except Exception:
-            log.info("Failed to load ToolWatcher (watchdog is likely unavailable) - proceeding without tool monitoring.")
-            return NullWatcher()
-    elif watch_tools_val == "polling":
-        log.info("Using less ineffecient polling toolbox watcher.")
-        return ToolWatcher(toolbox, observer_class=PollingObserver)
+def get_observer_class(config_value, default, monitor_what_str):
+    """
+    """
+    config_value = config_value or default
+    config_value = str(config_value).lower()
+    if config_value in ("true", "yes", "on", "auto"):
+        expect_observer = config_value != "auto"
+        observer_class = Observer
+    elif config_value == "polling":
+        expect_observer = True
+        observer_class = PollingObserver
+    else:
+        expect_observer = False
+        observer_class = None
+
+    if observer_class is None:
+        message = "Watchdog library unavailble, cannot monitor %s." % monitor_what_str
+        log.info(message)
+        if expect_observer:
+            raise Exception(message)
+
+    return observer_class
+
+
+def get_tool_conf_watcher(reload_callback):
+    return ToolConfWatcher(reload_callback)
+
+
+def get_tool_watcher(toolbox, config):
+    config_value = getattr(config, "watch_tools", None)
+    observer_class = get_observer_class(config_value, default="False", monitor_what_str="tools")
+
+    if observer_class is not None:
+        return ToolWatcher(toolbox, observer_class=observer_class)
     else:
         return NullWatcher()
 
 
+class ToolConfWatcher(object):
+
+    def __init__(self, reload_callback):
+        self.paths = {}
+        self._active = False
+        self._lock = threading.Lock()
+        self.thread = threading.Thread(target=self.check)
+        self.thread.daemon = True
+        self.event_handler = ToolConfFileEventHandler(reload_callback)
+
+    def start(self):
+        if not self._active:
+            self._active = True
+            self.thread.start()
+
+    def shutdown(self):
+        if self._active:
+            self._active = False
+            self.thread.join()
+
+    def check(self):
+        while self._active:
+            do_reload = False
+            with self._lock:
+                paths = list(self.paths.keys())
+            for path in paths:
+                if not os.path.exists(path):
+                    continue
+                mod_time = self.paths[path]
+                new_mod_time = None
+                if os.path.exists(path):
+                    new_mod_time = time.ctime(os.path.getmtime(path))
+                if new_mod_time != mod_time:
+                    self.paths[path] = new_mod_time
+                    do_reload = True
+
+            if do_reload:
+                t = threading.Thread(target=lambda: self.event_handler.on_any_event(None))
+                t.daemon = True
+                t.start()
+            time.sleep(1)
+
+    def monitor(self, path):
+        mod_time = None
+        if os.path.exists(path):
+            mod_time = time.ctime(os.path.getmtime(path))
+        with self._lock:
+            self.paths[path] = mod_time
+        self.start()
+
+    def watch_file(self, tool_conf_file):
+        self.monitor(tool_conf_file)
+
+
+class NullToolConfWatcher(object):
+
+    def start(self):
+        pass
+
+    def shutdown(self):
+        pass
+
+    def monitor(self, conf_path):
+        pass
+
+    def watch_file(self, tool_file, tool_id):
+        pass
+
+
+class ToolConfFileEventHandler(FileSystemEventHandler):
+
+    def __init__(self, reload_callback):
+        self.reload_callback = reload_callback
+
+    def on_any_event(self, event):
+        self._handle(event)
+
+    def _handle(self, event):
+        self.reload_callback()
+
+
 class ToolWatcher(object):
 
-    def __init__(self, toolbox, observer_class=None):
-        if not can_watch:
-            raise Exception("Watchdog library unavailble, cannot watch tools.")
-        if observer_class is None:
-            observer_class = Observer
+    def __init__(self, toolbox, observer_class):
         self.toolbox = toolbox
         self.tool_file_ids = {}
         self.tool_dir_callbacks = {}
@@ -48,6 +149,10 @@ class ToolWatcher(object):
     def start(self):
         self.observer.start()
 
+    def shutdown(self):
+        self.observer.stop()
+        self.observer.join()
+
     def monitor(self, dir):
         self.observer.schedule(self.event_handler, dir, recursive=False)
 

lib/galaxy/web/base/controllers/admin.py

@@ -68,26 +68,24 @@ class Admin( object ):
     def package_tool( self, trans, **kwd ):
         params = util.Params( kwd )
         message = util.restore_text( params.get( 'message', ''  ) )
-        status = params.get( 'status', 'done' )
         toolbox = self.app.toolbox
         tool_id = None
         if params.get( 'package_tool_button', False ):
             tool_id = params.get('tool_id', None)
-            tool_tarball, success, message = trans.app.toolbox.package_tool( trans, tool_id )
-            if success:
+            try:
+                tool_tarball = trans.app.toolbox.package_tool( trans, tool_id )
                 trans.response.set_content_type( 'application/x-gzip' )
                 download_file = open( tool_tarball )
                 os.unlink( tool_tarball )
                 tarball_path, filename = os.path.split( tool_tarball )
                 trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.tgz"' % ( tool_id )
                 return download_file
-            else:
-                status = 'error'
-        return trans.fill_template( '/admin/package_tool.mako',
-                                    tool_id=tool_id,
-                                    toolbox=toolbox,
-                                    message=message,
-                                    status=status )
+            except Exception:
+                return trans.fill_template( '/admin/package_tool.mako',
+                                            tool_id=tool_id,
+                                            toolbox=toolbox,
+                                            message=message,
+                                            status='error' )
 
     @web.expose
     @web.require_admin

lib/galaxy/webapps/galaxy/api/configuration.py

@@ -82,6 +82,15 @@ class ConfigurationController( BaseAPIController ):
             rval.append(entry)
         return rval
 
+    @expose_api
+    @require_admin
+    def reload_toolbox(self, trans):
+        """
+        PUT /api/configuration/toolbox
+        Reload the Galaxy toolbox (but not individual tools).
+        """
+        self.app.reload_toolbox()
+
 
 def _tool_conf_to_dict(conf):
     return dict(

lib/galaxy/webapps/galaxy/api/tools.py

@@ -198,12 +198,11 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ):
     @web.expose_api_raw
     @web.require_admin
     def download( self, trans, id, **kwds ):
-        tool_tarball, success, message = trans.app.toolbox.package_tool( trans, id )
-        if success:
-            trans.response.set_content_type( 'application/x-gzip' )
-            download_file = open( tool_tarball )
-            trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.tgz"' % ( id )
-            return download_file
+        tool_tarball = trans.app.toolbox.package_tool(trans, id)
+        trans.response.set_content_type('application/x-gzip')
+        download_file = open(tool_tarball, "rb")
+        trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.tgz"' % (id)
+        return download_file
 
     @expose_api_anonymous
     def create( self, trans, payload, **kwd ):

lib/galaxy/webapps/galaxy/buildapp.py

@@ -277,6 +277,12 @@ def populate_api_routes( webapp, app ):
         controller="configuration",
         action="tool_lineages"
     )
+    webapp.mapper.connect(
+        '/api/configuration/toolbox',
+        controller="configuration",
+        action="reload_toolbox",
+        conditions=dict( method=["PUT"] )
+    )
     webapp.mapper.resource( 'configuration', 'configuration', path_prefix='/api' )
     webapp.mapper.connect( "configuration_version",
                            "/api/version", controller="configuration",

lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py

@@ -74,7 +74,7 @@ class InstalledRepositoryMetadataManager( metadata_generator.MetadataGenerator )
                     load_relative_path = os.path.join( shed_conf_dict.get( 'tool_path' ), relative_path )
                 guid = tool_dict.get( 'guid', None )
                 if relative_path and guid:
-                    tool = self.app.toolbox.load_tool( os.path.abspath( load_relative_path ), guid=guid )
+                    tool = self.app.toolbox.load_tool( os.path.abspath( load_relative_path ), guid=guid, use_cached=False )
                 else:
                     tool = None
                 if tool:

lib/tool_shed/galaxy_install/tool_migration_manager.py

@@ -11,6 +11,7 @@ import logging
 
 from galaxy import util
 from galaxy.tools.toolbox import ToolSection
+from galaxy.tools.toolbox.parser import ensure_tool_conf_item
 from galaxy.util.odict import odict
 
 from tool_shed.galaxy_install import install_manager
@@ -272,7 +273,7 @@ class ToolMigrationManager( object ):
                         proprietary_tool_config = section_elem.get( 'file' )
                         if tool_config == proprietary_tool_config:
                             # The tool is loaded inside of the section_elem.
-                            tool_sections.append( ToolSection( proprietary_tool_panel_elem ) )
+                            tool_sections.append( ToolSection( ensure_tool_conf_item( proprietary_tool_panel_elem ) ) )
                             if not is_displayed:
                                 is_displayed = True
         return is_displayed, tool_sections
@@ -289,7 +290,7 @@ class ToolMigrationManager( object ):
                     filename = basic_util.strip_path( name )
                     if filename == tool_config_filename:
                         full_path = str( os.path.abspath( os.path.join( root, name ) ) )
-                        tool = self.toolbox.load_tool( full_path )
+                        tool = self.toolbox.load_tool( full_path, use_cached=False )
                         return suc.generate_tool_guid( repository_clone_url, tool )
         # Not quite sure what should happen here, throw an exception or what?
         return None

lib/tool_shed/tools/data_table_manager.py

@@ -83,7 +83,7 @@ class ToolDataTableManager( object ):
                     # TODO: Do more here than logging an exception.
                     log.debug( message )
             # Reload the tool into the local list of repository_tools_tups.
-            repository_tool = self.app.toolbox.load_tool( os.path.join( tool_path, tup_path ), guid=guid )
+            repository_tool = self.app.toolbox.load_tool( os.path.join( tool_path, tup_path ), guid=guid, use_cached=False )
             repository_tools_tups[ index ] = ( tup_path, guid, repository_tool )
             # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
             self.reset_tool_data_tables()

lib/tool_shed/tools/tool_validator.py

@@ -292,7 +292,7 @@ class ToolValidator( object ):
 
     def load_tool_from_config( self, repository_id, full_path ):
         try:
-            tool = self.app.toolbox.load_tool( full_path, repository_id=repository_id, allow_code_files=False )
+            tool = self.app.toolbox.load_tool( full_path, repository_id=repository_id, allow_code_files=False, use_cached=False )
             valid = True
             error_message = None
         except KeyError, e:

lib/tool_shed/util/tool_util.py

@@ -179,7 +179,7 @@ def handle_missing_index_file( app, tool_path, sample_files, repository_tools_tu
                         sample_files_copied.append( options.missing_index_file )
                         break
         # Reload the tool into the local list of repository_tools_tups.
-        repository_tool = app.toolbox.load_tool( os.path.join( tool_path, tup_path ), guid=guid )
+        repository_tool = app.toolbox.load_tool( os.path.join( tool_path, tup_path ), guid=guid, use_cached=False )
         repository_tools_tups[ index ] = ( tup_path, guid, repository_tool )
     return repository_tools_tups, sample_files_copied
 

templates/admin/reload_tool.mako

@@ -14,6 +14,14 @@ $().ready(function() {
 %endif
     focus_el.focus();
 });
+$().ready(function() {
+    $("#reload_toolbox").click(function(){
+        $.ajax({
+            url: "${h.url_for(controller="/api/configuration", action="toolbox")}",
+            type: 'PUT'
+        });
+    });
+});
 </script>
 
 %if message:
@@ -54,3 +62,18 @@ $().ready(function() {
     </form>
     </div>
 </div>
+<p>
+<div class="toolForm">
+    <div class="toolFormTitle">Reload Toolbox</div>
+    <div class="toolFormBody">
+    <form name="reload_toolbox_form" id="reload_toolbox_form" action="" method="" >
+        <div class="form-row">
+        Clicking <a href="#" id="reload_toolbox">here</a> will reload
+        the Galaxy toolbox. This will cause newly discovered tools
+        to be added, tools now missing from tool confs to be removed,
+        and items in the panel reordered. Individual tools, even ones
+        with modified tool descriptions willl not be reloaded.
+        </div>
+    </form>
+    </div>
+</div>

test/unit/tools/test_toolbox.py

@@ -1,3 +1,4 @@
+import json
 import os
 import string
 import unittest
@@ -117,10 +118,14 @@ class BaseToolBoxTestCase(  unittest.TestCase, tools_support.UsesApp, tools_supp
 </toolbox>"""
         self._add_config( template % (self.test_directory, CONFIG_TEST_TOOL_VERSION_1, CONFIG_TEST_TOOL_VERSION_2 ) )
 
-    def _add_config( self, xml, name="tool_conf.xml" ):
+    def _add_config( self, content, name="tool_conf.xml" ):
+        is_json = name.endswith(".json")
         path = self._tool_conf_path( name=name )
         with open( path, "w" ) as f:
-            f.write( xml )
+            if not is_json or isinstance(content, basestring):
+                f.write( content )
+            else:
+                json.dump(content, f)
         self.config_files.append( path )
 
     def _tool_conf_path( self, name="tool_conf.xml" ):
@@ -146,20 +151,22 @@ class ToolBoxTestCase( BaseToolBoxTestCase ):
         assert toolbox.get_tool( "not_a_test_tool" ) is None
 
     def test_to_dict_in_panel( self ):
-        self._init_tool_in_section()
-        mapper = routes.Mapper()
-        mapper.connect( "tool_runner", "/test/tool_runner" )
-        as_dict = self.toolbox.to_dict( mock_trans() )
-        test_section = self._find_section(as_dict, "t")
-        assert len(test_section["elems"]) == 1
-        assert test_section["elems"][0]["id"] == "test_tool"
+        for json_conf in [True, False]:
+            self._init_tool_in_section(json=json_conf)
+            mapper = routes.Mapper()
+            mapper.connect( "tool_runner", "/test/tool_runner" )
+            as_dict = self.toolbox.to_dict( mock_trans() )
+            test_section = self._find_section(as_dict, "t")
+            assert len(test_section["elems"]) == 1
+            assert test_section["elems"][0]["id"] == "test_tool"
 
     def test_to_dict_out_of_panel( self ):
-        self._init_tool_in_section()
-        mapper = routes.Mapper()
-        mapper.connect( "tool_runner", "/test/tool_runner" )
-        as_dict = self.toolbox.to_dict( mock_trans(), in_panel=False )
-        assert as_dict[0]["id"] == "test_tool"
+        for json_conf in [True, False]:
+            self._init_tool_in_section(json=json_conf)
+            mapper = routes.Mapper()
+            mapper.connect( "tool_runner", "/test/tool_runner" )
+            as_dict = self.toolbox.to_dict( mock_trans(), in_panel=False )
+            assert as_dict[0]["id"] == "test_tool"
 
     def test_out_of_panel_filtering( self ):
         self._init_tool_in_section()
@@ -183,7 +190,7 @@ class ToolBoxTestCase( BaseToolBoxTestCase ):
             if elem.get("id") == section_id:
                 assert elem["model_class"] == "ToolSection"
                 return elem
-        assert False, "Failed to find section with id [%s]" % id
+        assert False, "Failed to find section with id [%s]" % section_id
 
     def test_tool_shed_properties( self ):
         self._init_tool()
@@ -275,6 +282,13 @@ class ToolBoxTestCase( BaseToolBoxTestCase ):
         toolbox = self.toolbox
         assert toolbox.get_tool( "test_tool" ) is not None
 
+    def test_tool_dir_json( self ):
+        self._init_tool()
+        self._add_config({"items": [{"type": "tool_dir", "dir": self.test_directory}]}, name="tool_conf.json")
+
+        toolbox = self.toolbox
+        assert toolbox.get_tool( "test_tool" ) is not None
+
     def test_workflow_in_panel( self ):
         stored_workflow = self.__test_workflow()
         encoded_id = self.app.security.encode_id( stored_workflow.id )
@@ -305,9 +319,19 @@ class ToolBoxTestCase( BaseToolBoxTestCase ):
         section = self.toolbox._tool_panel[ 'tid' ]
         self.__check_test_labels( section.elems )
 
-    def _init_tool_in_section( self ):
+    def _init_tool_in_section( self, json=False ):
         self._init_tool()
-        self._add_config( """<toolbox><section id="t" name="test"><tool file="tool.xml" /></section></toolbox>""" )
+        if not json:
+            self._add_config( """<toolbox><section id="t" name="test"><tool file="tool.xml" /></section></toolbox>""" )
+        else:
+            section = {
+                "type": "section",
+                "id": "t",
+                "name": "test",
+                "items": [{"type": "tool",
+                           "file": "tool.xml"}],
+            }
+            self._add_config({"items": [section]}, name="tool_conf.json")
 
     def __check_test_labels( self, panel_dict ):
         assert panel_dict.keys() == ["label_lab1", "label_lab2"]

test/unit/tools/test_watcher.py

@@ -17,13 +17,43 @@ def test_watcher():
         tool_path = path.join(t, "test.xml")
         toolbox = Toolbox()
         open(tool_path, "w").write("a")
-        tool_watcher = watcher.get_watcher(toolbox, bunch.Bunch(
+        tool_watcher = watcher.get_tool_watcher(toolbox, bunch.Bunch(
             watch_tools=True
         ))
         tool_watcher.watch_file(tool_path, "cool_tool")
+        assert not toolbox.was_reloaded("cool_tool")
         open(tool_path, "w").write("b")
-        time.sleep(2)
-        toolbox.assert_reloaded("cool_tool")
+        wait_for_reload(lambda: toolbox.was_reloaded("cool_tool"))
+        tool_watcher.shutdown()
+        assert not tool_watcher.observer.is_alive()
+
+
+def test_tool_conf_watcher():
+    if not watcher.can_watch:
+        from nose.plugins.skip import SkipTest
+        raise SkipTest()
+
+    callback = CallbackRecorder()
+    conf_watcher = watcher.get_tool_conf_watcher(callback.call)
+
+    with __test_directory() as t:
+        tool_conf_path = path.join(t, "test_conf.xml")
+        conf_watcher.watch_file(tool_conf_path)
+
+        open(tool_conf_path, "w").write("b")
+        wait_for_reload(lambda: callback.called)
+        conf_watcher.shutdown()
+        assert not conf_watcher.thread.is_alive()
+
+
+def wait_for_reload(check):
+    reloaded = False
+    for i in range(10):
+        reloaded = check()
+        if reloaded:
+            break
+        time.sleep(.2)
+    assert reloaded
 
 
 class Toolbox(object):
@@ -34,8 +64,8 @@ class Toolbox(object):
     def reload_tool_by_id( self, tool_id ):
         self.reloaded[ tool_id ] = True
 
-    def assert_reloaded(self, tool_id):
-        assert self.reloaded.get( tool_id, False )
+    def was_reloaded(self, tool_id):
+        return self.reloaded.get( tool_id, False )
 
 
 class CallbackRecorder(object):