diff --git a/config/tool_conf.xml.sample b/config/tool_conf.xml.sample index d38db168b94..a3d88aa69eb 100644 --- a/config/tool_conf.xml.sample +++ b/config/tool_conf.xml.sample @@ -1,5 +1,5 @@ - +
diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index e9f7e9af4b0..87864c85d2c 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -13,6 +13,7 @@ import socket import string import sys import tempfile +import threading from datetime import timedelta from galaxy.exceptions import ConfigurationError from galaxy.util import listify @@ -744,18 +745,31 @@ class ConfiguresGalaxyMixin: def _configure_genome_builds( self, data_table_name="__dbkeys__", load_old_style=True ): self.genome_builds = GenomeBuilds( self, data_table_name=data_table_name, load_old_style=load_old_style ) - def _configure_toolbox( self ): + def reload_toolbox(self): # Initialize the tools, making sure the list of tool configs includes the reserved migrated_tools_conf.xml file. + tool_configs = self.config.tool_configs if self.config.migrated_tools_config not in tool_configs: tool_configs.append( self.config.migrated_tools_config ) + from galaxy import tools + with self._toolbox_lock: + old_toolbox = self.toolbox + self.toolbox = tools.ToolBox( tool_configs, self.config.tool_path, self ) + self.reindex_tool_search() + if old_toolbox: + old_toolbox.shutdown() + + def _configure_toolbox( self ): from galaxy.managers.citations import CitationsManager self.citations_manager = CitationsManager( self ) - from galaxy import tools - self.toolbox = tools.ToolBox( tool_configs, self.config.tool_path, self ) - self.reindex_tool_search() + from galaxy.tools.toolbox.cache import ToolCache + self.tool_cache = ToolCache() + + self._toolbox_lock = threading.Lock() + self.toolbox = None + self.reload_toolbox() from galaxy.tools.deps import containers galaxy_root_dir = os.path.abspath(self.config.root) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 32a3e799965..189bb3b7cba 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -7,6 +7,9 @@ import glob import json import logging import os +import re +import tarfile +import tempfile import threading import urllib from datetime import datetime @@ -1979,6 +1982,108 @@ class Tool( object, Dictifiable ): """ return output_collect.collect_dynamic_collections( self, output, **kwds ) + def to_archive(self): + tool = self.tool + tarball_files = [] + temp_files = [] + tool_xml = open( os.path.abspath( tool.config_file ), 'r' ).read() + # Retrieve tool help images and rewrite the tool's xml into a temporary file with the path + # modified to be relative to the repository root. + image_found = False + if tool.help is not None: + tool_help = tool.help._source + # Check each line of the rendered tool help for an image tag that points to a location under static/ + for help_line in tool_help.split( '\n' ): + image_regex = re.compile( 'img alt="[^"]+" src="\${static_path}/([^"]+)"' ) + matches = re.search( image_regex, help_line ) + if matches is not None: + tool_help_image = matches.group(1) + tarball_path = tool_help_image + filesystem_path = os.path.abspath( os.path.join( self.app.config.root, 'static', tool_help_image ) ) + if os.path.exists( filesystem_path ): + tarball_files.append( ( filesystem_path, tarball_path ) ) + image_found = True + tool_xml = tool_xml.replace( '${static_path}/%s' % tarball_path, tarball_path ) + # If one or more tool help images were found, add the modified tool XML to the tarball instead of the original. + if image_found: + fd, new_tool_config = tempfile.mkstemp( suffix='.xml' ) + os.close( fd ) + open( new_tool_config, 'w' ).write( tool_xml ) + tool_tup = ( os.path.abspath( new_tool_config ), os.path.split( tool.config_file )[-1] ) + temp_files.append( os.path.abspath( new_tool_config ) ) + else: + tool_tup = ( os.path.abspath( tool.config_file ), os.path.split( tool.config_file )[-1] ) + tarball_files.append( tool_tup ) + # TODO: This feels hacky. + tool_command = tool.command.strip().split()[0] + tool_path = os.path.dirname( os.path.abspath( tool.config_file ) ) + # Add the tool XML to the tuple that will be used to populate the tarball. + if os.path.exists( os.path.join( tool_path, tool_command ) ): + tarball_files.append( ( os.path.join( tool_path, tool_command ), tool_command ) ) + # Find and add macros and code files. + for external_file in tool.get_externally_referenced_paths( os.path.abspath( tool.config_file ) ): + external_file_abspath = os.path.abspath( os.path.join( tool_path, external_file ) ) + tarball_files.append( ( external_file_abspath, external_file ) ) + if os.path.exists( os.path.join( tool_path, "Dockerfile" ) ): + tarball_files.append( ( os.path.join( tool_path, "Dockerfile" ), "Dockerfile" ) ) + # Find tests, and check them for test data. + tests = tool.tests + if tests is not None: + for test in tests: + # Add input file tuples to the list. + for input in test.inputs: + for input_value in test.inputs[ input ]: + input_path = os.path.abspath( os.path.join( 'test-data', input_value ) ) + if os.path.exists( input_path ): + td_tup = ( input_path, os.path.join( 'test-data', input_value ) ) + tarball_files.append( td_tup ) + # And add output file tuples to the list. + for label, filename, _ in test.outputs: + output_filepath = os.path.abspath( os.path.join( 'test-data', filename ) ) + if os.path.exists( output_filepath ): + td_tup = ( output_filepath, os.path.join( 'test-data', filename ) ) + tarball_files.append( td_tup ) + for param in tool.input_params: + # Check for tool data table definitions. + if hasattr( param, 'options' ): + if hasattr( param.options, 'tool_data_table' ): + data_table = param.options.tool_data_table + if hasattr( data_table, 'filenames' ): + data_table_definitions = [] + for data_table_filename in data_table.filenames: + # FIXME: from_shed_config seems to always be False. + if not data_table.filenames[ data_table_filename ][ 'from_shed_config' ]: + tar_file = data_table.filenames[ data_table_filename ][ 'filename' ] + '.sample' + sample_file = os.path.join( data_table.filenames[ data_table_filename ][ 'tool_data_path' ], + tar_file ) + # Use the .sample file, if one exists. If not, skip this data table. + if os.path.exists( sample_file ): + tarfile_path, tarfile_name = os.path.split( tar_file ) + tarfile_path = os.path.join( 'tool-data', tarfile_name ) + tarball_files.append( ( sample_file, tarfile_path ) ) + data_table_definitions.append( data_table.xml_string ) + if len( data_table_definitions ) > 0: + # Put the data table definition XML in a temporary file. + table_definition = '\n\n %s' + table_definition = table_definition % '\n'.join( data_table_definitions ) + fd, table_conf = tempfile.mkstemp() + os.close( fd ) + open( table_conf, 'w' ).write( table_definition ) + tarball_files.append( ( table_conf, os.path.join( 'tool-data', 'tool_data_table_conf.xml.sample' ) ) ) + temp_files.append( table_conf ) + # Create the tarball. + fd, tarball_archive = tempfile.mkstemp( suffix='.tgz' ) + os.close( fd ) + tarball = tarfile.open( name=tarball_archive, mode='w:gz' ) + # Add the files from the previously generated list. + for fspath, tarpath in tarball_files: + tarball.add( fspath, arcname=tarpath ) + tarball.close() + # Delete any temporary files that were generated. + for temp_file in temp_files: + os.remove( temp_file ) + return tarball_archive + def to_dict( self, trans, link_details=False, io_details=False ): """ Returns dict of tool. """ diff --git a/lib/galaxy/tools/cwl/__init__.py b/lib/galaxy/tools/cwl/__init__.py new file mode 100644 index 00000000000..9b11c5eb320 --- /dev/null +++ b/lib/galaxy/tools/cwl/__init__.py @@ -0,0 +1,18 @@ +from .parser import tool_proxy +from .runtime_actions import handle_outputs +from .representation import to_cwl_job, to_galaxy_parameters + +from .cwltool_deps import ( + needs_shell_quoting, + shellescape, +) + + +__all__ = [ + 'tool_proxy', + 'handle_outputs', + 'to_cwl_job', + 'to_galaxy_parameters', + 'needs_shell_quoting', + 'shellescape', +] diff --git a/lib/galaxy/tools/cwl/cwltool_deps.py b/lib/galaxy/tools/cwl/cwltool_deps.py new file mode 100644 index 00000000000..fbc886fdd3f --- /dev/null +++ b/lib/galaxy/tools/cwl/cwltool_deps.py @@ -0,0 +1,48 @@ +""" This module contains logic for dealing with cwltool as an optional +dependency for Galaxy and/or applications which use Galaxy as a library. +""" + +try: + import requests +except ImportError: + requests = None + +try: + from cwltool import ( + main, + workflow, + job, + ) +except ImportError: + main = None + workflow = None + job = None + +try: + import shellescape +except ImportError: + shellescape = None + +if job is not None: + needs_shell_quoting = job.needs_shell_quoting +else: + needs_shell_quoting = None + + +def ensure_cwltool_available(): + if main is None or workflow is None or shellescape is None: + message = "This feature requires cwltool and dependencies to be available, they are not." + if requests is None: + message += " Library 'requests' unavailable." + if shellescape is None: + message += " Library 'shellescape' unavailable." + raise ImportError(message) + + +__all__ = [ + 'main', + 'workflow', + 'ensure_cwltool_available', + 'shellescape', + 'needs_shell_quoting', +] diff --git a/lib/galaxy/tools/cwl/parser.py b/lib/galaxy/tools/cwl/parser.py new file mode 100644 index 00000000000..903580f69b4 --- /dev/null +++ b/lib/galaxy/tools/cwl/parser.py @@ -0,0 +1,392 @@ +""" This module provides proxy objects around objects from the common +workflow language reference implementation library cwltool. These proxies +adapt cwltool to Galaxy features and abstract the library away from the rest +of the framework. +""" +from __future__ import absolute_import +from abc import ABCMeta, abstractmethod +from galaxy.util.odict import odict +import json +import os + +from .cwltool_deps import ( + main, + workflow, + ensure_cwltool_available, +) + +from galaxy.util.bunch import Bunch + +JOB_JSON_FILE = ".cwl_job.json" + + +def tool_proxy(tool_path): + """ Provide a proxy object to cwltool data structures to just + grab relevant data. + """ + ensure_cwltool_available() + tool = to_cwl_tool_object(tool_path) + return tool + + +def load_job_proxy(job_directory): + ensure_cwltool_available() + job_objects_path = os.path.join(job_directory, JOB_JSON_FILE) + job_objects = json.load(open(job_objects_path, "r")) + tool_path = job_objects["tool_path"] + job_inputs = job_objects["job_inputs"] + cwl_tool = tool_proxy(tool_path) + cwl_job = cwl_tool.job_proxy(job_inputs, job_directory=job_directory) + return cwl_job + + +def to_cwl_tool_object(tool_path): + if workflow is None: + raise Exception("Using CWL tools requires cwltool module.") + proxy_class = None + cwl_tool = None + make_tool = workflow.defaultMakeTool + cwl_tool = main.load_tool(tool_path, False, False, make_tool, False) + proxy_class = Draft2ToolProxy + if proxy_class is None: + raise Exception("Unsupported CWL object encountered.") + proxy = proxy_class(cwl_tool, tool_path) + return proxy + + +class ToolProxy( object ): + __metaclass__ = ABCMeta + + def __init__(self, tool, tool_path): + self._tool = tool + self._tool_path = tool_path + + def job_proxy(self, input_dict, job_directory="."): + """ Build a cwltool.job.Job describing computation using a input_json + Galaxy will generate mapping the Galaxy description of the inputs into + a cwltool compatible variant. + """ + return JobProxy(self, input_dict, job_directory=job_directory) + + @abstractmethod + def input_instances(self): + """ Return InputInstance objects describing mapping to Galaxy inputs. """ + + @abstractmethod + def output_instances(self): + """ Return OutputInstance objects describing mapping to Galaxy inputs. """ + + @abstractmethod + def docker_identifier(self): + """ Return docker identifier for embedding in tool description. """ + + @abstractmethod + def description(self): + """ Return description to tool. """ + + +class Draft2ToolProxy(ToolProxy): + + def description(self): + # Feels like I should be getting some abstract namespaced thing + # not actually description, is this correct? + return self._tool.tool.get('description') + + def input_instances(self): + return self._find_inputs(self._tool.inputs_record_schema) + + def _find_inputs(self, schema): + schema_type = schema["type"] + if isinstance(schema_type, list): + raise Exception("Union types not yet implemented.") + elif isinstance(schema_type, dict): + return self._find_inputs(schema_type) + else: + if schema_type in self._tool.schemaDefs: + schema = self._tool.schemaDefs[schema_type] + + if schema["type"] == "record": + return map(_simple_field_to_input, schema["fields"]) + + def output_instances(self): + outputs_schema = self._tool.outputs_record_schema + return self._find_outputs(outputs_schema) + + def _find_outputs(self, schema): + rval = [] + if not rval and schema["type"] == "record": + for output in schema["fields"]: + # TODO: Handle non-files differently? Make them + # JSON maybe?. + # output_type = output.get("type", None) + # if output_type != "File": + # template = "Unhandled output type [%s] encountered." + # raise Exception(template % output_type) + rval.append(_simple_field_to_output(output)) + + return rval + + def docker_identifier(self): + tool = self._tool.tool + reqs_and_hints = tool.get("requirements", []) + tool.get("hints", []) + for hint in reqs_and_hints: + if hint["class"] == "DockerRequirement": + if "dockerImageId" in hint: + return hint["dockerImageId"] + else: + return hint["dockerPull"] + return None + + +class JobProxy(object): + + def __init__(self, tool_proxy, input_dict, job_directory): + self._tool_proxy = tool_proxy + self._input_dict = input_dict + self._job_directory = job_directory + + def cwl_job(self): + return self._tool_proxy._tool.job( + self._input_dict, + self._job_directory, + self._output_callback, + use_container=False + ).next() + + def _output_callback(self, out): + pass + + def save_job(self): + job_file = JobProxy._job_file(self._job_directory) + job_objects = { + "tool_path": os.path.abspath(self._tool_proxy._tool_path), + "job_inputs": self._input_dict, + } + json.dump(job_objects, open(job_file, "w")) + + # @staticmethod + # def load_job(tool_proxy, job_directory): + # job_file = JobProxy._job_file(job_directory) + # input_dict = json.load(open(job_file, "r")) + # return JobProxy(tool_proxy, input_dict, job_directory) + + @staticmethod + def _job_file(job_directory): + return os.path.join(job_directory, JOB_JSON_FILE) + + +def _simple_field_union(field): + field_type = _field_to_field_type(field) + non_null_type = None + if len(field_type) == 2: + if "null" == field_type[0]: + non_null_type = field_type[1] + elif "null" == field_type[1]: + non_null_type = field_type[1] + + if non_null_type is None: + raise Exception("General union types not yet implemented (simple).") + + name, label, description = _field_metadata(field) + + case_name = "_cwl__type_" + case_label = "Specify %s" % label + + options = [ + {"value": "null", "label": "No", "selected": True}, + {"value": "%s" % non_null_type, "label": "Yes", "selected": False}, + ] + + case_input = SelectInputInstance( + name=case_name, + label=case_label, + description=False, + options=options, + ) + + non_null_type_kwds = _simple_field_to_input_type_kwds(field, field_type=non_null_type) + non_null_name = "_cwl__value_" + non_null_label = label + non_null_description = description + non_null_input = InputInstance( + non_null_name, + non_null_label, + non_null_description, + **non_null_type_kwds + ) + options = [ + ("null", []), + (non_null_type, [non_null_input]), + ] + return ConditionalInstance(name, case_input, options) + + +def _simple_field_to_input(field): + field_type = _field_to_field_type(field) + if isinstance(field_type, list): + # Length must be greater than 1... + return _simple_field_union(field) + + name, label, description = _field_metadata(field) + + type_kwds = _simple_field_to_input_type_kwds(field) + return InputInstance(name, label, description, **type_kwds) + + +def _simple_field_to_input_type_kwds(field, field_type=None): + simple_map_type_map = { + "File": INPUT_TYPE.DATA, + "int": INPUT_TYPE.INTEGER, + "string": INPUT_TYPE.TEXT, + "boolean": INPUT_TYPE.BOOLEAN, + } + + if field_type is None: + field_type = _field_to_field_type(field) + + if field_type in simple_map_type_map.keys(): + input_type = simple_map_type_map[field_type] + return {"input_type": input_type, "array": False} + elif field_type == "array": + if isinstance(field["type"], dict): + array_type = field["type"]["items"] + else: + array_type = field["items"] + if array_type in simple_map_type_map.keys(): + input_type = simple_map_type_map[array_type] + return {"input_type": input_type, "array": True} + else: + raise Exception("Unhandled simple field type encountered - [%s]." % field_type) + + +def _field_to_field_type(field): + field_type = field["type"] + if isinstance(field_type, dict): + field_type = field_type["type"] + if isinstance(field_type, list): + field_type_length = len(field_type) + if field_type_length == 0: + raise Exception("Zero-length type list encountered, invalid CWL?") + elif len(field_type) == 1: + field_type = field_type[0] + + return field_type + + +def _field_metadata(field): + name = field["name"] + label = field.get("label", None) + description = field.get("description", None) + return name, label, description + + +def _simple_field_to_output(field): + name = field["name"] + output_instance = OutputInstance(name, output_type=OUTPUT_TYPE.GLOB) + return output_instance + + +INPUT_TYPE = Bunch( + DATA="data", + INTEGER="integer", + TEXT="text", + BOOLEAN="boolean", + SELECT="select", + CONDITIONAL="conditional", +) + + +class ConditionalInstance(object): + + def __init__(self, name, case, whens): + self.input_type = INPUT_TYPE.CONDITIONAL + self.name = name + self.case = case + self.whens = whens + + def to_dict(self): + + as_dict = dict( + name=self.name, + type=INPUT_TYPE.CONDITIONAL, + test=self.case.to_dict(), + when=odict(), + ) + for value, block in self.whens: + as_dict["when"][value] = map(lambda i: i.to_dict(), block) + + return as_dict + + +class SelectInputInstance(object): + + def __init__(self, name, label, description, options): + self.input_type = INPUT_TYPE.SELECT + self.name = name + self.label = label + self.description = description + self.options = options + + def to_dict(self): + # TODO: serialize options... + as_dict = dict( + name=self.name, + label=self.label or self.name, + help=self.description, + type=self.input_type, + options=self.options, + ) + return as_dict + + +class InputInstance(object): + + def __init__(self, name, label, description, input_type, array=False): + self.input_type = input_type + self.name = name + self.label = label + self.description = description + self.required = True + self.array = array + + def to_dict(self, itemwise=True): + if itemwise and self.array: + as_dict = dict( + type="repeat", + name="%s_repeat" % self.name, + title="%s" % self.name, + blocks=[ + self.to_dict(itemwise=False) + ] + ) + else: + as_dict = dict( + name=self.name, + label=self.label or self.name, + help=self.description, + type=self.input_type, + optional=not self.required, + ) + if self.input_type == INPUT_TYPE.INTEGER: + as_dict["value"] = "0" + return as_dict + + +OUTPUT_TYPE = Bunch( + GLOB="glob", + STDOUT="stdout", +) + + +class OutputInstance(object): + + def __init__(self, name, output_type, path=None): + self.name = name + self.output_type = output_type + self.path = path + + +__all__ = [ + 'tool_proxy', + 'load_job_proxy', +] diff --git a/lib/galaxy/tools/cwl/representation.py b/lib/galaxy/tools/cwl/representation.py new file mode 100644 index 00000000000..337cb5f6c9f --- /dev/null +++ b/lib/galaxy/tools/cwl/representation.py @@ -0,0 +1,100 @@ +""" This module is responsible for converting between Galaxy's tool +input description and the CWL description for a job json. """ + +import logging + +from galaxy.util import string_as_bool + +log = logging.getLogger(__name__) + +NOT_PRESENT = object() + + +def to_cwl_job(tool, param_dict): + """ tool is Galaxy's representation of the tool and param_dict is the + parameter dictionary with wrapped values. + """ + inputs = tool.inputs + input_json = {} + + def simple_value(input, param_dict_value): + if input.type == "data": + return {"path": str(param_dict_value), "class": "File"} + elif input.type == "integer": + return int(str(param_dict_value)) + elif input.type == "boolean": + return string_as_bool(param_dict_value) + else: + return str(param_dict_value) + + for input_name, input in inputs.iteritems(): + if input.type == "repeat": + only_input = input.inputs.values()[0] + array_value = [] + for instance in param_dict[input_name]: + array_value.append(simple_value(only_input, instance[input_name[:-len("_repeat")]])) + input_json[input_name[:-len("_repeat")]] = array_value + elif input.type == "conditional": + assert input_name in param_dict, "No value for %s in %s" % (input_name, param_dict) + current_case = param_dict[input_name]["_cwl__type_"] + if current_case != "null": + # TODO: make sure trans is not needed here... + case_index = input.get_current_case( current_case, None ) + case_input = input.cases[ case_index ].inputs[ "_cwl__value_"] + case_value = param_dict[input_name]["_cwl__value_"] + input_json[input_name] = simple_value(case_input, case_value) + else: + input_json[input_name] = simple_value(input, param_dict[input_name]) + + input_json["allocatedResources"] = { + "cpu": "$GALAXY_SLOTS", + } + return input_json + + +def to_galaxy_parameters(tool, as_dict): + """ Tool is Galaxy's representation of the tool and as_dict is a Galaxified + representation of the input json (no paths, HDA references for instance). + """ + inputs = tool.inputs + galaxy_request = {} + + def from_simple_value(input, param_dict_value): + return param_dict_value + + for input_name, input in inputs.iteritems(): + as_dict_value = as_dict.get(input_name, NOT_PRESENT) + + if input.type == "repeat": + if input_name not in as_dict: + continue + + only_input = input.inputs.values()[0] + for index, value in enumerate(as_dict_value): + key = "%s_repeat_0|%s" % (input_name, only_input.name) + galaxy_value = from_simple_value(only_input, value) + galaxy_request[key] = galaxy_value + elif input.type == "conditional": + # TODO: less crazy handling of defaults... + if as_dict_value is NOT_PRESENT: + cwl_type = "null" + elif isinstance(as_dict_value, bool): + cwl_type = "boolean" + elif isinstance(as_dict_value, int): + cwl_type = "integer" + else: + cwl_type = "string" + galaxy_request["%s|_cwl__type_" % input_name] = cwl_type + if cwl_type != "null": + current_case_index = input.get_current_case(cwl_type, None) + current_case_input = input.cases[ current_case_index ].inputs[ "_cwl__value_" ] + galaxy_value = from_simple_value(current_case_input, as_dict_value) + galaxy_request["%s|_cwl__value_" % input_name] = galaxy_value + elif as_dict_value is NOT_PRESENT: + continue + else: + galaxy_value = from_simple_value(input, as_dict_value) + galaxy_request[input_name] = galaxy_value + + log.info("Converted galaxy_request is %s" % galaxy_request) + return galaxy_request diff --git a/lib/galaxy/tools/cwl/runtime_actions.py b/lib/galaxy/tools/cwl/runtime_actions.py new file mode 100644 index 00000000000..cd4b2a0a1cf --- /dev/null +++ b/lib/galaxy/tools/cwl/runtime_actions.py @@ -0,0 +1,29 @@ +import os +import shutil + +from .parser import ( + JOB_JSON_FILE, + load_job_proxy, +) + + +def handle_outputs(job_directory=None): + # Relocate dynamically collected files to pre-determined locations + # registered with ToolOutput objects via from_work_dir handling. + if job_directory is None: + job_directory = os.getcwd() + cwl_job_file = os.path.join(job_directory, JOB_JSON_FILE) + if not os.path.exists(cwl_job_file): + # Not a CWL job, just continue + return + job_proxy = load_job_proxy(job_directory) + cwl_job = job_proxy.cwl_job() + outputs = cwl_job.collect_outputs(job_directory) + for output_name, output in outputs.iteritems(): + target_path = os.path.join(job_directory, "__cwl_output_%s" % output_name) + shutil.move(output["path"], target_path) + + +__all__ = [ + 'handle_outputs', +] diff --git a/lib/galaxy/tools/loader_directory.py b/lib/galaxy/tools/loader_directory.py index 907adcf466e..57d387e5681 100644 --- a/lib/galaxy/tools/loader_directory.py +++ b/lib/galaxy/tools/loader_directory.py @@ -19,11 +19,36 @@ LOAD_FAILURE_ERROR = "Failed to load tool with path %s." TOOL_LOAD_ERROR = object() TOOL_REGEX = re.compile(r" tag. self._tool_root_dir = tool_root_dir self.app = app - self._tool_watcher = get_watcher( self, app.config ) + self._tool_watcher = get_tool_watcher( self, app.config ) + self._tool_conf_watcher = get_tool_conf_watcher( lambda: app.reload_toolbox() ) self._filter_factory = FilterFactory( self ) self._tool_tag_manager = tool_tag_manager( app ) self._init_tools_from_configs( config_filenames ) @@ -121,9 +123,8 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): """ log.info( "Parsing the tool configuration %s" % config_filename ) - tree = parse_xml( config_filename ) - root = tree.getroot() - tool_path = root.get( 'tool_path' ) + tool_conf_source = get_toolbox_parser(config_filename) + tool_path = tool_conf_source.parse_tool_path() if tool_path: # We're parsing a shed_tool_conf file since we have a tool_path attribute. parsing_shed_tool_conf = True @@ -134,16 +135,16 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): tool_path = self.__resolve_tool_path(tool_path, config_filename) # Only load the panel_dict under certain conditions. load_panel_dict = not self._integrated_tool_panel_config_has_contents - for _, elem in enumerate( root ): + for item in tool_conf_source.parse_items(): index = self._index self._index += 1 if parsing_shed_tool_conf: - config_elems.append( elem ) + config_elems.append( item.elem ) self.load_item( - elem, + item, tool_path=tool_path, load_panel_dict=load_panel_dict, - guid=elem.get( 'guid' ), + guid=item.get( 'guid' ), index=index, internal=True ) @@ -154,8 +155,12 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): config_elems=config_elems ) self._dynamic_tool_confs.append( shed_tool_conf_dict ) - def load_item( self, elem, tool_path, panel_dict=None, integrated_panel_dict=None, load_panel_dict=True, guid=None, index=None, internal=False ): - item_type = elem.tag + if tool_conf_source.parse_monitor(): + self._tool_conf_watcher.watch_file(config_filename) + + def load_item( self, item, tool_path, panel_dict=None, integrated_panel_dict=None, load_panel_dict=True, guid=None, index=None, internal=False ): + item = ensure_tool_conf_item(item) + item_type = item.type if item_type not in ['tool', 'section'] and not internal: # External calls from tool shed code cannot load labels or tool # directories. @@ -166,15 +171,15 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): if integrated_panel_dict is None: integrated_panel_dict = self._integrated_tool_panel if item_type == 'tool': - self._load_tool_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, tool_path=tool_path, load_panel_dict=load_panel_dict, guid=guid, index=index ) + self._load_tool_tag_set( item, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, tool_path=tool_path, load_panel_dict=load_panel_dict, guid=guid, index=index ) elif item_type == 'workflow': - self._load_workflow_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) + self._load_workflow_tag_set( item, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) elif item_type == 'section': - self._load_section_tag_set( elem, tool_path=tool_path, load_panel_dict=load_panel_dict, index=index ) + self._load_section_tag_set( item, tool_path=tool_path, load_panel_dict=load_panel_dict, index=index ) elif item_type == 'label': - self._load_label_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) + self._load_label_tag_set( item, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) elif item_type == 'tool_dir': - self._load_tooldir_tag_set( elem, panel_dict, tool_path, integrated_panel_dict, load_panel_dict=load_panel_dict ) + self._load_tooldir_tag_set( item, panel_dict, tool_path, integrated_panel_dict, load_panel_dict=load_panel_dict ) def get_shed_config_dict_by_filename( self, filename, default=None ): for shed_config_dict in self._dynamic_tool_confs: @@ -533,9 +538,9 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): else: remove_from_dict( self._tool_panel, self._integrated_tool_panel ) - def _load_tool_tag_set( self, elem, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None, index=None ): + def _load_tool_tag_set( self, item, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None, index=None ): try: - path = elem.get( "file" ) + path = item.get( "file" ) repository_id = None tool_shed_repository = None @@ -543,13 +548,13 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): if guid is not None: # The tool is contained in an installed tool shed repository, so load # the tool only if the repository has not been marked deleted. - tool_shed = elem.find( "tool_shed" ).text - repository_name = elem.find( "repository_name" ).text - repository_owner = elem.find( "repository_owner" ).text - installed_changeset_revision_elem = elem.find( "installed_changeset_revision" ) + tool_shed = item.elem.find( "tool_shed" ).text + repository_name = item.elem.find( "repository_name" ).text + repository_owner = item.elem.find( "repository_owner" ).text + installed_changeset_revision_elem = item.elem.find( "installed_changeset_revision" ) if installed_changeset_revision_elem is None: # Backward compatibility issue - the tag used to be named 'changeset_revision'. - installed_changeset_revision_elem = elem.find( "changeset_revision" ) + installed_changeset_revision_elem = item.elem.find( "changeset_revision" ) installed_changeset_revision = installed_changeset_revision_elem.text tool_shed_repository = self._get_tool_shed_repository( tool_shed, repository_name, @@ -563,7 +568,7 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): # Else there is not yet a tool_shed_repository record, we're in the process of installing # a new repository, so any included tools can be loaded into the tool panel. tool = self.load_tool( os.path.join( tool_path, path ), guid=guid, repository_id=repository_id ) - if string_as_bool(elem.get( 'hidden', False )): + if string_as_bool(item.get( 'hidden', False )): tool.hidden = True key = 'tool_%s' % str( tool.id ) if can_load_into_panel_dict: @@ -573,19 +578,21 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): tool.repository_owner = repository_owner tool.installed_changeset_revision = installed_changeset_revision tool.guid = guid - tool.version = elem.find( "version" ).text + tool.version = item.elem.find( "version" ).text # Make sure the tool has a tool_version. tool_lineage = self._lineage_map.register( tool, tool_shed_repository=tool_shed_repository ) # Load the tool's lineage ids. tool.lineage = tool_lineage - self._tool_tag_manager.handle_tags( tool.id, elem ) + if item.has_elem: + self._tool_tag_manager.handle_tags( tool.id, item.elem ) self.__add_tool( tool, load_panel_dict, panel_dict ) # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file. integrated_panel_dict.update_or_append( index, key, tool ) # If labels were specified in the toolbox config, attach them to # the tool. - if "labels" in elem.attrib: - tool.labels = [ label.strip() for label in elem.attrib["labels"].split( "," ) ] + labels = item.labels + if labels is not None: + tool.labels = labels except IOError: log.error( "Error reading tool configuration file from path: %s." % path ) except Exception: @@ -609,10 +616,10 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): if load_panel_dict: self.__add_tool_to_tool_panel( tool, panel_dict, section=isinstance( panel_dict, ToolSection ) ) - def _load_workflow_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): + def _load_workflow_tag_set( self, item, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): try: # TODO: should id be encoded? - workflow_id = elem.get( 'id' ) + workflow_id = item.get( 'id' ) workflow = self._load_workflow( workflow_id ) self._workflows_by_id[ workflow_id ] = workflow key = 'workflow_' + workflow_id @@ -623,35 +630,35 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): except: log.exception( "Error loading workflow: %s" % workflow_id ) - def _load_label_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): - label = ToolSectionLabel( elem ) + def _load_label_tag_set( self, item, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): + label = ToolSectionLabel( item ) key = 'label_' + label.id if load_panel_dict: panel_dict[ key ] = label integrated_panel_dict.update_or_append( index, key, label ) - def _load_section_tag_set( self, elem, tool_path, load_panel_dict, index=None ): - key = elem.get( "id" ) + def _load_section_tag_set( self, item, tool_path, load_panel_dict, index=None ): + key = item.get( "id" ) if key in self._tool_panel: section = self._tool_panel[ key ] elems = section.elems else: - section = ToolSection( elem ) + section = ToolSection( item ) elems = section.elems if key in self._integrated_tool_panel: integrated_section = self._integrated_tool_panel[ key ] integrated_elems = integrated_section.elems else: - integrated_section = ToolSection( elem ) + integrated_section = ToolSection( item ) integrated_elems = integrated_section.elems - for sub_index, sub_elem in enumerate( elem ): + for sub_index, sub_item in enumerate( item.items ): self.load_item( - sub_elem, + sub_item, tool_path=tool_path, panel_dict=elems, integrated_panel_dict=integrated_elems, load_panel_dict=load_panel_dict, - guid=sub_elem.get( 'guid' ), + guid=sub_item.get( 'guid' ), index=sub_index, internal=True, ) @@ -668,9 +675,9 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): # Always load sections into the integrated_tool_panel. self._integrated_tool_panel.update_or_append( index, key, integrated_section ) - def _load_tooldir_tag_set(self, sub_elem, elems, tool_path, integrated_elems, load_panel_dict): - directory = os.path.join( tool_path, sub_elem.attrib.get("dir") ) - recursive = string_as_bool( sub_elem.attrib.get("recursive", True) ) + def _load_tooldir_tag_set(self, item, elems, tool_path, integrated_elems, load_panel_dict): + directory = os.path.join( tool_path, item.get("dir") ) + recursive = string_as_bool( item.get("recursive", True) ) self.__watch_directory( directory, elems, integrated_elems, load_panel_dict, recursive, force_watch=True ) def __watch_directory( self, directory, elems, integrated_elems, load_panel_dict, recursive, force_watch=False ): @@ -702,10 +709,15 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): if tool_loaded or force_watch: self._tool_watcher.watch_directory( directory, quick_load ) - def load_tool( self, config_file, guid=None, repository_id=None, **kwds ): + def load_tool( self, config_file, guid=None, repository_id=None, use_cached=True, **kwds ): """Load a single tool from the file named by `config_file` and return an instance of `Tool`.""" # Parse XML configuration file and get the root element - tool = self.create_tool( config_file=config_file, repository_id=repository_id, guid=guid, **kwds ) + tool_cache = getattr( self.app, 'tool_cache', None ) + tool = use_cached and tool_cache and tool_cache.get_tool( config_file ) + if not tool: + tool = self.create_tool( config_file=config_file, repository_id=repository_id, guid=guid, **kwds ) + if tool_cache: + self.app.tool_cache.cache_tool(config_file, tool) tool_id = tool.id if not tool_id.startswith("__"): # do not monitor special tools written to tmp directory - no reason @@ -750,109 +762,10 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): """ # Make sure the tool is actually loaded. if tool_id not in self._tools_by_id: - return None, False, "No tool with id %s" % escape( tool_id ) + raise ObjectNotFound("No tool found with id '%s'." % escape( tool_id )) else: tool = self._tools_by_id[ tool_id ] - tarball_files = [] - temp_files = [] - tool_xml = open( os.path.abspath( tool.config_file ), 'r' ).read() - # Retrieve tool help images and rewrite the tool's xml into a temporary file with the path - # modified to be relative to the repository root. - image_found = False - if tool.help is not None: - tool_help = tool.help._source - # Check each line of the rendered tool help for an image tag that points to a location under static/ - for help_line in tool_help.split( '\n' ): - image_regex = re.compile( 'img alt="[^"]+" src="\${static_path}/([^"]+)"' ) - matches = re.search( image_regex, help_line ) - if matches is not None: - tool_help_image = matches.group(1) - tarball_path = tool_help_image - filesystem_path = os.path.abspath( os.path.join( trans.app.config.root, 'static', tool_help_image ) ) - if os.path.exists( filesystem_path ): - tarball_files.append( ( filesystem_path, tarball_path ) ) - image_found = True - tool_xml = tool_xml.replace( '${static_path}/%s' % tarball_path, tarball_path ) - # If one or more tool help images were found, add the modified tool XML to the tarball instead of the original. - if image_found: - fd, new_tool_config = tempfile.mkstemp( suffix='.xml' ) - os.close( fd ) - open( new_tool_config, 'w' ).write( tool_xml ) - tool_tup = ( os.path.abspath( new_tool_config ), os.path.split( tool.config_file )[-1] ) - temp_files.append( os.path.abspath( new_tool_config ) ) - else: - tool_tup = ( os.path.abspath( tool.config_file ), os.path.split( tool.config_file )[-1] ) - tarball_files.append( tool_tup ) - # TODO: This feels hacky. - tool_command = tool.command.strip().split()[0] - tool_path = os.path.dirname( os.path.abspath( tool.config_file ) ) - # Add the tool XML to the tuple that will be used to populate the tarball. - if os.path.exists( os.path.join( tool_path, tool_command ) ): - tarball_files.append( ( os.path.join( tool_path, tool_command ), tool_command ) ) - # Find and add macros and code files. - for external_file in tool.get_externally_referenced_paths( os.path.abspath( tool.config_file ) ): - external_file_abspath = os.path.abspath( os.path.join( tool_path, external_file ) ) - tarball_files.append( ( external_file_abspath, external_file ) ) - if os.path.exists( os.path.join( tool_path, "Dockerfile" ) ): - tarball_files.append( ( os.path.join( tool_path, "Dockerfile" ), "Dockerfile" ) ) - # Find tests, and check them for test data. - tests = tool.tests - if tests is not None: - for test in tests: - # Add input file tuples to the list. - for input in test.inputs: - for input_value in test.inputs[ input ]: - input_path = os.path.abspath( os.path.join( 'test-data', input_value ) ) - if os.path.exists( input_path ): - td_tup = ( input_path, os.path.join( 'test-data', input_value ) ) - tarball_files.append( td_tup ) - # And add output file tuples to the list. - for label, filename, _ in test.outputs: - output_filepath = os.path.abspath( os.path.join( 'test-data', filename ) ) - if os.path.exists( output_filepath ): - td_tup = ( output_filepath, os.path.join( 'test-data', filename ) ) - tarball_files.append( td_tup ) - for param in tool.input_params: - # Check for tool data table definitions. - if hasattr( param, 'options' ): - if hasattr( param.options, 'tool_data_table' ): - data_table = param.options.tool_data_table - if hasattr( data_table, 'filenames' ): - data_table_definitions = [] - for data_table_filename in data_table.filenames: - # FIXME: from_shed_config seems to always be False. - if not data_table.filenames[ data_table_filename ][ 'from_shed_config' ]: - tar_file = data_table.filenames[ data_table_filename ][ 'filename' ] + '.sample' - sample_file = os.path.join( data_table.filenames[ data_table_filename ][ 'tool_data_path' ], - tar_file ) - # Use the .sample file, if one exists. If not, skip this data table. - if os.path.exists( sample_file ): - tarfile_path, tarfile_name = os.path.split( tar_file ) - tarfile_path = os.path.join( 'tool-data', tarfile_name ) - tarball_files.append( ( sample_file, tarfile_path ) ) - data_table_definitions.append( data_table.xml_string ) - if len( data_table_definitions ) > 0: - # Put the data table definition XML in a temporary file. - table_definition = '\n\n %s' - table_definition = table_definition % '\n'.join( data_table_definitions ) - fd, table_conf = tempfile.mkstemp() - os.close( fd ) - open( table_conf, 'w' ).write( table_definition ) - tarball_files.append( ( table_conf, os.path.join( 'tool-data', 'tool_data_table_conf.xml.sample' ) ) ) - temp_files.append( table_conf ) - # Create the tarball. - fd, tarball_archive = tempfile.mkstemp( suffix='.tgz' ) - os.close( fd ) - tarball = tarfile.open( name=tarball_archive, mode='w:gz' ) - # Add the files from the previously generated list. - for fspath, tarpath in tarball_files: - tarball.add( fspath, arcname=tarpath ) - tarball.close() - # Delete any temporary files that were generated. - for temp_file in temp_files: - os.remove( temp_file ) - return tarball_archive, True, None - return None, False, "An unknown error occurred." + return tool.to_archive() def reload_tool_by_id( self, tool_id ): """ @@ -908,6 +821,9 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): else: tool = self._tools_by_id[ tool_id ] del self._tools_by_id[ tool_id ] + tool_cache = getattr( self.app, 'tool_cache', None ) + if tool_cache: + tool_cache.expire_tool( tool_id ) if remove_from_panel: tool_key = 'tool_' + tool_id for key, val in self._tool_panel.items(): @@ -995,6 +911,21 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): return rval + def shutdown(self): + exception = None + try: + self._tool_watcher.shutdown() + except Exception as e: + exception = e + + try: + self._tool_conf_watcher.shutdown() + except Exception as e: + exception = exception or e + + if exception: + raise exception + def _lineage_in_panel( self, panel_dict, tool=None, tool_lineage=None ): """ If tool with same lineage already in panel (or section) - find and return it. Otherwise return None. diff --git a/lib/galaxy/tools/toolbox/cache.py b/lib/galaxy/tools/toolbox/cache.py new file mode 100644 index 00000000000..3bc524024eb --- /dev/null +++ b/lib/galaxy/tools/toolbox/cache.py @@ -0,0 +1,26 @@ + + +class ToolCache(object): + """ Cache tool defintions to allow quickly reloading the whole + toolbox. + """ + + def __init__(self): + self._tools_by_path = {} + self._tool_paths_by_id = {} + + def get_tool(self, config_filename): + """ Get the tool from the cache if the tool is up to date. + """ + return self._tools_by_path.get(config_filename, None) + + def expire_tool(self, tool_id): + if tool_id in self._tool_paths_by_id: + config_filename = self._tool_paths_by_id[tool_id] + del self._tool_paths_by_id[tool_id] + del self._tools_by_path[config_filename] + + def cache_tool(self, config_filename, tool): + tool_id = str( tool.id ) + self._tool_paths_by_id[tool_id] = config_filename + self._tools_by_path[config_filename] = tool diff --git a/lib/galaxy/tools/toolbox/panel.py b/lib/galaxy/tools/toolbox/panel.py index 9e77231aceb..5dd77e071c4 100644 --- a/lib/galaxy/tools/toolbox/panel.py +++ b/lib/galaxy/tools/toolbox/panel.py @@ -3,6 +3,7 @@ from abc import abstractmethod from galaxy.util.odict import odict from galaxy.util import bunch from galaxy.util.dictifiable import Dictifiable +from .parser import ensure_tool_conf_item from six import iteritems @@ -49,13 +50,13 @@ class ToolSection( Dictifiable, HasPanelItems, object ): dict_collection_visible_keys = ( 'id', 'name', 'version' ) - def __init__( self, elem=None ): + def __init__( self, item=None ): """ Build a ToolSection from an ElementTree element or a dictionary. """ - f = lambda elem, val: elem is not None and elem.get( val ) or '' - self.name = f( elem, 'name' ) - self.id = f( elem, 'id' ) - self.version = f( elem, 'version' ) + f = lambda item, val: item is not None and item.get( val ) or '' + self.name = f( item, 'name' ) + self.id = f( item, 'id' ) + self.version = f( item, 'version' ) self.elems = ToolPanelElements() def copy( self ): @@ -93,13 +94,14 @@ class ToolSectionLabel( Dictifiable, object ): dict_collection_visible_keys = ( 'id', 'text', 'version' ) - def __init__( self, elem ): + def __init__( self, item ): """ Build a ToolSectionLabel from an ElementTree element or a dictionary. """ - self.text = elem.get( "text" ) - self.id = elem.get( "id" ) - self.version = elem.get( "version" ) or '' + item = ensure_tool_conf_item(item) + self.text = item.get( "text" ) + self.id = item.get( "id" ) + self.version = item.get( "version" ) or '' def to_dict( self, **kwds ): return super( ToolSectionLabel, self ).to_dict() diff --git a/lib/galaxy/tools/toolbox/parser.py b/lib/galaxy/tools/toolbox/parser.py new file mode 100644 index 00000000000..83d0055c1de --- /dev/null +++ b/lib/galaxy/tools/toolbox/parser.py @@ -0,0 +1,138 @@ +from abc import ABCMeta +from abc import abstractmethod + +from galaxy.util import parse_xml, string_as_bool +import yaml + +DEFAULT_MONITOR = False + + +class ToolConfSource(object): + """ This interface represents an abstract source to parse tool + information from. + """ + __metaclass__ = ABCMeta + + @abstractmethod + def parse_items(self): + """ Return a list of ToolConfItem + """ + + @abstractmethod + def parse_tool_path(self): + """ Return tool_path for tools in this toolbox. + """ + + def parse_monitor(self): + """ Monitor the toolbox configuration source for changes and + reload. """ + return DEFAULT_MONITOR + + +class XmlToolConfSource(ToolConfSource): + + def __init__(self, config_filename): + tree = parse_xml(config_filename) + self.root = tree.getroot() + + def parse_tool_path(self): + return self.root.get('tool_path') + + def parse_items(self): + return map(ensure_tool_conf_item, self.root.getchildren()) + + def parse_monitor(self): + return string_as_bool(self.root.get('monitor', DEFAULT_MONITOR)) + + +class YamlToolConfSource(ToolConfSource): + + def __init__(self, config_filename): + with open(config_filename, "r") as f: + as_dict = yaml.load(f) + self.as_dict = as_dict + + def parse_tool_path(self): + return self.as_dict.get('tool_path') + + def parse_items(self): + return map(ToolConfItem.from_dict, self.as_dict.get('items')) + + def parse_monitor(self): + return self.as_dict.get('monitor', DEFAULT_MONITOR) + + +class ToolConfItem(object): + """ This interface represents an abstract source to parse tool + information from. + """ + + def __init__(self, type, attributes, elem=None): + self.type = type + self.attributes = attributes + self._elem = elem + + @classmethod + def from_dict(cls, _as_dict): + as_dict = _as_dict.copy() + type = as_dict.get('type') + del as_dict['type'] + attributes = as_dict + if type == 'section': + items = map(cls.from_dict, as_dict['items']) + del as_dict['items'] + item = ToolConfSection(attributes, items) + else: + item = ToolConfItem(type, attributes) + return item + + def get(self, key, default=None): + return self.attributes.get(key, default) + + @property + def has_elem(self): + return self._elem is not None + + @property + def elem(self): + if self._elem is None: + raise Exception("item.elem called on toolbox element from non-XML source") + return self._elem + + @property + def labels(self): + labels = None + if "labels" in self.attributes: + labels = [ label.strip() for label in self.attributes["labels"].split( "," ) ] + return labels + + +class ToolConfSection(ToolConfItem): + + def __init__(self, attributes, items, elem=None): + super(ToolConfSection, self).__init__('section', attributes, elem) + self.items = items + + +def ensure_tool_conf_item(xml_or_item): + if xml_or_item is None: + return None + elif isinstance(xml_or_item, ToolConfItem): + return xml_or_item + else: + elem = xml_or_item + type = elem.tag + attributes = elem.attrib + if type != "section": + return ToolConfItem(type, attributes, elem) + else: + items = map(ensure_tool_conf_item, elem.getchildren()) + return ToolConfSection(attributes, items, elem=elem) + + +def get_toolbox_parser(config_filename): + is_yaml = any(map(lambda e: config_filename.endswith(e), [".yml", ".yaml", ".json"])) + if is_yaml: + return YamlToolConfSource(config_filename) + else: + return XmlToolConfSource(config_filename) diff --git a/lib/galaxy/tools/toolbox/watcher.py b/lib/galaxy/tools/toolbox/watcher.py index c1a3ca7d7a9..ef4b65947bd 100644 --- a/lib/galaxy/tools/toolbox/watcher.py +++ b/lib/galaxy/tools/toolbox/watcher.py @@ -1,4 +1,8 @@ +import logging import os.path +import threading +import time + try: from watchdog.events import FileSystemEventHandler from watchdog.observers import Observer @@ -9,34 +13,131 @@ except ImportError: PollingObserver = object can_watch = False -import logging log = logging.getLogger( __name__ ) -def get_watcher(toolbox, config): - watch_tools_val = str( getattr(config, "watch_tools", False) ).lower() - if watch_tools_val in ( 'true', 'yes', 'on' ): - return ToolWatcher(toolbox) - elif watch_tools_val == "auto": - try: - return ToolWatcher(toolbox) - except Exception: - log.info("Failed to load ToolWatcher (watchdog is likely unavailable) - proceeding without tool monitoring.") - return NullWatcher() - elif watch_tools_val == "polling": - log.info("Using less ineffecient polling toolbox watcher.") - return ToolWatcher(toolbox, observer_class=PollingObserver) +def get_observer_class(config_value, default, monitor_what_str): + """ + """ + config_value = config_value or default + config_value = str(config_value).lower() + if config_value in ("true", "yes", "on", "auto"): + expect_observer = config_value != "auto" + observer_class = Observer + elif config_value == "polling": + expect_observer = True + observer_class = PollingObserver + else: + expect_observer = False + observer_class = None + + if observer_class is None: + message = "Watchdog library unavailble, cannot monitor %s." % monitor_what_str + log.info(message) + if expect_observer: + raise Exception(message) + + return observer_class + + +def get_tool_conf_watcher(reload_callback): + return ToolConfWatcher(reload_callback) + + +def get_tool_watcher(toolbox, config): + config_value = getattr(config, "watch_tools", None) + observer_class = get_observer_class(config_value, default="False", monitor_what_str="tools") + + if observer_class is not None: + return ToolWatcher(toolbox, observer_class=observer_class) else: return NullWatcher() +class ToolConfWatcher(object): + + def __init__(self, reload_callback): + self.paths = {} + self._active = False + self._lock = threading.Lock() + self.thread = threading.Thread(target=self.check) + self.thread.daemon = True + self.event_handler = ToolConfFileEventHandler(reload_callback) + + def start(self): + if not self._active: + self._active = True + self.thread.start() + + def shutdown(self): + if self._active: + self._active = False + self.thread.join() + + def check(self): + while self._active: + do_reload = False + with self._lock: + paths = list(self.paths.keys()) + for path in paths: + if not os.path.exists(path): + continue + mod_time = self.paths[path] + new_mod_time = None + if os.path.exists(path): + new_mod_time = time.ctime(os.path.getmtime(path)) + if new_mod_time != mod_time: + self.paths[path] = new_mod_time + do_reload = True + + if do_reload: + t = threading.Thread(target=lambda: self.event_handler.on_any_event(None)) + t.daemon = True + t.start() + time.sleep(1) + + def monitor(self, path): + mod_time = None + if os.path.exists(path): + mod_time = time.ctime(os.path.getmtime(path)) + with self._lock: + self.paths[path] = mod_time + self.start() + + def watch_file(self, tool_conf_file): + self.monitor(tool_conf_file) + + +class NullToolConfWatcher(object): + + def start(self): + pass + + def shutdown(self): + pass + + def monitor(self, conf_path): + pass + + def watch_file(self, tool_file, tool_id): + pass + + +class ToolConfFileEventHandler(FileSystemEventHandler): + + def __init__(self, reload_callback): + self.reload_callback = reload_callback + + def on_any_event(self, event): + self._handle(event) + + def _handle(self, event): + self.reload_callback() + + class ToolWatcher(object): - def __init__(self, toolbox, observer_class=None): - if not can_watch: - raise Exception("Watchdog library unavailble, cannot watch tools.") - if observer_class is None: - observer_class = Observer + def __init__(self, toolbox, observer_class): self.toolbox = toolbox self.tool_file_ids = {} self.tool_dir_callbacks = {} @@ -48,6 +149,10 @@ class ToolWatcher(object): def start(self): self.observer.start() + def shutdown(self): + self.observer.stop() + self.observer.join() + def monitor(self, dir): self.observer.schedule(self.event_handler, dir, recursive=False) diff --git a/lib/galaxy/web/base/controllers/admin.py b/lib/galaxy/web/base/controllers/admin.py index 360e2c98907..57ffca7b485 100644 --- a/lib/galaxy/web/base/controllers/admin.py +++ b/lib/galaxy/web/base/controllers/admin.py @@ -68,26 +68,24 @@ class Admin( object ): def package_tool( self, trans, **kwd ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) toolbox = self.app.toolbox tool_id = None if params.get( 'package_tool_button', False ): tool_id = params.get('tool_id', None) - tool_tarball, success, message = trans.app.toolbox.package_tool( trans, tool_id ) - if success: + try: + tool_tarball = trans.app.toolbox.package_tool( trans, tool_id ) trans.response.set_content_type( 'application/x-gzip' ) download_file = open( tool_tarball ) os.unlink( tool_tarball ) tarball_path, filename = os.path.split( tool_tarball ) trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.tgz"' % ( tool_id ) return download_file - else: - status = 'error' - return trans.fill_template( '/admin/package_tool.mako', - tool_id=tool_id, - toolbox=toolbox, - message=message, - status=status ) + except Exception: + return trans.fill_template( '/admin/package_tool.mako', + tool_id=tool_id, + toolbox=toolbox, + message=message, + status='error' ) @web.expose @web.require_admin diff --git a/lib/galaxy/webapps/galaxy/api/configuration.py b/lib/galaxy/webapps/galaxy/api/configuration.py index e4748244989..78ca2554410 100644 --- a/lib/galaxy/webapps/galaxy/api/configuration.py +++ b/lib/galaxy/webapps/galaxy/api/configuration.py @@ -82,6 +82,15 @@ class ConfigurationController( BaseAPIController ): rval.append(entry) return rval + @expose_api + @require_admin + def reload_toolbox(self, trans): + """ + PUT /api/configuration/toolbox + Reload the Galaxy toolbox (but not individual tools). + """ + self.app.reload_toolbox() + def _tool_conf_to_dict(conf): return dict( diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index e4a3112d07e..dfa2febb1e8 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -198,12 +198,11 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): @web.expose_api_raw @web.require_admin def download( self, trans, id, **kwds ): - tool_tarball, success, message = trans.app.toolbox.package_tool( trans, id ) - if success: - trans.response.set_content_type( 'application/x-gzip' ) - download_file = open( tool_tarball ) - trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.tgz"' % ( id ) - return download_file + tool_tarball = trans.app.toolbox.package_tool(trans, id) + trans.response.set_content_type('application/x-gzip') + download_file = open(tool_tarball, "rb") + trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.tgz"' % (id) + return download_file @expose_api_anonymous def create( self, trans, payload, **kwd ): diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 350be892581..4ce1816e69b 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -277,6 +277,12 @@ def populate_api_routes( webapp, app ): controller="configuration", action="tool_lineages" ) + webapp.mapper.connect( + '/api/configuration/toolbox', + controller="configuration", + action="reload_toolbox", + conditions=dict( method=["PUT"] ) + ) webapp.mapper.resource( 'configuration', 'configuration', path_prefix='/api' ) webapp.mapper.connect( "configuration_version", "/api/version", controller="configuration", diff --git a/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py b/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py index 736522d1657..e03e07ffce3 100644 --- a/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py +++ b/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py @@ -74,7 +74,7 @@ class InstalledRepositoryMetadataManager( metadata_generator.MetadataGenerator ) load_relative_path = os.path.join( shed_conf_dict.get( 'tool_path' ), relative_path ) guid = tool_dict.get( 'guid', None ) if relative_path and guid: - tool = self.app.toolbox.load_tool( os.path.abspath( load_relative_path ), guid=guid ) + tool = self.app.toolbox.load_tool( os.path.abspath( load_relative_path ), guid=guid, use_cached=False ) else: tool = None if tool: diff --git a/lib/tool_shed/galaxy_install/tool_migration_manager.py b/lib/tool_shed/galaxy_install/tool_migration_manager.py index 49bcaf08900..64842c07d1f 100644 --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -11,6 +11,7 @@ import logging from galaxy import util from galaxy.tools.toolbox import ToolSection +from galaxy.tools.toolbox.parser import ensure_tool_conf_item from galaxy.util.odict import odict from tool_shed.galaxy_install import install_manager @@ -272,7 +273,7 @@ class ToolMigrationManager( object ): proprietary_tool_config = section_elem.get( 'file' ) if tool_config == proprietary_tool_config: # The tool is loaded inside of the section_elem. - tool_sections.append( ToolSection( proprietary_tool_panel_elem ) ) + tool_sections.append( ToolSection( ensure_tool_conf_item( proprietary_tool_panel_elem ) ) ) if not is_displayed: is_displayed = True return is_displayed, tool_sections @@ -289,7 +290,7 @@ class ToolMigrationManager( object ): filename = basic_util.strip_path( name ) if filename == tool_config_filename: full_path = str( os.path.abspath( os.path.join( root, name ) ) ) - tool = self.toolbox.load_tool( full_path ) + tool = self.toolbox.load_tool( full_path, use_cached=False ) return suc.generate_tool_guid( repository_clone_url, tool ) # Not quite sure what should happen here, throw an exception or what? return None diff --git a/lib/tool_shed/tools/data_table_manager.py b/lib/tool_shed/tools/data_table_manager.py index 5b27fd87ada..ae4b8595685 100644 --- a/lib/tool_shed/tools/data_table_manager.py +++ b/lib/tool_shed/tools/data_table_manager.py @@ -83,7 +83,7 @@ class ToolDataTableManager( object ): # TODO: Do more here than logging an exception. log.debug( message ) # Reload the tool into the local list of repository_tools_tups. - repository_tool = self.app.toolbox.load_tool( os.path.join( tool_path, tup_path ), guid=guid ) + repository_tool = self.app.toolbox.load_tool( os.path.join( tool_path, tup_path ), guid=guid, use_cached=False ) repository_tools_tups[ index ] = ( tup_path, guid, repository_tool ) # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. self.reset_tool_data_tables() diff --git a/lib/tool_shed/tools/tool_validator.py b/lib/tool_shed/tools/tool_validator.py index c9bcf31765d..1502c896de0 100644 --- a/lib/tool_shed/tools/tool_validator.py +++ b/lib/tool_shed/tools/tool_validator.py @@ -292,7 +292,7 @@ class ToolValidator( object ): def load_tool_from_config( self, repository_id, full_path ): try: - tool = self.app.toolbox.load_tool( full_path, repository_id=repository_id, allow_code_files=False ) + tool = self.app.toolbox.load_tool( full_path, repository_id=repository_id, allow_code_files=False, use_cached=False ) valid = True error_message = None except KeyError, e: diff --git a/lib/tool_shed/util/tool_util.py b/lib/tool_shed/util/tool_util.py index 9df2d6a347e..d88d9df14ca 100644 --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -179,7 +179,7 @@ def handle_missing_index_file( app, tool_path, sample_files, repository_tools_tu sample_files_copied.append( options.missing_index_file ) break # Reload the tool into the local list of repository_tools_tups. - repository_tool = app.toolbox.load_tool( os.path.join( tool_path, tup_path ), guid=guid ) + repository_tool = app.toolbox.load_tool( os.path.join( tool_path, tup_path ), guid=guid, use_cached=False ) repository_tools_tups[ index ] = ( tup_path, guid, repository_tool ) return repository_tools_tups, sample_files_copied diff --git a/templates/admin/reload_tool.mako b/templates/admin/reload_tool.mako index c29a233e2e5..76b48a702ae 100644 --- a/templates/admin/reload_tool.mako +++ b/templates/admin/reload_tool.mako @@ -14,6 +14,14 @@ $().ready(function() { %endif focus_el.focus(); }); +$().ready(function() { + $("#reload_toolbox").click(function(){ + $.ajax({ + url: "${h.url_for(controller="/api/configuration", action="toolbox")}", + type: 'PUT' + }); + }); +}); %if message: @@ -54,3 +62,18 @@ $().ready(function() { +

+

+
Reload Toolbox
+
+
+
+ Clicking here will reload + the Galaxy toolbox. This will cause newly discovered tools + to be added, tools now missing from tool confs to be removed, + and items in the panel reordered. Individual tools, even ones + with modified tool descriptions willl not be reloaded. +
+
+
+
diff --git a/test/unit/tools/test_toolbox.py b/test/unit/tools/test_toolbox.py index a29c36246b7..5e784a21172 100644 --- a/test/unit/tools/test_toolbox.py +++ b/test/unit/tools/test_toolbox.py @@ -1,3 +1,4 @@ +import json import os import string import unittest @@ -117,10 +118,14 @@ class BaseToolBoxTestCase( unittest.TestCase, tools_support.UsesApp, tools_supp """ self._add_config( template % (self.test_directory, CONFIG_TEST_TOOL_VERSION_1, CONFIG_TEST_TOOL_VERSION_2 ) ) - def _add_config( self, xml, name="tool_conf.xml" ): + def _add_config( self, content, name="tool_conf.xml" ): + is_json = name.endswith(".json") path = self._tool_conf_path( name=name ) with open( path, "w" ) as f: - f.write( xml ) + if not is_json or isinstance(content, basestring): + f.write( content ) + else: + json.dump(content, f) self.config_files.append( path ) def _tool_conf_path( self, name="tool_conf.xml" ): @@ -146,20 +151,22 @@ class ToolBoxTestCase( BaseToolBoxTestCase ): assert toolbox.get_tool( "not_a_test_tool" ) is None def test_to_dict_in_panel( self ): - self._init_tool_in_section() - mapper = routes.Mapper() - mapper.connect( "tool_runner", "/test/tool_runner" ) - as_dict = self.toolbox.to_dict( mock_trans() ) - test_section = self._find_section(as_dict, "t") - assert len(test_section["elems"]) == 1 - assert test_section["elems"][0]["id"] == "test_tool" + for json_conf in [True, False]: + self._init_tool_in_section(json=json_conf) + mapper = routes.Mapper() + mapper.connect( "tool_runner", "/test/tool_runner" ) + as_dict = self.toolbox.to_dict( mock_trans() ) + test_section = self._find_section(as_dict, "t") + assert len(test_section["elems"]) == 1 + assert test_section["elems"][0]["id"] == "test_tool" def test_to_dict_out_of_panel( self ): - self._init_tool_in_section() - mapper = routes.Mapper() - mapper.connect( "tool_runner", "/test/tool_runner" ) - as_dict = self.toolbox.to_dict( mock_trans(), in_panel=False ) - assert as_dict[0]["id"] == "test_tool" + for json_conf in [True, False]: + self._init_tool_in_section(json=json_conf) + mapper = routes.Mapper() + mapper.connect( "tool_runner", "/test/tool_runner" ) + as_dict = self.toolbox.to_dict( mock_trans(), in_panel=False ) + assert as_dict[0]["id"] == "test_tool" def test_out_of_panel_filtering( self ): self._init_tool_in_section() @@ -183,7 +190,7 @@ class ToolBoxTestCase( BaseToolBoxTestCase ): if elem.get("id") == section_id: assert elem["model_class"] == "ToolSection" return elem - assert False, "Failed to find section with id [%s]" % id + assert False, "Failed to find section with id [%s]" % section_id def test_tool_shed_properties( self ): self._init_tool() @@ -275,6 +282,13 @@ class ToolBoxTestCase( BaseToolBoxTestCase ): toolbox = self.toolbox assert toolbox.get_tool( "test_tool" ) is not None + def test_tool_dir_json( self ): + self._init_tool() + self._add_config({"items": [{"type": "tool_dir", "dir": self.test_directory}]}, name="tool_conf.json") + + toolbox = self.toolbox + assert toolbox.get_tool( "test_tool" ) is not None + def test_workflow_in_panel( self ): stored_workflow = self.__test_workflow() encoded_id = self.app.security.encode_id( stored_workflow.id ) @@ -305,9 +319,19 @@ class ToolBoxTestCase( BaseToolBoxTestCase ): section = self.toolbox._tool_panel[ 'tid' ] self.__check_test_labels( section.elems ) - def _init_tool_in_section( self ): + def _init_tool_in_section( self, json=False ): self._init_tool() - self._add_config( """
""" ) + if not json: + self._add_config( """
""" ) + else: + section = { + "type": "section", + "id": "t", + "name": "test", + "items": [{"type": "tool", + "file": "tool.xml"}], + } + self._add_config({"items": [section]}, name="tool_conf.json") def __check_test_labels( self, panel_dict ): assert panel_dict.keys() == ["label_lab1", "label_lab2"] diff --git a/test/unit/tools/test_watcher.py b/test/unit/tools/test_watcher.py index 3a67ec1dfa4..a745f8f950c 100644 --- a/test/unit/tools/test_watcher.py +++ b/test/unit/tools/test_watcher.py @@ -17,13 +17,43 @@ def test_watcher(): tool_path = path.join(t, "test.xml") toolbox = Toolbox() open(tool_path, "w").write("a") - tool_watcher = watcher.get_watcher(toolbox, bunch.Bunch( + tool_watcher = watcher.get_tool_watcher(toolbox, bunch.Bunch( watch_tools=True )) tool_watcher.watch_file(tool_path, "cool_tool") + assert not toolbox.was_reloaded("cool_tool") open(tool_path, "w").write("b") - time.sleep(2) - toolbox.assert_reloaded("cool_tool") + wait_for_reload(lambda: toolbox.was_reloaded("cool_tool")) + tool_watcher.shutdown() + assert not tool_watcher.observer.is_alive() + + +def test_tool_conf_watcher(): + if not watcher.can_watch: + from nose.plugins.skip import SkipTest + raise SkipTest() + + callback = CallbackRecorder() + conf_watcher = watcher.get_tool_conf_watcher(callback.call) + + with __test_directory() as t: + tool_conf_path = path.join(t, "test_conf.xml") + conf_watcher.watch_file(tool_conf_path) + + open(tool_conf_path, "w").write("b") + wait_for_reload(lambda: callback.called) + conf_watcher.shutdown() + assert not conf_watcher.thread.is_alive() + + +def wait_for_reload(check): + reloaded = False + for i in range(10): + reloaded = check() + if reloaded: + break + time.sleep(.2) + assert reloaded class Toolbox(object): @@ -34,8 +64,8 @@ class Toolbox(object): def reload_tool_by_id( self, tool_id ): self.reloaded[ tool_id ] = True - def assert_reloaded(self, tool_id): - assert self.reloaded.get( tool_id, False ) + def was_reloaded(self, tool_id): + return self.reloaded.get( tool_id, False ) class CallbackRecorder(object):