From 0da66ee8dcf5fef52b11558228c1eb95479cbce9 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 18 Dec 2015 13:02:53 +0000 Subject: [PATCH 01/12] Small improvements to load_directory for downstream extension. --- lib/galaxy/tools/loader_directory.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tools/loader_directory.py b/lib/galaxy/tools/loader_directory.py index 907adcf466e..bfe620e9699 100644 --- a/lib/galaxy/tools/loader_directory.py +++ b/lib/galaxy/tools/loader_directory.py @@ -19,6 +19,8 @@ LOAD_FAILURE_ERROR = "Failed to load tool with path %s." TOOL_LOAD_ERROR = object() TOOL_REGEX = re.compile(r" Date: Fri, 18 Dec 2015 13:04:33 +0000 Subject: [PATCH 02/12] Add CWL suppoort to tool checking and parsing. To be clear, this provides none of the infrastructure for running the tools. This is just so that planemo and future tool registry efforts can have a unified interface for accessing and reasoning about tools using the same methods Galaxy does. --- lib/galaxy/tools/loader_directory.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/lib/galaxy/tools/loader_directory.py b/lib/galaxy/tools/loader_directory.py index bfe620e9699..14c3175fb6f 100644 --- a/lib/galaxy/tools/loader_directory.py +++ b/lib/galaxy/tools/loader_directory.py @@ -20,6 +20,7 @@ TOOL_LOAD_ERROR = object() TOOL_REGEX = re.compile(r" Date: Sun, 20 Dec 2015 18:48:00 +0000 Subject: [PATCH 03/12] Infrastructure for parsing CWL tools. --- lib/galaxy/tools/cwl/__init__.py | 18 ++ lib/galaxy/tools/cwl/cwltool_deps.py | 48 +++ lib/galaxy/tools/cwl/parser.py | 392 ++++++++++++++++++++++++ lib/galaxy/tools/cwl/representation.py | 100 ++++++ lib/galaxy/tools/cwl/runtime_actions.py | 29 ++ lib/galaxy/tools/parser/cwl.py | 145 +++++++++ lib/galaxy/tools/parser/factory.py | 4 + 7 files changed, 736 insertions(+) create mode 100644 lib/galaxy/tools/cwl/__init__.py create mode 100644 lib/galaxy/tools/cwl/cwltool_deps.py create mode 100644 lib/galaxy/tools/cwl/parser.py create mode 100644 lib/galaxy/tools/cwl/representation.py create mode 100644 lib/galaxy/tools/cwl/runtime_actions.py create mode 100644 lib/galaxy/tools/parser/cwl.py diff --git a/lib/galaxy/tools/cwl/__init__.py b/lib/galaxy/tools/cwl/__init__.py new file mode 100644 index 00000000000..9b11c5eb320 --- /dev/null +++ b/lib/galaxy/tools/cwl/__init__.py @@ -0,0 +1,18 @@ +from .parser import tool_proxy +from .runtime_actions import handle_outputs +from .representation import to_cwl_job, to_galaxy_parameters + +from .cwltool_deps import ( + needs_shell_quoting, + shellescape, +) + + +__all__ = [ + 'tool_proxy', + 'handle_outputs', + 'to_cwl_job', + 'to_galaxy_parameters', + 'needs_shell_quoting', + 'shellescape', +] diff --git a/lib/galaxy/tools/cwl/cwltool_deps.py b/lib/galaxy/tools/cwl/cwltool_deps.py new file mode 100644 index 00000000000..fbc886fdd3f --- /dev/null +++ b/lib/galaxy/tools/cwl/cwltool_deps.py @@ -0,0 +1,48 @@ +""" This module contains logic for dealing with cwltool as an optional +dependency for Galaxy and/or applications which use Galaxy as a library. +""" + +try: + import requests +except ImportError: + requests = None + +try: + from cwltool import ( + main, + workflow, + job, + ) +except ImportError: + main = None + workflow = None + job = None + +try: + import shellescape +except ImportError: + shellescape = None + +if job is not None: + needs_shell_quoting = job.needs_shell_quoting +else: + needs_shell_quoting = None + + +def ensure_cwltool_available(): + if main is None or workflow is None or shellescape is None: + message = "This feature requires cwltool and dependencies to be available, they are not." + if requests is None: + message += " Library 'requests' unavailable." + if shellescape is None: + message += " Library 'shellescape' unavailable." + raise ImportError(message) + + +__all__ = [ + 'main', + 'workflow', + 'ensure_cwltool_available', + 'shellescape', + 'needs_shell_quoting', +] diff --git a/lib/galaxy/tools/cwl/parser.py b/lib/galaxy/tools/cwl/parser.py new file mode 100644 index 00000000000..903580f69b4 --- /dev/null +++ b/lib/galaxy/tools/cwl/parser.py @@ -0,0 +1,392 @@ +""" This module provides proxy objects around objects from the common +workflow language reference implementation library cwltool. These proxies +adapt cwltool to Galaxy features and abstract the library away from the rest +of the framework. +""" +from __future__ import absolute_import +from abc import ABCMeta, abstractmethod +from galaxy.util.odict import odict +import json +import os + +from .cwltool_deps import ( + main, + workflow, + ensure_cwltool_available, +) + +from galaxy.util.bunch import Bunch + +JOB_JSON_FILE = ".cwl_job.json" + + +def tool_proxy(tool_path): + """ Provide a proxy object to cwltool data structures to just + grab relevant data. + """ + ensure_cwltool_available() + tool = to_cwl_tool_object(tool_path) + return tool + + +def load_job_proxy(job_directory): + ensure_cwltool_available() + job_objects_path = os.path.join(job_directory, JOB_JSON_FILE) + job_objects = json.load(open(job_objects_path, "r")) + tool_path = job_objects["tool_path"] + job_inputs = job_objects["job_inputs"] + cwl_tool = tool_proxy(tool_path) + cwl_job = cwl_tool.job_proxy(job_inputs, job_directory=job_directory) + return cwl_job + + +def to_cwl_tool_object(tool_path): + if workflow is None: + raise Exception("Using CWL tools requires cwltool module.") + proxy_class = None + cwl_tool = None + make_tool = workflow.defaultMakeTool + cwl_tool = main.load_tool(tool_path, False, False, make_tool, False) + proxy_class = Draft2ToolProxy + if proxy_class is None: + raise Exception("Unsupported CWL object encountered.") + proxy = proxy_class(cwl_tool, tool_path) + return proxy + + +class ToolProxy( object ): + __metaclass__ = ABCMeta + + def __init__(self, tool, tool_path): + self._tool = tool + self._tool_path = tool_path + + def job_proxy(self, input_dict, job_directory="."): + """ Build a cwltool.job.Job describing computation using a input_json + Galaxy will generate mapping the Galaxy description of the inputs into + a cwltool compatible variant. + """ + return JobProxy(self, input_dict, job_directory=job_directory) + + @abstractmethod + def input_instances(self): + """ Return InputInstance objects describing mapping to Galaxy inputs. """ + + @abstractmethod + def output_instances(self): + """ Return OutputInstance objects describing mapping to Galaxy inputs. """ + + @abstractmethod + def docker_identifier(self): + """ Return docker identifier for embedding in tool description. """ + + @abstractmethod + def description(self): + """ Return description to tool. """ + + +class Draft2ToolProxy(ToolProxy): + + def description(self): + # Feels like I should be getting some abstract namespaced thing + # not actually description, is this correct? + return self._tool.tool.get('description') + + def input_instances(self): + return self._find_inputs(self._tool.inputs_record_schema) + + def _find_inputs(self, schema): + schema_type = schema["type"] + if isinstance(schema_type, list): + raise Exception("Union types not yet implemented.") + elif isinstance(schema_type, dict): + return self._find_inputs(schema_type) + else: + if schema_type in self._tool.schemaDefs: + schema = self._tool.schemaDefs[schema_type] + + if schema["type"] == "record": + return map(_simple_field_to_input, schema["fields"]) + + def output_instances(self): + outputs_schema = self._tool.outputs_record_schema + return self._find_outputs(outputs_schema) + + def _find_outputs(self, schema): + rval = [] + if not rval and schema["type"] == "record": + for output in schema["fields"]: + # TODO: Handle non-files differently? Make them + # JSON maybe?. + # output_type = output.get("type", None) + # if output_type != "File": + # template = "Unhandled output type [%s] encountered." + # raise Exception(template % output_type) + rval.append(_simple_field_to_output(output)) + + return rval + + def docker_identifier(self): + tool = self._tool.tool + reqs_and_hints = tool.get("requirements", []) + tool.get("hints", []) + for hint in reqs_and_hints: + if hint["class"] == "DockerRequirement": + if "dockerImageId" in hint: + return hint["dockerImageId"] + else: + return hint["dockerPull"] + return None + + +class JobProxy(object): + + def __init__(self, tool_proxy, input_dict, job_directory): + self._tool_proxy = tool_proxy + self._input_dict = input_dict + self._job_directory = job_directory + + def cwl_job(self): + return self._tool_proxy._tool.job( + self._input_dict, + self._job_directory, + self._output_callback, + use_container=False + ).next() + + def _output_callback(self, out): + pass + + def save_job(self): + job_file = JobProxy._job_file(self._job_directory) + job_objects = { + "tool_path": os.path.abspath(self._tool_proxy._tool_path), + "job_inputs": self._input_dict, + } + json.dump(job_objects, open(job_file, "w")) + + # @staticmethod + # def load_job(tool_proxy, job_directory): + # job_file = JobProxy._job_file(job_directory) + # input_dict = json.load(open(job_file, "r")) + # return JobProxy(tool_proxy, input_dict, job_directory) + + @staticmethod + def _job_file(job_directory): + return os.path.join(job_directory, JOB_JSON_FILE) + + +def _simple_field_union(field): + field_type = _field_to_field_type(field) + non_null_type = None + if len(field_type) == 2: + if "null" == field_type[0]: + non_null_type = field_type[1] + elif "null" == field_type[1]: + non_null_type = field_type[1] + + if non_null_type is None: + raise Exception("General union types not yet implemented (simple).") + + name, label, description = _field_metadata(field) + + case_name = "_cwl__type_" + case_label = "Specify %s" % label + + options = [ + {"value": "null", "label": "No", "selected": True}, + {"value": "%s" % non_null_type, "label": "Yes", "selected": False}, + ] + + case_input = SelectInputInstance( + name=case_name, + label=case_label, + description=False, + options=options, + ) + + non_null_type_kwds = _simple_field_to_input_type_kwds(field, field_type=non_null_type) + non_null_name = "_cwl__value_" + non_null_label = label + non_null_description = description + non_null_input = InputInstance( + non_null_name, + non_null_label, + non_null_description, + **non_null_type_kwds + ) + options = [ + ("null", []), + (non_null_type, [non_null_input]), + ] + return ConditionalInstance(name, case_input, options) + + +def _simple_field_to_input(field): + field_type = _field_to_field_type(field) + if isinstance(field_type, list): + # Length must be greater than 1... + return _simple_field_union(field) + + name, label, description = _field_metadata(field) + + type_kwds = _simple_field_to_input_type_kwds(field) + return InputInstance(name, label, description, **type_kwds) + + +def _simple_field_to_input_type_kwds(field, field_type=None): + simple_map_type_map = { + "File": INPUT_TYPE.DATA, + "int": INPUT_TYPE.INTEGER, + "string": INPUT_TYPE.TEXT, + "boolean": INPUT_TYPE.BOOLEAN, + } + + if field_type is None: + field_type = _field_to_field_type(field) + + if field_type in simple_map_type_map.keys(): + input_type = simple_map_type_map[field_type] + return {"input_type": input_type, "array": False} + elif field_type == "array": + if isinstance(field["type"], dict): + array_type = field["type"]["items"] + else: + array_type = field["items"] + if array_type in simple_map_type_map.keys(): + input_type = simple_map_type_map[array_type] + return {"input_type": input_type, "array": True} + else: + raise Exception("Unhandled simple field type encountered - [%s]." % field_type) + + +def _field_to_field_type(field): + field_type = field["type"] + if isinstance(field_type, dict): + field_type = field_type["type"] + if isinstance(field_type, list): + field_type_length = len(field_type) + if field_type_length == 0: + raise Exception("Zero-length type list encountered, invalid CWL?") + elif len(field_type) == 1: + field_type = field_type[0] + + return field_type + + +def _field_metadata(field): + name = field["name"] + label = field.get("label", None) + description = field.get("description", None) + return name, label, description + + +def _simple_field_to_output(field): + name = field["name"] + output_instance = OutputInstance(name, output_type=OUTPUT_TYPE.GLOB) + return output_instance + + +INPUT_TYPE = Bunch( + DATA="data", + INTEGER="integer", + TEXT="text", + BOOLEAN="boolean", + SELECT="select", + CONDITIONAL="conditional", +) + + +class ConditionalInstance(object): + + def __init__(self, name, case, whens): + self.input_type = INPUT_TYPE.CONDITIONAL + self.name = name + self.case = case + self.whens = whens + + def to_dict(self): + + as_dict = dict( + name=self.name, + type=INPUT_TYPE.CONDITIONAL, + test=self.case.to_dict(), + when=odict(), + ) + for value, block in self.whens: + as_dict["when"][value] = map(lambda i: i.to_dict(), block) + + return as_dict + + +class SelectInputInstance(object): + + def __init__(self, name, label, description, options): + self.input_type = INPUT_TYPE.SELECT + self.name = name + self.label = label + self.description = description + self.options = options + + def to_dict(self): + # TODO: serialize options... + as_dict = dict( + name=self.name, + label=self.label or self.name, + help=self.description, + type=self.input_type, + options=self.options, + ) + return as_dict + + +class InputInstance(object): + + def __init__(self, name, label, description, input_type, array=False): + self.input_type = input_type + self.name = name + self.label = label + self.description = description + self.required = True + self.array = array + + def to_dict(self, itemwise=True): + if itemwise and self.array: + as_dict = dict( + type="repeat", + name="%s_repeat" % self.name, + title="%s" % self.name, + blocks=[ + self.to_dict(itemwise=False) + ] + ) + else: + as_dict = dict( + name=self.name, + label=self.label or self.name, + help=self.description, + type=self.input_type, + optional=not self.required, + ) + if self.input_type == INPUT_TYPE.INTEGER: + as_dict["value"] = "0" + return as_dict + + +OUTPUT_TYPE = Bunch( + GLOB="glob", + STDOUT="stdout", +) + + +class OutputInstance(object): + + def __init__(self, name, output_type, path=None): + self.name = name + self.output_type = output_type + self.path = path + + +__all__ = [ + 'tool_proxy', + 'load_job_proxy', +] diff --git a/lib/galaxy/tools/cwl/representation.py b/lib/galaxy/tools/cwl/representation.py new file mode 100644 index 00000000000..337cb5f6c9f --- /dev/null +++ b/lib/galaxy/tools/cwl/representation.py @@ -0,0 +1,100 @@ +""" This module is responsible for converting between Galaxy's tool +input description and the CWL description for a job json. """ + +import logging + +from galaxy.util import string_as_bool + +log = logging.getLogger(__name__) + +NOT_PRESENT = object() + + +def to_cwl_job(tool, param_dict): + """ tool is Galaxy's representation of the tool and param_dict is the + parameter dictionary with wrapped values. + """ + inputs = tool.inputs + input_json = {} + + def simple_value(input, param_dict_value): + if input.type == "data": + return {"path": str(param_dict_value), "class": "File"} + elif input.type == "integer": + return int(str(param_dict_value)) + elif input.type == "boolean": + return string_as_bool(param_dict_value) + else: + return str(param_dict_value) + + for input_name, input in inputs.iteritems(): + if input.type == "repeat": + only_input = input.inputs.values()[0] + array_value = [] + for instance in param_dict[input_name]: + array_value.append(simple_value(only_input, instance[input_name[:-len("_repeat")]])) + input_json[input_name[:-len("_repeat")]] = array_value + elif input.type == "conditional": + assert input_name in param_dict, "No value for %s in %s" % (input_name, param_dict) + current_case = param_dict[input_name]["_cwl__type_"] + if current_case != "null": + # TODO: make sure trans is not needed here... + case_index = input.get_current_case( current_case, None ) + case_input = input.cases[ case_index ].inputs[ "_cwl__value_"] + case_value = param_dict[input_name]["_cwl__value_"] + input_json[input_name] = simple_value(case_input, case_value) + else: + input_json[input_name] = simple_value(input, param_dict[input_name]) + + input_json["allocatedResources"] = { + "cpu": "$GALAXY_SLOTS", + } + return input_json + + +def to_galaxy_parameters(tool, as_dict): + """ Tool is Galaxy's representation of the tool and as_dict is a Galaxified + representation of the input json (no paths, HDA references for instance). + """ + inputs = tool.inputs + galaxy_request = {} + + def from_simple_value(input, param_dict_value): + return param_dict_value + + for input_name, input in inputs.iteritems(): + as_dict_value = as_dict.get(input_name, NOT_PRESENT) + + if input.type == "repeat": + if input_name not in as_dict: + continue + + only_input = input.inputs.values()[0] + for index, value in enumerate(as_dict_value): + key = "%s_repeat_0|%s" % (input_name, only_input.name) + galaxy_value = from_simple_value(only_input, value) + galaxy_request[key] = galaxy_value + elif input.type == "conditional": + # TODO: less crazy handling of defaults... + if as_dict_value is NOT_PRESENT: + cwl_type = "null" + elif isinstance(as_dict_value, bool): + cwl_type = "boolean" + elif isinstance(as_dict_value, int): + cwl_type = "integer" + else: + cwl_type = "string" + galaxy_request["%s|_cwl__type_" % input_name] = cwl_type + if cwl_type != "null": + current_case_index = input.get_current_case(cwl_type, None) + current_case_input = input.cases[ current_case_index ].inputs[ "_cwl__value_" ] + galaxy_value = from_simple_value(current_case_input, as_dict_value) + galaxy_request["%s|_cwl__value_" % input_name] = galaxy_value + elif as_dict_value is NOT_PRESENT: + continue + else: + galaxy_value = from_simple_value(input, as_dict_value) + galaxy_request[input_name] = galaxy_value + + log.info("Converted galaxy_request is %s" % galaxy_request) + return galaxy_request diff --git a/lib/galaxy/tools/cwl/runtime_actions.py b/lib/galaxy/tools/cwl/runtime_actions.py new file mode 100644 index 00000000000..cd4b2a0a1cf --- /dev/null +++ b/lib/galaxy/tools/cwl/runtime_actions.py @@ -0,0 +1,29 @@ +import os +import shutil + +from .parser import ( + JOB_JSON_FILE, + load_job_proxy, +) + + +def handle_outputs(job_directory=None): + # Relocate dynamically collected files to pre-determined locations + # registered with ToolOutput objects via from_work_dir handling. + if job_directory is None: + job_directory = os.getcwd() + cwl_job_file = os.path.join(job_directory, JOB_JSON_FILE) + if not os.path.exists(cwl_job_file): + # Not a CWL job, just continue + return + job_proxy = load_job_proxy(job_directory) + cwl_job = job_proxy.cwl_job() + outputs = cwl_job.collect_outputs(job_directory) + for output_name, output in outputs.iteritems(): + target_path = os.path.join(job_directory, "__cwl_output_%s" % output_name) + shutil.move(output["path"], target_path) + + +__all__ = [ + 'handle_outputs', +] diff --git a/lib/galaxy/tools/parser/cwl.py b/lib/galaxy/tools/parser/cwl.py new file mode 100644 index 00000000000..f19007c71f1 --- /dev/null +++ b/lib/galaxy/tools/parser/cwl.py @@ -0,0 +1,145 @@ +import logging +import os + +from .interface import ToolSource +from .interface import PagesSource +from .interface import PageSource +from .interface import ToolStdioExitCode +from .yaml import YamlInputSource + +from .output_actions import ToolOutputActionGroup +from .output_objects import ToolOutput + +from galaxy.tools.deps import requirements +from galaxy.tools.cwl import tool_proxy + +from galaxy.util.odict import odict + +log = logging.getLogger(__name__) + + +class CwlToolSource(ToolSource): + + def __init__(self, tool_file): + self._cwl_tool_file = tool_file + self._id, _ = os.path.splitext(os.path.basename(tool_file)) + self._tool_proxy = tool_proxy(tool_file) + + @property + def tool_proxy(self): + return self._tool_proxy + + def parse_tool_type(self): + return 'cwl' + + def parse_id(self): + log.warn("TOOL ID is %s" % self._id) + return self._id + + def parse_name(self): + return self._id + + def parse_command(self): + return "$__cwl_command" + + def parse_environment_variables(self): + environment_variables = [] + # TODO: Is this even possible from here, should instead this be moved + # into the job. + + # for environment_variable_el in environment_variables_el.findall("environment_variable"): + # definition = { + # "name": environment_variable_el.get("name"), + # "template": environment_variable_el.text, + # } + # environment_variables.append( + # definition + # ) + + return environment_variables + + def parse_help(self): + return "" + + def parse_stdio(self): + # TODO: remove duplication with YAML + from galaxy.jobs.error_level import StdioErrorLevel + + # New format - starting out just using exit code. + exit_code_lower = ToolStdioExitCode() + exit_code_lower.range_start = float("-inf") + exit_code_lower.range_end = -1 + exit_code_lower.error_level = StdioErrorLevel.FATAL + exit_code_high = ToolStdioExitCode() + exit_code_high.range_start = 1 + exit_code_high.range_end = float("inf") + exit_code_lower.error_level = StdioErrorLevel.FATAL + return [exit_code_lower, exit_code_high], [] + + def parse_interpreter(self): + return None + + def parse_version(self): + return "0.0.1" + + def parse_description(self): + return self._tool_proxy.description() or "" + + def parse_input_pages(self): + page_source = CwlPageSource(self._tool_proxy) + return PagesSource([page_source]) + + def parse_outputs(self, tool): + output_instances = self._tool_proxy.output_instances() + outputs = odict() + output_defs = [] + for output_instance in output_instances: + output_defs.append(self._parse_output(tool, output_instance)) + # TODO: parse outputs collections + for output_def in output_defs: + outputs[output_def.name] = output_def + return outputs, odict() + + def _parse_output(self, tool, output_instance): + name = output_instance.name + # TODO: handle filters, actions, change_format + output = ToolOutput( name ) + output.format = "_sniff_" + output.change_format = [] + output.format_source = None + output.metadata_source = "" + output.parent = None + output.label = None + output.count = None + output.filters = [] + output.tool = tool + output.from_work_dir = "__cwl_output_%s" % name + output.hidden = "" + output.dataset_collectors = [] + output.actions = ToolOutputActionGroup( output, None ) + return output + + def parse_requirements_and_containers(self): + containers = [] + docker_identifier = self._tool_proxy.docker_identifier() + if docker_identifier: + containers.append({"type": "docker", + "identifier": docker_identifier}) + return requirements.parse_requirements_from_dict(dict( + requirements=[], # TODO: enable via extensions + containers=containers, + )) + + +class CwlPageSource(PageSource): + + def __init__(self, tool_proxy): + cwl_instances = tool_proxy.input_instances() + self._input_list = map(self._to_input_source, cwl_instances) + + def _to_input_source(self, input_instance): + as_dict = input_instance.to_dict() + return YamlInputSource(as_dict) + + def parse_input_sources(self): + return self._input_list diff --git a/lib/galaxy/tools/parser/factory.py b/lib/galaxy/tools/parser/factory.py index 17661defb50..e24dda1144a 100644 --- a/lib/galaxy/tools/parser/factory.py +++ b/lib/galaxy/tools/parser/factory.py @@ -5,6 +5,7 @@ import yaml from .yaml import YamlToolSource from .xml import XmlToolSource from .xml import XmlInputSource +from .cwl import CwlToolSource from .interface import InputSource @@ -27,6 +28,9 @@ def get_tool_source(config_file, enable_beta_formats=True): with open(config_file, "r") as f: as_dict = ordered_load(f) return YamlToolSource(as_dict) + elif config_file.endswith(".json") or config_file.endswith(".cwl"): + log.info("Loading CWL tool - this is experimental - tool likely will not function in future at least in same way.") + return CwlToolSource(config_file) else: tree = load_tool_xml(config_file) root = tree.getroot() From db845fa1875bcb08aa2129830f1d405d1d894cfb Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 18 Dec 2015 15:30:21 +0000 Subject: [PATCH 04/12] Refactor loader_directory to support beta formats in other ways. --- lib/galaxy/tools/loader_directory.py | 61 ++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/tools/loader_directory.py b/lib/galaxy/tools/loader_directory.py index 14c3175fb6f..57d387e5681 100644 --- a/lib/galaxy/tools/loader_directory.py +++ b/lib/galaxy/tools/loader_directory.py @@ -27,6 +27,28 @@ def load_exception_handler(path, exc_info): log.warn(LOAD_FAILURE_ERROR % path, exc_info=exc_info) +def find_possible_tools_from_path( + path, + recursive=False, + enable_beta_formats=False, +): + possible_tool_files = [] + for possible_tool_file in __find_tool_files( + path, recursive=recursive, + enable_beta_formats=enable_beta_formats + ): + try: + does_look_like_a_tool = looks_like_a_tool(possible_tool_file) + except IOError: + # Some problem reading the tool file, skip. + continue + + if does_look_like_a_tool: + possible_tool_files.append(possible_tool_file) + + return possible_tool_files + + def load_tool_elements_from_path( path, load_exception_handler=load_exception_handler, @@ -34,21 +56,18 @@ def load_tool_elements_from_path( register_load_errors=False, ): tool_elements = [] - for file in __find_tool_files(path, recursive=recursive): + for possible_tool_file in find_possible_tools_from_path( + path, + recursive=recursive, + enable_beta_formats=False, + ): try: - does_look_like_a_tool = looks_like_a_tool(file) - except IOError: - # Some problem reading the tool file, skip. - continue - - if does_look_like_a_tool: - try: - tool_elements.append((file, loader.load_tool(file))) - except Exception: - exc_info = sys.exc_info() - load_exception_handler(file, exc_info) - if register_load_errors: - tool_elements.append((file, TOOL_LOAD_ERROR)) + tool_elements.append((file, loader.load_tool(file))) + except Exception: + exc_info = sys.exc_info() + load_exception_handler(file, exc_info) + if register_load_errors: + tool_elements.append((file, TOOL_LOAD_ERROR)) return tool_elements @@ -141,7 +160,7 @@ def looks_like_a_tool_cwl(path): return file_class == "CommandLineTool" and file_cwl_version -def __find_tool_files(path, recursive): +def __find_tool_files(path, recursive, enable_beta_formats): is_file = not os.path.isdir(path) if not os.path.exists(path): raise Exception(PATH_DOES_NOT_EXIST_ERROR) @@ -150,10 +169,16 @@ def __find_tool_files(path, recursive): elif is_file: return [os.path.abspath(path)] else: - if not recursive: - files = glob.glob(path + "/*.xml") + if enable_beta_formats: + if not recursive: + files = glob.glob(path + "/*") + else: + files = _find_files(path, "*") else: - files = _find_files(path, "*.xml") + if not recursive: + files = glob.glob(path + "/*.xml") + else: + files = _find_files(path, "*.xml") return map(os.path.abspath, files) From fcc83d4e09d4603c5fbd831e9e1933916bf00827 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 21 Dec 2015 20:02:31 +0000 Subject: [PATCH 05/12] Introduce ToolBox parser. Abstract simpler ToolBox interactions to separate XML representation from handling. Doesn't cover tool shed operations since the tool shed explicitly writes XML. --- lib/galaxy/tools/toolbox/base.py | 83 ++++++++++++++------------- lib/galaxy/tools/toolbox/panel.py | 21 ++++--- lib/galaxy/tools/toolbox/parser.py | 90 ++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+), 49 deletions(-) create mode 100644 lib/galaxy/tools/toolbox/parser.py diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index 402dfe9de62..c95b7805776 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -17,6 +17,7 @@ from galaxy.util import parse_xml from galaxy.util import string_as_bool from galaxy.util.bunch import Bunch +from .parser import get_toolbox_parser, ensure_tool_conf_item from .panel import ToolPanelElements from .panel import ToolSectionLabel from .panel import ToolSection @@ -121,9 +122,8 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): """ log.info( "Parsing the tool configuration %s" % config_filename ) - tree = parse_xml( config_filename ) - root = tree.getroot() - tool_path = root.get( 'tool_path' ) + tool_conf_source = get_toolbox_parser(config_filename) + tool_path = tool_conf_source.parse_tool_path() if tool_path: # We're parsing a shed_tool_conf file since we have a tool_path attribute. parsing_shed_tool_conf = True @@ -134,16 +134,16 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): tool_path = self.__resolve_tool_path(tool_path, config_filename) # Only load the panel_dict under certain conditions. load_panel_dict = not self._integrated_tool_panel_config_has_contents - for _, elem in enumerate( root ): + for item in tool_conf_source.parse_items(): index = self._index self._index += 1 if parsing_shed_tool_conf: - config_elems.append( elem ) + config_elems.append( item.elem ) self.load_item( - elem, + item, tool_path=tool_path, load_panel_dict=load_panel_dict, - guid=elem.get( 'guid' ), + guid=item.get( 'guid' ), index=index, internal=True ) @@ -154,8 +154,9 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): config_elems=config_elems ) self._dynamic_tool_confs.append( shed_tool_conf_dict ) - def load_item( self, elem, tool_path, panel_dict=None, integrated_panel_dict=None, load_panel_dict=True, guid=None, index=None, internal=False ): - item_type = elem.tag + def load_item( self, item, tool_path, panel_dict=None, integrated_panel_dict=None, load_panel_dict=True, guid=None, index=None, internal=False ): + item = ensure_tool_conf_item(item) + item_type = item.type if item_type not in ['tool', 'section'] and not internal: # External calls from tool shed code cannot load labels or tool # directories. @@ -166,15 +167,15 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): if integrated_panel_dict is None: integrated_panel_dict = self._integrated_tool_panel if item_type == 'tool': - self._load_tool_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, tool_path=tool_path, load_panel_dict=load_panel_dict, guid=guid, index=index ) + self._load_tool_tag_set( item, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, tool_path=tool_path, load_panel_dict=load_panel_dict, guid=guid, index=index ) elif item_type == 'workflow': - self._load_workflow_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) + self._load_workflow_tag_set( item, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) elif item_type == 'section': - self._load_section_tag_set( elem, tool_path=tool_path, load_panel_dict=load_panel_dict, index=index ) + self._load_section_tag_set( item, tool_path=tool_path, load_panel_dict=load_panel_dict, index=index ) elif item_type == 'label': - self._load_label_tag_set( elem, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) + self._load_label_tag_set( item, panel_dict=panel_dict, integrated_panel_dict=integrated_panel_dict, load_panel_dict=load_panel_dict, index=index ) elif item_type == 'tool_dir': - self._load_tooldir_tag_set( elem, panel_dict, tool_path, integrated_panel_dict, load_panel_dict=load_panel_dict ) + self._load_tooldir_tag_set( item, panel_dict, tool_path, integrated_panel_dict, load_panel_dict=load_panel_dict ) def get_shed_config_dict_by_filename( self, filename, default=None ): for shed_config_dict in self._dynamic_tool_confs: @@ -533,9 +534,9 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): else: remove_from_dict( self._tool_panel, self._integrated_tool_panel ) - def _load_tool_tag_set( self, elem, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None, index=None ): + def _load_tool_tag_set( self, item, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None, index=None ): try: - path = elem.get( "file" ) + path = item.get( "file" ) repository_id = None tool_shed_repository = None @@ -543,13 +544,13 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): if guid is not None: # The tool is contained in an installed tool shed repository, so load # the tool only if the repository has not been marked deleted. - tool_shed = elem.find( "tool_shed" ).text - repository_name = elem.find( "repository_name" ).text - repository_owner = elem.find( "repository_owner" ).text - installed_changeset_revision_elem = elem.find( "installed_changeset_revision" ) + tool_shed = item.elem.find( "tool_shed" ).text + repository_name = item.elem.find( "repository_name" ).text + repository_owner = item.elem.find( "repository_owner" ).text + installed_changeset_revision_elem = item.elem.find( "installed_changeset_revision" ) if installed_changeset_revision_elem is None: # Backward compatibility issue - the tag used to be named 'changeset_revision'. - installed_changeset_revision_elem = elem.find( "changeset_revision" ) + installed_changeset_revision_elem = item.elem.find( "changeset_revision" ) installed_changeset_revision = installed_changeset_revision_elem.text tool_shed_repository = self._get_tool_shed_repository( tool_shed, repository_name, @@ -563,7 +564,7 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): # Else there is not yet a tool_shed_repository record, we're in the process of installing # a new repository, so any included tools can be loaded into the tool panel. tool = self.load_tool( os.path.join( tool_path, path ), guid=guid, repository_id=repository_id ) - if string_as_bool(elem.get( 'hidden', False )): + if string_as_bool(item.get( 'hidden', False )): tool.hidden = True key = 'tool_%s' % str( tool.id ) if can_load_into_panel_dict: @@ -573,19 +574,21 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): tool.repository_owner = repository_owner tool.installed_changeset_revision = installed_changeset_revision tool.guid = guid - tool.version = elem.find( "version" ).text + tool.version = item.elem.find( "version" ).text # Make sure the tool has a tool_version. tool_lineage = self._lineage_map.register( tool, tool_shed_repository=tool_shed_repository ) # Load the tool's lineage ids. tool.lineage = tool_lineage - self._tool_tag_manager.handle_tags( tool.id, elem ) + if item.has_elem: + self._tool_tag_manager.handle_tags( tool.id, item.elem ) self.__add_tool( tool, load_panel_dict, panel_dict ) # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file. integrated_panel_dict.update_or_append( index, key, tool ) # If labels were specified in the toolbox config, attach them to # the tool. - if "labels" in elem.attrib: - tool.labels = [ label.strip() for label in elem.attrib["labels"].split( "," ) ] + labels = item.labels + if labels is not None: + tool.labels = labels except IOError: log.error( "Error reading tool configuration file from path: %s." % path ) except Exception: @@ -609,10 +612,10 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): if load_panel_dict: self.__add_tool_to_tool_panel( tool, panel_dict, section=isinstance( panel_dict, ToolSection ) ) - def _load_workflow_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): + def _load_workflow_tag_set( self, item, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): try: # TODO: should id be encoded? - workflow_id = elem.get( 'id' ) + workflow_id = item.get( 'id' ) workflow = self._load_workflow( workflow_id ) self._workflows_by_id[ workflow_id ] = workflow key = 'workflow_' + workflow_id @@ -623,35 +626,35 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): except: log.exception( "Error loading workflow: %s" % workflow_id ) - def _load_label_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): - label = ToolSectionLabel( elem ) + def _load_label_tag_set( self, item, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): + label = ToolSectionLabel( item ) key = 'label_' + label.id if load_panel_dict: panel_dict[ key ] = label integrated_panel_dict.update_or_append( index, key, label ) - def _load_section_tag_set( self, elem, tool_path, load_panel_dict, index=None ): - key = elem.get( "id" ) + def _load_section_tag_set( self, item, tool_path, load_panel_dict, index=None ): + key = item.get( "id" ) if key in self._tool_panel: section = self._tool_panel[ key ] elems = section.elems else: - section = ToolSection( elem ) + section = ToolSection( item ) elems = section.elems if key in self._integrated_tool_panel: integrated_section = self._integrated_tool_panel[ key ] integrated_elems = integrated_section.elems else: - integrated_section = ToolSection( elem ) + integrated_section = ToolSection( item ) integrated_elems = integrated_section.elems - for sub_index, sub_elem in enumerate( elem ): + for sub_index, sub_item in enumerate( item.items ): self.load_item( - sub_elem, + sub_item, tool_path=tool_path, panel_dict=elems, integrated_panel_dict=integrated_elems, load_panel_dict=load_panel_dict, - guid=sub_elem.get( 'guid' ), + guid=sub_item.get( 'guid' ), index=sub_index, internal=True, ) @@ -668,9 +671,9 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): # Always load sections into the integrated_tool_panel. self._integrated_tool_panel.update_or_append( index, key, integrated_section ) - def _load_tooldir_tag_set(self, sub_elem, elems, tool_path, integrated_elems, load_panel_dict): - directory = os.path.join( tool_path, sub_elem.attrib.get("dir") ) - recursive = string_as_bool( sub_elem.attrib.get("recursive", True) ) + def _load_tooldir_tag_set(self, item, elems, tool_path, integrated_elems, load_panel_dict): + directory = os.path.join( tool_path, item.get("dir") ) + recursive = string_as_bool( item.get("recursive", True) ) self.__watch_directory( directory, elems, integrated_elems, load_panel_dict, recursive, force_watch=True ) def __watch_directory( self, directory, elems, integrated_elems, load_panel_dict, recursive, force_watch=False ): diff --git a/lib/galaxy/tools/toolbox/panel.py b/lib/galaxy/tools/toolbox/panel.py index 9e77231aceb..bb1d6bbe3c5 100644 --- a/lib/galaxy/tools/toolbox/panel.py +++ b/lib/galaxy/tools/toolbox/panel.py @@ -3,6 +3,7 @@ from abc import abstractmethod from galaxy.util.odict import odict from galaxy.util import bunch from galaxy.util.dictifiable import Dictifiable +from .parser import ensure_tool_conf_item from six import iteritems @@ -49,13 +50,14 @@ class ToolSection( Dictifiable, HasPanelItems, object ): dict_collection_visible_keys = ( 'id', 'name', 'version' ) - def __init__( self, elem=None ): + def __init__( self, item=None ): """ Build a ToolSection from an ElementTree element or a dictionary. """ - f = lambda elem, val: elem is not None and elem.get( val ) or '' - self.name = f( elem, 'name' ) - self.id = f( elem, 'id' ) - self.version = f( elem, 'version' ) + item = ensure_tool_conf_item(item) + f = lambda item, val: item is not None and item.get( val ) or '' + self.name = f( item, 'name' ) + self.id = f( item, 'id' ) + self.version = f( item, 'version' ) self.elems = ToolPanelElements() def copy( self ): @@ -93,13 +95,14 @@ class ToolSectionLabel( Dictifiable, object ): dict_collection_visible_keys = ( 'id', 'text', 'version' ) - def __init__( self, elem ): + def __init__( self, item ): """ Build a ToolSectionLabel from an ElementTree element or a dictionary. """ - self.text = elem.get( "text" ) - self.id = elem.get( "id" ) - self.version = elem.get( "version" ) or '' + item = ensure_tool_conf_item(item) + self.text = item.get( "text" ) + self.id = item.get( "id" ) + self.version = item.get( "version" ) or '' def to_dict( self, **kwds ): return super( ToolSectionLabel, self ).to_dict() diff --git a/lib/galaxy/tools/toolbox/parser.py b/lib/galaxy/tools/toolbox/parser.py new file mode 100644 index 00000000000..c6a5ad6dd15 --- /dev/null +++ b/lib/galaxy/tools/toolbox/parser.py @@ -0,0 +1,90 @@ +from abc import ABCMeta +from abc import abstractmethod + +from galaxy.util import parse_xml + + +class ToolConfSource(object): + """ This interface represents an abstract source to parse tool + information from. + """ + __metaclass__ = ABCMeta + + @abstractmethod + def parse_items(self): + """ Return a list of ToolConfItem + """ + + @abstractmethod + def parse_tool_path(self): + """ Return tool_path for tools in this toolbox. + """ + + +class XmlToolConfSource(ToolConfSource): + + def __init__(self, config_filename): + tree = parse_xml(config_filename) + self.root = tree.getroot() + + def parse_tool_path(self): + return self.root.get('tool_path') + + def parse_items(self): + return map(ensure_tool_conf_item, self.root.getchildren()) + + +class ToolConfItem(object): + """ This interface represents an abstract source to parse tool + information from. + """ + + def __init__(self, type, attributes, elem=None): + self.type = type + self.attributes = attributes + self._elem = elem + + def get(self, key, default=None): + return self.attributes.get(key, default) + + @property + def has_elem(self): + return self._elem is not None + + @property + def elem(self): + if self._elem is None: + raise Exception("item.elem called on toolbox element from non-XML source") + return self._elem + + @property + def labels(self): + labels = None + if "labels" in self.attributes: + labels = [ label.strip() for label in self.attributes["labels"].split( "," ) ] + return labels + + +class ToolConfSection(ToolConfItem): + + def __init__(self, attributes, items, elem=None): + super(ToolConfSection, self).__init__('section', attributes, elem) + self.items = items + + +def ensure_tool_conf_item(xml_or_item): + if isinstance(xml_or_item, ToolConfItem): + return xml_or_item + else: + elem = xml_or_item + type = elem.tag + attributes = elem.attrib + if type != "section": + return ToolConfItem(type, attributes, elem) + else: + items = map(ensure_tool_conf_item, elem.getchildren()) + return ToolConfSection(attributes, items, elem=elem) + + +def get_toolbox_parser(config_filename): + return XmlToolConfSource(config_filename) From b68991c4caa44aefeaba752748ae62db98a446fd Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 21 Dec 2015 20:28:36 +0000 Subject: [PATCH 06/12] Allow loading tool confs written in YAML and JSON. Because lets say someone was writing a programatic way to manage elements in a tool conf, that hypothetical person probably shouldn't be writing XML. --- lib/galaxy/tools/toolbox/panel.py | 1 - lib/galaxy/tools/toolbox/parser.py | 39 ++++++++++++++++++-- test/unit/tools/test_toolbox.py | 58 +++++++++++++++++++++--------- 3 files changed, 78 insertions(+), 20 deletions(-) diff --git a/lib/galaxy/tools/toolbox/panel.py b/lib/galaxy/tools/toolbox/panel.py index bb1d6bbe3c5..5dd77e071c4 100644 --- a/lib/galaxy/tools/toolbox/panel.py +++ b/lib/galaxy/tools/toolbox/panel.py @@ -53,7 +53,6 @@ class ToolSection( Dictifiable, HasPanelItems, object ): def __init__( self, item=None ): """ Build a ToolSection from an ElementTree element or a dictionary. """ - item = ensure_tool_conf_item(item) f = lambda item, val: item is not None and item.get( val ) or '' self.name = f( item, 'name' ) self.id = f( item, 'id' ) diff --git a/lib/galaxy/tools/toolbox/parser.py b/lib/galaxy/tools/toolbox/parser.py index c6a5ad6dd15..6a075095972 100644 --- a/lib/galaxy/tools/toolbox/parser.py +++ b/lib/galaxy/tools/toolbox/parser.py @@ -2,6 +2,7 @@ from abc import ABCMeta from abc import abstractmethod from galaxy.util import parse_xml +import yaml class ToolConfSource(object): @@ -34,6 +35,20 @@ class XmlToolConfSource(ToolConfSource): return map(ensure_tool_conf_item, self.root.getchildren()) +class YamlToolConfSource(ToolConfSource): + + def __init__(self, config_filename): + with open(config_filename, "r") as f: + as_dict = yaml.load(f) + self.as_dict = as_dict + + def parse_tool_path(self): + return self.as_dict.get('tool_path') + + def parse_items(self): + return map(ToolConfItem.from_dict, self.as_dict.get('items')) + + class ToolConfItem(object): """ This interface represents an abstract source to parse tool information from. @@ -44,6 +59,20 @@ class ToolConfItem(object): self.attributes = attributes self._elem = elem + @classmethod + def from_dict(cls, _as_dict): + as_dict = _as_dict.copy() + type = as_dict.get('type') + del as_dict['type'] + attributes = as_dict + if type == 'section': + items = map(cls.from_dict, as_dict['items']) + del as_dict['items'] + item = ToolConfSection(attributes, items) + else: + item = ToolConfItem(type, attributes) + return item + def get(self, key, default=None): return self.attributes.get(key, default) @@ -73,7 +102,9 @@ class ToolConfSection(ToolConfItem): def ensure_tool_conf_item(xml_or_item): - if isinstance(xml_or_item, ToolConfItem): + if xml_or_item is None: + return None + elif isinstance(xml_or_item, ToolConfItem): return xml_or_item else: elem = xml_or_item @@ -87,4 +118,8 @@ def ensure_tool_conf_item(xml_or_item): def get_toolbox_parser(config_filename): - return XmlToolConfSource(config_filename) + is_yaml = any(map(lambda e: config_filename.endswith(e), [".yml", ".yaml", ".json"])) + if is_yaml: + return YamlToolConfSource(config_filename) + else: + return XmlToolConfSource(config_filename) diff --git a/test/unit/tools/test_toolbox.py b/test/unit/tools/test_toolbox.py index a29c36246b7..5e784a21172 100644 --- a/test/unit/tools/test_toolbox.py +++ b/test/unit/tools/test_toolbox.py @@ -1,3 +1,4 @@ +import json import os import string import unittest @@ -117,10 +118,14 @@ class BaseToolBoxTestCase( unittest.TestCase, tools_support.UsesApp, tools_supp """ self._add_config( template % (self.test_directory, CONFIG_TEST_TOOL_VERSION_1, CONFIG_TEST_TOOL_VERSION_2 ) ) - def _add_config( self, xml, name="tool_conf.xml" ): + def _add_config( self, content, name="tool_conf.xml" ): + is_json = name.endswith(".json") path = self._tool_conf_path( name=name ) with open( path, "w" ) as f: - f.write( xml ) + if not is_json or isinstance(content, basestring): + f.write( content ) + else: + json.dump(content, f) self.config_files.append( path ) def _tool_conf_path( self, name="tool_conf.xml" ): @@ -146,20 +151,22 @@ class ToolBoxTestCase( BaseToolBoxTestCase ): assert toolbox.get_tool( "not_a_test_tool" ) is None def test_to_dict_in_panel( self ): - self._init_tool_in_section() - mapper = routes.Mapper() - mapper.connect( "tool_runner", "/test/tool_runner" ) - as_dict = self.toolbox.to_dict( mock_trans() ) - test_section = self._find_section(as_dict, "t") - assert len(test_section["elems"]) == 1 - assert test_section["elems"][0]["id"] == "test_tool" + for json_conf in [True, False]: + self._init_tool_in_section(json=json_conf) + mapper = routes.Mapper() + mapper.connect( "tool_runner", "/test/tool_runner" ) + as_dict = self.toolbox.to_dict( mock_trans() ) + test_section = self._find_section(as_dict, "t") + assert len(test_section["elems"]) == 1 + assert test_section["elems"][0]["id"] == "test_tool" def test_to_dict_out_of_panel( self ): - self._init_tool_in_section() - mapper = routes.Mapper() - mapper.connect( "tool_runner", "/test/tool_runner" ) - as_dict = self.toolbox.to_dict( mock_trans(), in_panel=False ) - assert as_dict[0]["id"] == "test_tool" + for json_conf in [True, False]: + self._init_tool_in_section(json=json_conf) + mapper = routes.Mapper() + mapper.connect( "tool_runner", "/test/tool_runner" ) + as_dict = self.toolbox.to_dict( mock_trans(), in_panel=False ) + assert as_dict[0]["id"] == "test_tool" def test_out_of_panel_filtering( self ): self._init_tool_in_section() @@ -183,7 +190,7 @@ class ToolBoxTestCase( BaseToolBoxTestCase ): if elem.get("id") == section_id: assert elem["model_class"] == "ToolSection" return elem - assert False, "Failed to find section with id [%s]" % id + assert False, "Failed to find section with id [%s]" % section_id def test_tool_shed_properties( self ): self._init_tool() @@ -275,6 +282,13 @@ class ToolBoxTestCase( BaseToolBoxTestCase ): toolbox = self.toolbox assert toolbox.get_tool( "test_tool" ) is not None + def test_tool_dir_json( self ): + self._init_tool() + self._add_config({"items": [{"type": "tool_dir", "dir": self.test_directory}]}, name="tool_conf.json") + + toolbox = self.toolbox + assert toolbox.get_tool( "test_tool" ) is not None + def test_workflow_in_panel( self ): stored_workflow = self.__test_workflow() encoded_id = self.app.security.encode_id( stored_workflow.id ) @@ -305,9 +319,19 @@ class ToolBoxTestCase( BaseToolBoxTestCase ): section = self.toolbox._tool_panel[ 'tid' ] self.__check_test_labels( section.elems ) - def _init_tool_in_section( self ): + def _init_tool_in_section( self, json=False ): self._init_tool() - self._add_config( """
""" ) + if not json: + self._add_config( """
""" ) + else: + section = { + "type": "section", + "id": "t", + "name": "test", + "items": [{"type": "tool", + "file": "tool.xml"}], + } + self._add_config({"items": [section]}, name="tool_conf.json") def __check_test_labels( self, panel_dict ): assert panel_dict.keys() == ["label_lab1", "label_lab2"] From 864cc32892a1b4bf91b5d2fe70f7e64eec736393 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 21 Dec 2015 21:17:16 +0000 Subject: [PATCH 07/12] Allow toolbox reloading via app, UI, and API. Introduce an app ToolCache to help ensure this operation remains speedy. --- lib/galaxy/config.py | 15 ++++++++--- lib/galaxy/tools/toolbox/base.py | 7 ++++- lib/galaxy/tools/toolbox/cache.py | 17 ++++++++++++ .../webapps/galaxy/api/configuration.py | 9 +++++++ lib/galaxy/webapps/galaxy/buildapp.py | 6 +++++ .../galaxy_install/tool_migration_manager.py | 3 ++- templates/admin/reload_tool.mako | 26 +++++++++++++++++++ 7 files changed, 77 insertions(+), 6 deletions(-) create mode 100644 lib/galaxy/tools/toolbox/cache.py diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index e9f7e9af4b0..53031c4424e 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -744,19 +744,26 @@ class ConfiguresGalaxyMixin: def _configure_genome_builds( self, data_table_name="__dbkeys__", load_old_style=True ): self.genome_builds = GenomeBuilds( self, data_table_name=data_table_name, load_old_style=load_old_style ) - def _configure_toolbox( self ): + def reload_toolbox(self): # Initialize the tools, making sure the list of tool configs includes the reserved migrated_tools_conf.xml file. + tool_configs = self.config.tool_configs if self.config.migrated_tools_config not in tool_configs: tool_configs.append( self.config.migrated_tools_config ) - from galaxy.managers.citations import CitationsManager - self.citations_manager = CitationsManager( self ) - from galaxy import tools self.toolbox = tools.ToolBox( tool_configs, self.config.tool_path, self ) self.reindex_tool_search() + def _configure_toolbox( self ): + from galaxy.managers.citations import CitationsManager + self.citations_manager = CitationsManager( self ) + + from galaxy.tools.toolbox.cache import ToolCache + self.tool_cache = ToolCache() + + self.reload_toolbox() + from galaxy.tools.deps import containers galaxy_root_dir = os.path.abspath(self.config.root) file_path = os.path.abspath(getattr(self.config, "file_path")) diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index c95b7805776..dfe4a1212e4 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -708,7 +708,12 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): def load_tool( self, config_file, guid=None, repository_id=None, **kwds ): """Load a single tool from the file named by `config_file` and return an instance of `Tool`.""" # Parse XML configuration file and get the root element - tool = self.create_tool( config_file=config_file, repository_id=repository_id, guid=guid, **kwds ) + tool_cache = getattr( self.app, 'tool_cache', None ) + tool = tool_cache and tool_cache.get_tool( config_file ) + if tool is None: + tool = self.create_tool( config_file=config_file, repository_id=repository_id, guid=guid, **kwds ) + if tool_cache: + self.app.tool_cache.cache_tool(config_file, tool) tool_id = tool.id if not tool_id.startswith("__"): # do not monitor special tools written to tmp directory - no reason diff --git a/lib/galaxy/tools/toolbox/cache.py b/lib/galaxy/tools/toolbox/cache.py new file mode 100644 index 00000000000..8521dc31dd2 --- /dev/null +++ b/lib/galaxy/tools/toolbox/cache.py @@ -0,0 +1,17 @@ + + +class ToolCache(object): + """ Cache tool defintions to allow quickly reloading the whole + toolbox. + """ + + def __init__(self): + self._tools_by_path = {} + + def get_tool(self, config_filename): + """ Get the tool from the cache if the tool is up to date. + """ + return self._tools_by_path.get(config_filename, None) + + def cache_tool(self, config_filename, tool): + self._tools_by_path[config_filename] = tool diff --git a/lib/galaxy/webapps/galaxy/api/configuration.py b/lib/galaxy/webapps/galaxy/api/configuration.py index e4748244989..78ca2554410 100644 --- a/lib/galaxy/webapps/galaxy/api/configuration.py +++ b/lib/galaxy/webapps/galaxy/api/configuration.py @@ -82,6 +82,15 @@ class ConfigurationController( BaseAPIController ): rval.append(entry) return rval + @expose_api + @require_admin + def reload_toolbox(self, trans): + """ + PUT /api/configuration/toolbox + Reload the Galaxy toolbox (but not individual tools). + """ + self.app.reload_toolbox() + def _tool_conf_to_dict(conf): return dict( diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 350be892581..4ce1816e69b 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -277,6 +277,12 @@ def populate_api_routes( webapp, app ): controller="configuration", action="tool_lineages" ) + webapp.mapper.connect( + '/api/configuration/toolbox', + controller="configuration", + action="reload_toolbox", + conditions=dict( method=["PUT"] ) + ) webapp.mapper.resource( 'configuration', 'configuration', path_prefix='/api' ) webapp.mapper.connect( "configuration_version", "/api/version", controller="configuration", diff --git a/lib/tool_shed/galaxy_install/tool_migration_manager.py b/lib/tool_shed/galaxy_install/tool_migration_manager.py index 49bcaf08900..e7f906c98ee 100644 --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -11,6 +11,7 @@ import logging from galaxy import util from galaxy.tools.toolbox import ToolSection +from galaxy.tools.toolbox.parser import ensure_tool_conf_item from galaxy.util.odict import odict from tool_shed.galaxy_install import install_manager @@ -272,7 +273,7 @@ class ToolMigrationManager( object ): proprietary_tool_config = section_elem.get( 'file' ) if tool_config == proprietary_tool_config: # The tool is loaded inside of the section_elem. - tool_sections.append( ToolSection( proprietary_tool_panel_elem ) ) + tool_sections.append( ToolSection( ensure_tool_conf_item( proprietary_tool_panel_elem ) ) ) if not is_displayed: is_displayed = True return is_displayed, tool_sections diff --git a/templates/admin/reload_tool.mako b/templates/admin/reload_tool.mako index c29a233e2e5..674101303e9 100644 --- a/templates/admin/reload_tool.mako +++ b/templates/admin/reload_tool.mako @@ -14,6 +14,17 @@ $().ready(function() { %endif focus_el.focus(); }); +$().ready(function() { + $("#reload_toolbox").click(function(){ + $.ajax({ + url: "${h.url_for(controller="/api/configuration", action="toolbox")}", + type: 'PUT', + success: function(result) { + alert("Toolbox reloaded."); + } + }); + }); +}); %if message: @@ -54,3 +65,18 @@ $().ready(function() { +

+

+
Reload Toolbox
+
+
+
+ Clicking here will reload + the Galaxy toolbox. This will cause newly discovered tools + to be added, tools now missing from tool confs to be removed, + and items in the panel reordered. Individual tools, even ones + with modified tool descriptions willl not be reloaded. +
+
+
+
From 4086dd61dafe562ec01a8b1fce687a539aedbcfe Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 23 Dec 2015 17:12:23 +0000 Subject: [PATCH 08/12] Implement toolbox monitoring. - Add monitor tag on toolbox root elements to force Galaxy to watch the toolbox for changes. - Refactoring and test improvements for existing tool monitoring code. - Implement toolbox shutdown process to ensure different watchers don't compete with each other. - Set the default tool conf to be monitored. --- config/tool_conf.xml.sample | 2 +- lib/galaxy/config.py | 11 ++- lib/galaxy/tools/toolbox/base.py | 24 ++++- lib/galaxy/tools/toolbox/parser.py | 15 ++- lib/galaxy/tools/toolbox/watcher.py | 142 ++++++++++++++++++++++++---- test/unit/tools/test_watcher.py | 40 +++++++- 6 files changed, 204 insertions(+), 30 deletions(-) diff --git a/config/tool_conf.xml.sample b/config/tool_conf.xml.sample index d38db168b94..a3d88aa69eb 100644 --- a/config/tool_conf.xml.sample +++ b/config/tool_conf.xml.sample @@ -1,5 +1,5 @@ - +
diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 53031c4424e..87864c85d2c 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -13,6 +13,7 @@ import socket import string import sys import tempfile +import threading from datetime import timedelta from galaxy.exceptions import ConfigurationError from galaxy.util import listify @@ -752,8 +753,12 @@ class ConfiguresGalaxyMixin: tool_configs.append( self.config.migrated_tools_config ) from galaxy import tools - self.toolbox = tools.ToolBox( tool_configs, self.config.tool_path, self ) - self.reindex_tool_search() + with self._toolbox_lock: + old_toolbox = self.toolbox + self.toolbox = tools.ToolBox( tool_configs, self.config.tool_path, self ) + self.reindex_tool_search() + if old_toolbox: + old_toolbox.shutdown() def _configure_toolbox( self ): from galaxy.managers.citations import CitationsManager @@ -762,6 +767,8 @@ class ConfiguresGalaxyMixin: from galaxy.tools.toolbox.cache import ToolCache self.tool_cache = ToolCache() + self._toolbox_lock = threading.Lock() + self.toolbox = None self.reload_toolbox() from galaxy.tools.deps import containers diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index dfe4a1212e4..0742b51951e 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -28,7 +28,8 @@ from .lineages import LineageMap from .tags import tool_tag_manager from .filters import FilterFactory -from .watcher import get_watcher +from .watcher import get_tool_watcher +from .watcher import get_tool_conf_watcher # Extra tool dependency not used by AbstractToolBox but by # BaseGalaxyToolBox @@ -72,7 +73,8 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): # (e.g., shed_tool_conf.xml) files include the tool_path attribute within the tag. self._tool_root_dir = tool_root_dir self.app = app - self._tool_watcher = get_watcher( self, app.config ) + self._tool_watcher = get_tool_watcher( self, app.config ) + self._tool_conf_watcher = get_tool_conf_watcher( lambda: app.reload_toolbox() ) self._filter_factory = FilterFactory( self ) self._tool_tag_manager = tool_tag_manager( app ) self._init_tools_from_configs( config_filenames ) @@ -154,6 +156,9 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): config_elems=config_elems ) self._dynamic_tool_confs.append( shed_tool_conf_dict ) + if tool_conf_source.parse_monitor(): + self._tool_conf_watcher.watch_file(config_filename) + def load_item( self, item, tool_path, panel_dict=None, integrated_panel_dict=None, load_panel_dict=True, guid=None, index=None, internal=False ): item = ensure_tool_conf_item(item) item_type = item.type @@ -1003,6 +1008,21 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): return rval + def shutdown(self): + exception = None + try: + self._tool_watcher.shutdown() + except Exception as e: + exception = e + + try: + self._tool_conf_watcher.shutdown() + except Exception as e: + exception = exception or e + + if exception: + raise exception + def _lineage_in_panel( self, panel_dict, tool=None, tool_lineage=None ): """ If tool with same lineage already in panel (or section) - find and return it. Otherwise return None. diff --git a/lib/galaxy/tools/toolbox/parser.py b/lib/galaxy/tools/toolbox/parser.py index 6a075095972..83d0055c1de 100644 --- a/lib/galaxy/tools/toolbox/parser.py +++ b/lib/galaxy/tools/toolbox/parser.py @@ -1,9 +1,11 @@ from abc import ABCMeta from abc import abstractmethod -from galaxy.util import parse_xml +from galaxy.util import parse_xml, string_as_bool import yaml +DEFAULT_MONITOR = False + class ToolConfSource(object): """ This interface represents an abstract source to parse tool @@ -21,6 +23,11 @@ class ToolConfSource(object): """ Return tool_path for tools in this toolbox. """ + def parse_monitor(self): + """ Monitor the toolbox configuration source for changes and + reload. """ + return DEFAULT_MONITOR + class XmlToolConfSource(ToolConfSource): @@ -34,6 +41,9 @@ class XmlToolConfSource(ToolConfSource): def parse_items(self): return map(ensure_tool_conf_item, self.root.getchildren()) + def parse_monitor(self): + return string_as_bool(self.root.get('monitor', DEFAULT_MONITOR)) + class YamlToolConfSource(ToolConfSource): @@ -48,6 +58,9 @@ class YamlToolConfSource(ToolConfSource): def parse_items(self): return map(ToolConfItem.from_dict, self.as_dict.get('items')) + def parse_monitor(self): + return self.as_dict.get('monitor', DEFAULT_MONITOR) + class ToolConfItem(object): """ This interface represents an abstract source to parse tool diff --git a/lib/galaxy/tools/toolbox/watcher.py b/lib/galaxy/tools/toolbox/watcher.py index c1a3ca7d7a9..1919fca423f 100644 --- a/lib/galaxy/tools/toolbox/watcher.py +++ b/lib/galaxy/tools/toolbox/watcher.py @@ -1,4 +1,8 @@ +import logging import os.path +import threading +import time + try: from watchdog.events import FileSystemEventHandler from watchdog.observers import Observer @@ -9,34 +13,130 @@ except ImportError: PollingObserver = object can_watch = False -import logging log = logging.getLogger( __name__ ) -def get_watcher(toolbox, config): - watch_tools_val = str( getattr(config, "watch_tools", False) ).lower() - if watch_tools_val in ( 'true', 'yes', 'on' ): - return ToolWatcher(toolbox) - elif watch_tools_val == "auto": - try: - return ToolWatcher(toolbox) - except Exception: - log.info("Failed to load ToolWatcher (watchdog is likely unavailable) - proceeding without tool monitoring.") - return NullWatcher() - elif watch_tools_val == "polling": - log.info("Using less ineffecient polling toolbox watcher.") - return ToolWatcher(toolbox, observer_class=PollingObserver) +def get_observer_class(config_value, default, monitor_what_str): + """ + """ + config_value = config_value or default + config_value = str(config_value).lower() + if config_value in ("true", "yes", "on", "auto"): + expect_observer = config_value != "auto" + observer_class = Observer + elif config_value == "polling": + expect_observer = True + observer_class = PollingObserver + else: + expect_observer = False + observer_class = None + + if observer_class is None: + message = "Watchdog library unavailble, cannot monitor %s." % monitor_what_str + log.info(message) + if expect_observer: + raise Exception(message) + + return observer_class + + +def get_tool_conf_watcher(reload_callback): + return ToolConfWatcher(reload_callback) + + +def get_tool_watcher(toolbox, config): + config_value = getattr(config, "watch_tools", None) + observer_class = get_observer_class(config_value, default="False", monitor_what_str="tools") + + if observer_class is not None: + return ToolWatcher(toolbox, observer_class=observer_class) else: return NullWatcher() +class ToolConfWatcher(object): + + def __init__(self, reload_callback): + self.paths = {} + self._active = False + self._lock = threading.Lock() + self.thread = threading.Thread(target=self.check) + self.thread.daemon = True + self.event_handler = ToolConfFileEventHandler(reload_callback) + + def start(self): + if not self._active: + self._active = True + self.thread.start() + + def shutdown(self): + self._active = False + self.thread.join() + + def check(self): + while self._active: + do_reload = False + with self._lock: + paths = list(self.paths.keys()) + for path in paths: + if not os.path.exists(path): + continue + mod_time = self.paths[path] + new_mod_time = None + if os.path.exists(path): + new_mod_time = time.ctime(os.path.getmtime(path)) + if new_mod_time != mod_time: + self.paths[path] = new_mod_time + do_reload = True + + if do_reload: + t = threading.Thread(target=lambda: self.event_handler.on_any_event(None)) + t.daemon = True + t.start() + time.sleep(1) + + def monitor(self, path): + mod_time = None + if os.path.exists(path): + mod_time = time.ctime(os.path.getmtime(path)) + with self._lock: + self.paths[path] = mod_time + self.start() + + def watch_file(self, tool_conf_file): + self.monitor(tool_conf_file) + + +class NullToolConfWatcher(object): + + def start(self): + pass + + def shutdown(self): + pass + + def monitor(self, conf_path): + pass + + def watch_file(self, tool_file, tool_id): + pass + + +class ToolConfFileEventHandler(FileSystemEventHandler): + + def __init__(self, reload_callback): + self.reload_callback = reload_callback + + def on_any_event(self, event): + self._handle(event) + + def _handle(self, event): + self.reload_callback() + + class ToolWatcher(object): - def __init__(self, toolbox, observer_class=None): - if not can_watch: - raise Exception("Watchdog library unavailble, cannot watch tools.") - if observer_class is None: - observer_class = Observer + def __init__(self, toolbox, observer_class): self.toolbox = toolbox self.tool_file_ids = {} self.tool_dir_callbacks = {} @@ -48,6 +148,10 @@ class ToolWatcher(object): def start(self): self.observer.start() + def shutdown(self): + self.observer.stop() + self.observer.join() + def monitor(self, dir): self.observer.schedule(self.event_handler, dir, recursive=False) diff --git a/test/unit/tools/test_watcher.py b/test/unit/tools/test_watcher.py index 3a67ec1dfa4..a745f8f950c 100644 --- a/test/unit/tools/test_watcher.py +++ b/test/unit/tools/test_watcher.py @@ -17,13 +17,43 @@ def test_watcher(): tool_path = path.join(t, "test.xml") toolbox = Toolbox() open(tool_path, "w").write("a") - tool_watcher = watcher.get_watcher(toolbox, bunch.Bunch( + tool_watcher = watcher.get_tool_watcher(toolbox, bunch.Bunch( watch_tools=True )) tool_watcher.watch_file(tool_path, "cool_tool") + assert not toolbox.was_reloaded("cool_tool") open(tool_path, "w").write("b") - time.sleep(2) - toolbox.assert_reloaded("cool_tool") + wait_for_reload(lambda: toolbox.was_reloaded("cool_tool")) + tool_watcher.shutdown() + assert not tool_watcher.observer.is_alive() + + +def test_tool_conf_watcher(): + if not watcher.can_watch: + from nose.plugins.skip import SkipTest + raise SkipTest() + + callback = CallbackRecorder() + conf_watcher = watcher.get_tool_conf_watcher(callback.call) + + with __test_directory() as t: + tool_conf_path = path.join(t, "test_conf.xml") + conf_watcher.watch_file(tool_conf_path) + + open(tool_conf_path, "w").write("b") + wait_for_reload(lambda: callback.called) + conf_watcher.shutdown() + assert not conf_watcher.thread.is_alive() + + +def wait_for_reload(check): + reloaded = False + for i in range(10): + reloaded = check() + if reloaded: + break + time.sleep(.2) + assert reloaded class Toolbox(object): @@ -34,8 +64,8 @@ class Toolbox(object): def reload_tool_by_id( self, tool_id ): self.reloaded[ tool_id ] = True - def assert_reloaded(self, tool_id): - assert self.reloaded.get( tool_id, False ) + def was_reloaded(self, tool_id): + return self.reloaded.get( tool_id, False ) class CallbackRecorder(object): From 8027228639a771e4c62131ab15ba4ffbb656498a Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 24 Dec 2015 11:49:41 +0000 Subject: [PATCH 09/12] Revise tool caching for tool shed tool loading... The tool shed appears to load the same tool in the toolbox multiple times with different settings. Get around this by not using the cache when loading tools from a few tool shed components later in this process. --- lib/galaxy/tools/toolbox/base.py | 9 ++++++--- lib/galaxy/tools/toolbox/cache.py | 9 +++++++++ .../metadata/installed_repository_metadata_manager.py | 2 +- lib/tool_shed/galaxy_install/tool_migration_manager.py | 2 +- lib/tool_shed/tools/data_table_manager.py | 2 +- lib/tool_shed/tools/tool_validator.py | 2 +- lib/tool_shed/util/tool_util.py | 2 +- 7 files changed, 20 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index 0742b51951e..b1fd6335820 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -710,12 +710,12 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): if tool_loaded or force_watch: self._tool_watcher.watch_directory( directory, quick_load ) - def load_tool( self, config_file, guid=None, repository_id=None, **kwds ): + def load_tool( self, config_file, guid=None, repository_id=None, use_cached=True, **kwds ): """Load a single tool from the file named by `config_file` and return an instance of `Tool`.""" # Parse XML configuration file and get the root element tool_cache = getattr( self.app, 'tool_cache', None ) - tool = tool_cache and tool_cache.get_tool( config_file ) - if tool is None: + tool = use_cached and tool_cache and tool_cache.get_tool( config_file ) + if not tool: tool = self.create_tool( config_file=config_file, repository_id=repository_id, guid=guid, **kwds ) if tool_cache: self.app.tool_cache.cache_tool(config_file, tool) @@ -921,6 +921,9 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): else: tool = self._tools_by_id[ tool_id ] del self._tools_by_id[ tool_id ] + tool_cache = getattr( self.app, 'tool_cache', None ) + if tool_cache: + tool_cache.expire_tool( tool_id ) if remove_from_panel: tool_key = 'tool_' + tool_id for key, val in self._tool_panel.items(): diff --git a/lib/galaxy/tools/toolbox/cache.py b/lib/galaxy/tools/toolbox/cache.py index 8521dc31dd2..3bc524024eb 100644 --- a/lib/galaxy/tools/toolbox/cache.py +++ b/lib/galaxy/tools/toolbox/cache.py @@ -7,11 +7,20 @@ class ToolCache(object): def __init__(self): self._tools_by_path = {} + self._tool_paths_by_id = {} def get_tool(self, config_filename): """ Get the tool from the cache if the tool is up to date. """ return self._tools_by_path.get(config_filename, None) + def expire_tool(self, tool_id): + if tool_id in self._tool_paths_by_id: + config_filename = self._tool_paths_by_id[tool_id] + del self._tool_paths_by_id[tool_id] + del self._tools_by_path[config_filename] + def cache_tool(self, config_filename, tool): + tool_id = str( tool.id ) + self._tool_paths_by_id[tool_id] = config_filename self._tools_by_path[config_filename] = tool diff --git a/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py b/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py index 736522d1657..e03e07ffce3 100644 --- a/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py +++ b/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py @@ -74,7 +74,7 @@ class InstalledRepositoryMetadataManager( metadata_generator.MetadataGenerator ) load_relative_path = os.path.join( shed_conf_dict.get( 'tool_path' ), relative_path ) guid = tool_dict.get( 'guid', None ) if relative_path and guid: - tool = self.app.toolbox.load_tool( os.path.abspath( load_relative_path ), guid=guid ) + tool = self.app.toolbox.load_tool( os.path.abspath( load_relative_path ), guid=guid, use_cached=False ) else: tool = None if tool: diff --git a/lib/tool_shed/galaxy_install/tool_migration_manager.py b/lib/tool_shed/galaxy_install/tool_migration_manager.py index e7f906c98ee..64842c07d1f 100644 --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -290,7 +290,7 @@ class ToolMigrationManager( object ): filename = basic_util.strip_path( name ) if filename == tool_config_filename: full_path = str( os.path.abspath( os.path.join( root, name ) ) ) - tool = self.toolbox.load_tool( full_path ) + tool = self.toolbox.load_tool( full_path, use_cached=False ) return suc.generate_tool_guid( repository_clone_url, tool ) # Not quite sure what should happen here, throw an exception or what? return None diff --git a/lib/tool_shed/tools/data_table_manager.py b/lib/tool_shed/tools/data_table_manager.py index 5b27fd87ada..ae4b8595685 100644 --- a/lib/tool_shed/tools/data_table_manager.py +++ b/lib/tool_shed/tools/data_table_manager.py @@ -83,7 +83,7 @@ class ToolDataTableManager( object ): # TODO: Do more here than logging an exception. log.debug( message ) # Reload the tool into the local list of repository_tools_tups. - repository_tool = self.app.toolbox.load_tool( os.path.join( tool_path, tup_path ), guid=guid ) + repository_tool = self.app.toolbox.load_tool( os.path.join( tool_path, tup_path ), guid=guid, use_cached=False ) repository_tools_tups[ index ] = ( tup_path, guid, repository_tool ) # Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file. self.reset_tool_data_tables() diff --git a/lib/tool_shed/tools/tool_validator.py b/lib/tool_shed/tools/tool_validator.py index c9bcf31765d..1502c896de0 100644 --- a/lib/tool_shed/tools/tool_validator.py +++ b/lib/tool_shed/tools/tool_validator.py @@ -292,7 +292,7 @@ class ToolValidator( object ): def load_tool_from_config( self, repository_id, full_path ): try: - tool = self.app.toolbox.load_tool( full_path, repository_id=repository_id, allow_code_files=False ) + tool = self.app.toolbox.load_tool( full_path, repository_id=repository_id, allow_code_files=False, use_cached=False ) valid = True error_message = None except KeyError, e: diff --git a/lib/tool_shed/util/tool_util.py b/lib/tool_shed/util/tool_util.py index 9df2d6a347e..d88d9df14ca 100644 --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -179,7 +179,7 @@ def handle_missing_index_file( app, tool_path, sample_files, repository_tools_tu sample_files_copied.append( options.missing_index_file ) break # Reload the tool into the local list of repository_tools_tups. - repository_tool = app.toolbox.load_tool( os.path.join( tool_path, tup_path ), guid=guid ) + repository_tool = app.toolbox.load_tool( os.path.join( tool_path, tup_path ), guid=guid, use_cached=False ) repository_tools_tups[ index ] = ( tup_path, guid, repository_tool ) return repository_tools_tups, sample_files_copied From eddfec36fdf6d19bae398d09ee16952136d2b379 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 26 Dec 2015 15:31:25 +0000 Subject: [PATCH 10/12] Refactor package_tool for better separation of concerns. The tool should know how to package itself, it shouldn't be the responsiblity of the toolbox. Update the code to reflect this and use more pythonic exception handling. --- lib/galaxy/tools/__init__.py | 105 ++++++++++++++++++++++ lib/galaxy/tools/toolbox/base.py | 108 +---------------------- lib/galaxy/web/base/controllers/admin.py | 18 ++-- lib/galaxy/webapps/galaxy/api/tools.py | 11 ++- 4 files changed, 122 insertions(+), 120 deletions(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 32a3e799965..189bb3b7cba 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -7,6 +7,9 @@ import glob import json import logging import os +import re +import tarfile +import tempfile import threading import urllib from datetime import datetime @@ -1979,6 +1982,108 @@ class Tool( object, Dictifiable ): """ return output_collect.collect_dynamic_collections( self, output, **kwds ) + def to_archive(self): + tool = self.tool + tarball_files = [] + temp_files = [] + tool_xml = open( os.path.abspath( tool.config_file ), 'r' ).read() + # Retrieve tool help images and rewrite the tool's xml into a temporary file with the path + # modified to be relative to the repository root. + image_found = False + if tool.help is not None: + tool_help = tool.help._source + # Check each line of the rendered tool help for an image tag that points to a location under static/ + for help_line in tool_help.split( '\n' ): + image_regex = re.compile( 'img alt="[^"]+" src="\${static_path}/([^"]+)"' ) + matches = re.search( image_regex, help_line ) + if matches is not None: + tool_help_image = matches.group(1) + tarball_path = tool_help_image + filesystem_path = os.path.abspath( os.path.join( self.app.config.root, 'static', tool_help_image ) ) + if os.path.exists( filesystem_path ): + tarball_files.append( ( filesystem_path, tarball_path ) ) + image_found = True + tool_xml = tool_xml.replace( '${static_path}/%s' % tarball_path, tarball_path ) + # If one or more tool help images were found, add the modified tool XML to the tarball instead of the original. + if image_found: + fd, new_tool_config = tempfile.mkstemp( suffix='.xml' ) + os.close( fd ) + open( new_tool_config, 'w' ).write( tool_xml ) + tool_tup = ( os.path.abspath( new_tool_config ), os.path.split( tool.config_file )[-1] ) + temp_files.append( os.path.abspath( new_tool_config ) ) + else: + tool_tup = ( os.path.abspath( tool.config_file ), os.path.split( tool.config_file )[-1] ) + tarball_files.append( tool_tup ) + # TODO: This feels hacky. + tool_command = tool.command.strip().split()[0] + tool_path = os.path.dirname( os.path.abspath( tool.config_file ) ) + # Add the tool XML to the tuple that will be used to populate the tarball. + if os.path.exists( os.path.join( tool_path, tool_command ) ): + tarball_files.append( ( os.path.join( tool_path, tool_command ), tool_command ) ) + # Find and add macros and code files. + for external_file in tool.get_externally_referenced_paths( os.path.abspath( tool.config_file ) ): + external_file_abspath = os.path.abspath( os.path.join( tool_path, external_file ) ) + tarball_files.append( ( external_file_abspath, external_file ) ) + if os.path.exists( os.path.join( tool_path, "Dockerfile" ) ): + tarball_files.append( ( os.path.join( tool_path, "Dockerfile" ), "Dockerfile" ) ) + # Find tests, and check them for test data. + tests = tool.tests + if tests is not None: + for test in tests: + # Add input file tuples to the list. + for input in test.inputs: + for input_value in test.inputs[ input ]: + input_path = os.path.abspath( os.path.join( 'test-data', input_value ) ) + if os.path.exists( input_path ): + td_tup = ( input_path, os.path.join( 'test-data', input_value ) ) + tarball_files.append( td_tup ) + # And add output file tuples to the list. + for label, filename, _ in test.outputs: + output_filepath = os.path.abspath( os.path.join( 'test-data', filename ) ) + if os.path.exists( output_filepath ): + td_tup = ( output_filepath, os.path.join( 'test-data', filename ) ) + tarball_files.append( td_tup ) + for param in tool.input_params: + # Check for tool data table definitions. + if hasattr( param, 'options' ): + if hasattr( param.options, 'tool_data_table' ): + data_table = param.options.tool_data_table + if hasattr( data_table, 'filenames' ): + data_table_definitions = [] + for data_table_filename in data_table.filenames: + # FIXME: from_shed_config seems to always be False. + if not data_table.filenames[ data_table_filename ][ 'from_shed_config' ]: + tar_file = data_table.filenames[ data_table_filename ][ 'filename' ] + '.sample' + sample_file = os.path.join( data_table.filenames[ data_table_filename ][ 'tool_data_path' ], + tar_file ) + # Use the .sample file, if one exists. If not, skip this data table. + if os.path.exists( sample_file ): + tarfile_path, tarfile_name = os.path.split( tar_file ) + tarfile_path = os.path.join( 'tool-data', tarfile_name ) + tarball_files.append( ( sample_file, tarfile_path ) ) + data_table_definitions.append( data_table.xml_string ) + if len( data_table_definitions ) > 0: + # Put the data table definition XML in a temporary file. + table_definition = '\n\n %s' + table_definition = table_definition % '\n'.join( data_table_definitions ) + fd, table_conf = tempfile.mkstemp() + os.close( fd ) + open( table_conf, 'w' ).write( table_definition ) + tarball_files.append( ( table_conf, os.path.join( 'tool-data', 'tool_data_table_conf.xml.sample' ) ) ) + temp_files.append( table_conf ) + # Create the tarball. + fd, tarball_archive = tempfile.mkstemp( suffix='.tgz' ) + os.close( fd ) + tarball = tarfile.open( name=tarball_archive, mode='w:gz' ) + # Add the files from the previously generated list. + for fspath, tarpath in tarball_files: + tarball.add( fspath, arcname=tarpath ) + tarball.close() + # Delete any temporary files that were generated. + for temp_file in temp_files: + os.remove( temp_file ) + return tarball_archive + def to_dict( self, trans, link_details=False, io_details=False ): """ Returns dict of tool. """ diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index b1fd6335820..098874d3dd7 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -1,14 +1,13 @@ import logging import os -import re import string -import tarfile -import tempfile from markupsafe import escape from six.moves.urllib.parse import urlparse from six import iteritems +from galaxy.exceptions import ObjectNotFound + from galaxy.util.dictifiable import Dictifiable from galaxy.util.odict import odict @@ -763,109 +762,10 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): """ # Make sure the tool is actually loaded. if tool_id not in self._tools_by_id: - return None, False, "No tool with id %s" % escape( tool_id ) + raise ObjectNotFound("No tool found with id '%s'." % escape( tool_id )) else: tool = self._tools_by_id[ tool_id ] - tarball_files = [] - temp_files = [] - tool_xml = open( os.path.abspath( tool.config_file ), 'r' ).read() - # Retrieve tool help images and rewrite the tool's xml into a temporary file with the path - # modified to be relative to the repository root. - image_found = False - if tool.help is not None: - tool_help = tool.help._source - # Check each line of the rendered tool help for an image tag that points to a location under static/ - for help_line in tool_help.split( '\n' ): - image_regex = re.compile( 'img alt="[^"]+" src="\${static_path}/([^"]+)"' ) - matches = re.search( image_regex, help_line ) - if matches is not None: - tool_help_image = matches.group(1) - tarball_path = tool_help_image - filesystem_path = os.path.abspath( os.path.join( trans.app.config.root, 'static', tool_help_image ) ) - if os.path.exists( filesystem_path ): - tarball_files.append( ( filesystem_path, tarball_path ) ) - image_found = True - tool_xml = tool_xml.replace( '${static_path}/%s' % tarball_path, tarball_path ) - # If one or more tool help images were found, add the modified tool XML to the tarball instead of the original. - if image_found: - fd, new_tool_config = tempfile.mkstemp( suffix='.xml' ) - os.close( fd ) - open( new_tool_config, 'w' ).write( tool_xml ) - tool_tup = ( os.path.abspath( new_tool_config ), os.path.split( tool.config_file )[-1] ) - temp_files.append( os.path.abspath( new_tool_config ) ) - else: - tool_tup = ( os.path.abspath( tool.config_file ), os.path.split( tool.config_file )[-1] ) - tarball_files.append( tool_tup ) - # TODO: This feels hacky. - tool_command = tool.command.strip().split()[0] - tool_path = os.path.dirname( os.path.abspath( tool.config_file ) ) - # Add the tool XML to the tuple that will be used to populate the tarball. - if os.path.exists( os.path.join( tool_path, tool_command ) ): - tarball_files.append( ( os.path.join( tool_path, tool_command ), tool_command ) ) - # Find and add macros and code files. - for external_file in tool.get_externally_referenced_paths( os.path.abspath( tool.config_file ) ): - external_file_abspath = os.path.abspath( os.path.join( tool_path, external_file ) ) - tarball_files.append( ( external_file_abspath, external_file ) ) - if os.path.exists( os.path.join( tool_path, "Dockerfile" ) ): - tarball_files.append( ( os.path.join( tool_path, "Dockerfile" ), "Dockerfile" ) ) - # Find tests, and check them for test data. - tests = tool.tests - if tests is not None: - for test in tests: - # Add input file tuples to the list. - for input in test.inputs: - for input_value in test.inputs[ input ]: - input_path = os.path.abspath( os.path.join( 'test-data', input_value ) ) - if os.path.exists( input_path ): - td_tup = ( input_path, os.path.join( 'test-data', input_value ) ) - tarball_files.append( td_tup ) - # And add output file tuples to the list. - for label, filename, _ in test.outputs: - output_filepath = os.path.abspath( os.path.join( 'test-data', filename ) ) - if os.path.exists( output_filepath ): - td_tup = ( output_filepath, os.path.join( 'test-data', filename ) ) - tarball_files.append( td_tup ) - for param in tool.input_params: - # Check for tool data table definitions. - if hasattr( param, 'options' ): - if hasattr( param.options, 'tool_data_table' ): - data_table = param.options.tool_data_table - if hasattr( data_table, 'filenames' ): - data_table_definitions = [] - for data_table_filename in data_table.filenames: - # FIXME: from_shed_config seems to always be False. - if not data_table.filenames[ data_table_filename ][ 'from_shed_config' ]: - tar_file = data_table.filenames[ data_table_filename ][ 'filename' ] + '.sample' - sample_file = os.path.join( data_table.filenames[ data_table_filename ][ 'tool_data_path' ], - tar_file ) - # Use the .sample file, if one exists. If not, skip this data table. - if os.path.exists( sample_file ): - tarfile_path, tarfile_name = os.path.split( tar_file ) - tarfile_path = os.path.join( 'tool-data', tarfile_name ) - tarball_files.append( ( sample_file, tarfile_path ) ) - data_table_definitions.append( data_table.xml_string ) - if len( data_table_definitions ) > 0: - # Put the data table definition XML in a temporary file. - table_definition = '\n\n %s' - table_definition = table_definition % '\n'.join( data_table_definitions ) - fd, table_conf = tempfile.mkstemp() - os.close( fd ) - open( table_conf, 'w' ).write( table_definition ) - tarball_files.append( ( table_conf, os.path.join( 'tool-data', 'tool_data_table_conf.xml.sample' ) ) ) - temp_files.append( table_conf ) - # Create the tarball. - fd, tarball_archive = tempfile.mkstemp( suffix='.tgz' ) - os.close( fd ) - tarball = tarfile.open( name=tarball_archive, mode='w:gz' ) - # Add the files from the previously generated list. - for fspath, tarpath in tarball_files: - tarball.add( fspath, arcname=tarpath ) - tarball.close() - # Delete any temporary files that were generated. - for temp_file in temp_files: - os.remove( temp_file ) - return tarball_archive, True, None - return None, False, "An unknown error occurred." + return tool.to_archive() def reload_tool_by_id( self, tool_id ): """ diff --git a/lib/galaxy/web/base/controllers/admin.py b/lib/galaxy/web/base/controllers/admin.py index 360e2c98907..57ffca7b485 100644 --- a/lib/galaxy/web/base/controllers/admin.py +++ b/lib/galaxy/web/base/controllers/admin.py @@ -68,26 +68,24 @@ class Admin( object ): def package_tool( self, trans, **kwd ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) toolbox = self.app.toolbox tool_id = None if params.get( 'package_tool_button', False ): tool_id = params.get('tool_id', None) - tool_tarball, success, message = trans.app.toolbox.package_tool( trans, tool_id ) - if success: + try: + tool_tarball = trans.app.toolbox.package_tool( trans, tool_id ) trans.response.set_content_type( 'application/x-gzip' ) download_file = open( tool_tarball ) os.unlink( tool_tarball ) tarball_path, filename = os.path.split( tool_tarball ) trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.tgz"' % ( tool_id ) return download_file - else: - status = 'error' - return trans.fill_template( '/admin/package_tool.mako', - tool_id=tool_id, - toolbox=toolbox, - message=message, - status=status ) + except Exception: + return trans.fill_template( '/admin/package_tool.mako', + tool_id=tool_id, + toolbox=toolbox, + message=message, + status='error' ) @web.expose @web.require_admin diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index e4a3112d07e..dfa2febb1e8 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -198,12 +198,11 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): @web.expose_api_raw @web.require_admin def download( self, trans, id, **kwds ): - tool_tarball, success, message = trans.app.toolbox.package_tool( trans, id ) - if success: - trans.response.set_content_type( 'application/x-gzip' ) - download_file = open( tool_tarball ) - trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.tgz"' % ( id ) - return download_file + tool_tarball = trans.app.toolbox.package_tool(trans, id) + trans.response.set_content_type('application/x-gzip') + download_file = open(tool_tarball, "rb") + trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.tgz"' % (id) + return download_file @expose_api_anonymous def create( self, trans, payload, **kwd ): From ce24ec6821bdb87f14f3047eef186c246de2d5ed Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 29 Dec 2015 15:20:57 +0000 Subject: [PATCH 11/12] Remove alert from toolbox reload. --- templates/admin/reload_tool.mako | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/templates/admin/reload_tool.mako b/templates/admin/reload_tool.mako index 674101303e9..76b48a702ae 100644 --- a/templates/admin/reload_tool.mako +++ b/templates/admin/reload_tool.mako @@ -18,10 +18,7 @@ $().ready(function() { $("#reload_toolbox").click(function(){ $.ajax({ url: "${h.url_for(controller="/api/configuration", action="toolbox")}", - type: 'PUT', - success: function(result) { - alert("Toolbox reloaded."); - } + type: 'PUT' }); }); }); From 523e2b40063d2abb7662a163f0144902541bed18 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 3 Jan 2016 12:52:31 +0000 Subject: [PATCH 12/12] Fix shutdown of tool watcher if never activated. Caused a harmless, but distracting stacktrace to be printed on toolbox reload as pointed out by @bgruening https://github.com/galaxyproject/galaxy/pull/1398#issuecomment-168499678. --- lib/galaxy/tools/toolbox/watcher.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tools/toolbox/watcher.py b/lib/galaxy/tools/toolbox/watcher.py index 1919fca423f..ef4b65947bd 100644 --- a/lib/galaxy/tools/toolbox/watcher.py +++ b/lib/galaxy/tools/toolbox/watcher.py @@ -70,8 +70,9 @@ class ToolConfWatcher(object): self.thread.start() def shutdown(self): - self._active = False - self.thread.join() + if self._active: + self._active = False + self.thread.join() def check(self): while self._active: