From 1d240e4ee2e2f9394df3cfdecd32790bab28dbb8 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 7 Feb 2018 08:29:05 -0500 Subject: [PATCH 01/14] Library methods for dealing with sandboxed JavaScript expressions. Based on work by Peter Amstutz in cwltool (https://github.com/common-workflow-language/cwltool). --- lib/galaxy/config.py | 1 + lib/galaxy/tools/__init__.py | 11 ++ lib/galaxy/tools/expressions/__init__.py | 12 ++ lib/galaxy/tools/expressions/cwlNodeEngine.js | 46 ++++++ lib/galaxy/tools/expressions/evaluation.py | 37 +++++ lib/galaxy/tools/expressions/sandbox.py | 151 ++++++++++++++++++ lib/galaxy/tools/expressions/util.py | 13 ++ test/unit/tools/test_expression_basics.py | 8 + 8 files changed, 279 insertions(+) create mode 100644 lib/galaxy/tools/expressions/__init__.py create mode 100644 lib/galaxy/tools/expressions/cwlNodeEngine.js create mode 100644 lib/galaxy/tools/expressions/evaluation.py create mode 100644 lib/galaxy/tools/expressions/sandbox.py create mode 100644 lib/galaxy/tools/expressions/util.py create mode 100644 test/unit/tools/test_expression_basics.py diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index e529fddf706..bb1877b2d83 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -313,6 +313,7 @@ class Configuration(object): log.warning("preserve_python_environment set to unknown value [%s], defaulting to legacy_only") preserve_python_environment = "legacy_only" self.preserve_python_environment = preserve_python_environment + self.nodejs_path = kwargs.get("nodejs_path", None) # Older default container cache path, I don't think anyone is using it anymore and it wasn't documented - we # should probably drop the backward compatiblity to save the path check. self.container_image_cache_path = self.resolve_path(kwargs.get("container_image_cache_path", "database/container_images")) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 0e2c3da4abd..19618bf4ca2 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -36,6 +36,7 @@ from galaxy.tools.actions.model_operations import ModelOperationToolAction from galaxy.tools.deps import ( CachedDependencyManager, ) +from galaxy.tools import expressions from galaxy.tools.fetcher import ToolLocationFetcher from galaxy.tools.parameters import ( check_param, @@ -102,6 +103,12 @@ from .provided_metadata import parse_tool_provided_metadata log = logging.getLogger(__name__) +REQUIRES_JS_RUNTIME_MESSAGE = ("The tool [%s] requires a nodejs runtime to execute " + "but node nor nodejs could be found on Galaxy's PATH and " + "no runtime was configured using the nodejs_path option in " + "galaxy.ini.") + + HELP_UNINITIALIZED = threading.Lock() MODEL_TOOLS_PATH = os.path.abspath(os.path.dirname(__file__)) # Tools that require Galaxy's Python environment to be preserved. @@ -747,6 +754,10 @@ class Tool(Dictifiable): module, cls = action mod = __import__(module, globals(), locals(), [cls]) self.tool_action = getattr(mod, cls)() + if getattr(self.tool_action, "requires_js_runtime", False): + if expressions.find_engine(self.app.config) is None: + message = REQUIRES_JS_RUNTIME_MESSAGE % self.tool_id + raise Exception(message) # Tests self.__parse_tests(tool_source) diff --git a/lib/galaxy/tools/expressions/__init__.py b/lib/galaxy/tools/expressions/__init__.py new file mode 100644 index 00000000000..b7519e5f57c --- /dev/null +++ b/lib/galaxy/tools/expressions/__init__.py @@ -0,0 +1,12 @@ +from .evaluation import evaluate +from .sandbox import execjs, interpolate +from .util import jshead, find_engine + + +__all__ = ( + 'evaluate', + 'execjs', + 'find_engine', + 'interpolate', + 'jshead', +) diff --git a/lib/galaxy/tools/expressions/cwlNodeEngine.js b/lib/galaxy/tools/expressions/cwlNodeEngine.js new file mode 100644 index 00000000000..1129584977f --- /dev/null +++ b/lib/galaxy/tools/expressions/cwlNodeEngine.js @@ -0,0 +1,46 @@ +#!/usr/bin/env nodejs + +"use strict"; + +process.stdin.setEncoding('utf8'); + +var incoming = ""; + +process.stdin.on('readable', function() { + var chunk = process.stdin.read(); + if (chunk !== null) { + incoming += chunk; + } +}); + +process.stdin.on('end', function() { + var j = JSON.parse(incoming); + var exp = "" + + if (j.script[0] == "{") { + exp = "{return function()" + j.script + "();}"; + } + else { + exp = "{return " + j.script + ";}"; + } + + var fn = '"use strict";\n'; + + if (j.engineConfig) { + for (var index = 0; index < j.engineConfig.length; ++index) { + fn += j.engineConfig[index] + "\n"; + } + } + + fn += "var $job = " + JSON.stringify(j.job) + ";\n"; + fn += "var $self = " + JSON.stringify(j.context) + ";\n" + + fn += "var $runtime = " + JSON.stringify(j.runtime) + ";\n" + fn += "var $tmpdir = " + JSON.stringify(j.tmpdir) + ";\n" + fn += "var $outdir = " + JSON.stringify(j.outdir) + ";\n" + + + fn += "(function()" + exp + ")()"; + + process.stdout.write(JSON.stringify(require("vm").runInNewContext(fn, {}))); +}); diff --git a/lib/galaxy/tools/expressions/evaluation.py b/lib/galaxy/tools/expressions/evaluation.py new file mode 100644 index 00000000000..89becc1f862 --- /dev/null +++ b/lib/galaxy/tools/expressions/evaluation.py @@ -0,0 +1,37 @@ +import json +import os +import subprocess + +from .util import find_engine + +FILE_DIRECTORY = os.path.normpath(os.path.dirname(os.path.join(__file__))) +NODE_ENGINE = os.path.join(FILE_DIRECTORY, "cwlNodeEngine.js") + + +def evaluate(config, input): + application = find_engine(config) + + default_context = { + "engineConfig": [], + "job": {}, + "context": None, + "outdir": None, + "tmpdir": None, + } + + new_input = default_context + new_input.update(input) + + sp = subprocess.Popen([application, NODE_ENGINE], + shell=False, + close_fds=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + + (stdoutdata, stderrdata) = sp.communicate(json.dumps(new_input) + "\n\n") + if sp.returncode != 0: + args = (json.dumps(new_input, indent=4), stdoutdata, stderrdata) + message = "Expression engine returned non-zero exit code on evaluation of\n%s%s%s" % args + raise Exception(message) + + return json.loads(stdoutdata) diff --git a/lib/galaxy/tools/expressions/sandbox.py b/lib/galaxy/tools/expressions/sandbox.py new file mode 100644 index 00000000000..a3646e100d4 --- /dev/null +++ b/lib/galaxy/tools/expressions/sandbox.py @@ -0,0 +1,151 @@ +import subprocess +import json +import threading + +from .util import find_engine + + +class JavascriptException(Exception): + pass + + +def execjs(config, js, jslib): + application = find_engine(config) + try: + nodejs = subprocess.Popen([application], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except OSError as e: + if e.errno == 2: + nodejs = subprocess.Popen(["docker", "run", + "--attach=STDIN", "--attach=STDOUT", "--attach=STDERR", + "--interactive", + "--rm", + "commonworkflowlanguage/nodejs-engine", "nodejs"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + else: + raise + + fn = "\"use strict\";%s\n(function()%s)()" % (jslib, js if isinstance(js, basestring) and len(js) > 1 and js[0] == '{' else ("{return (%s);}" % js)) + script = "console.log(JSON.stringify(require(\"vm\").runInNewContext(%s, {})));\n" % json.dumps(fn) + + def term(): + try: + nodejs.terminate() + except OSError: + pass + + # Time out after 5 seconds + tm = threading.Timer(5, term) + tm.start() + + stdoutdata, stderrdata = nodejs.communicate(script) + tm.cancel() + + if nodejs.returncode != 0: + raise JavascriptException("Returncode was: %s\nscript was: %s\nstdout was: '%s'\nstderr was: '%s'\n" % (nodejs.returncode, script, stdoutdata, stderrdata)) + else: + return json.loads(stdoutdata) + + +class SubstitutionError(Exception): + pass + + +DEFAULT = 0 +DOLLAR = 1 +PAREN = 2 +BRACE = 3 +SINGLE_QUOTE = 4 +DOUBLE_QUOTE = 5 +BACKSLASH = 6 + + +def scanner(scan): + + i = 0 + stack = [DEFAULT] + start = 0 + while i < len(scan): + state = stack[-1] + c = scan[i] + + if state == DEFAULT: + if c == '$': + stack.append(DOLLAR) + elif c == '\\': + stack.append(BACKSLASH) + elif state == BACKSLASH: + stack.pop() + if stack[-1] == DEFAULT: + return [i - 1, i + 1] + elif state == DOLLAR: + if c == '(': + start = i - 1 + stack.append(PAREN) + elif c == '{': + start = i - 1 + stack.append(BRACE) + elif state == PAREN: + if c == '(': + stack.append(PAREN) + elif c == ')': + stack.pop() + if stack[-1] == DOLLAR: + return [start, i + 1] + elif c == "'": + stack.append(SINGLE_QUOTE) + elif c == '"': + stack.append(DOUBLE_QUOTE) + elif state == BRACE: + if c == '{': + stack.append(BRACE) + elif c == '}': + stack.pop() + if stack[-1] == DOLLAR: + return [start, i + 1] + elif c == "'": + stack.append(SINGLE_QUOTE) + elif c == '"': + stack.append(DOUBLE_QUOTE) + elif state == SINGLE_QUOTE: + if c == "'": + stack.pop() + elif c == '\\': + stack.append(BACKSLASH) + elif state == DOUBLE_QUOTE: + if c == '"': + stack.pop() + elif c == '\\': + stack.append(BACKSLASH) + i += 1 + + if len(stack) > 1: + raise SubstitutionError("Substitution error, unfinished block starting at position {}: {}".format(start, scan[start:])) + else: + return None + + +def interpolate(scan, jslib): + scan = scan.strip() + parts = [] + w = scanner(scan) + while w: + parts.append(scan[0:w[0]]) + + if scan[w[0]] == '$': + e = execjs(scan[w[0] + 1:w[1]], jslib) + if w[0] == 0 and w[1] == len(scan): + return e + leaf = json.dumps(e, sort_keys=True) + if leaf[0] == '"': + leaf = leaf[1:-1] + parts.append(leaf) + elif scan[w[0]] == '\\': + e = scan[w[1] - 1] + parts.append(e) + + scan = scan[w[1]:] + w = scanner(scan) + parts.append(scan) + return ''.join(parts) diff --git a/lib/galaxy/tools/expressions/util.py b/lib/galaxy/tools/expressions/util.py new file mode 100644 index 00000000000..b7ea1bf090a --- /dev/null +++ b/lib/galaxy/tools/expressions/util.py @@ -0,0 +1,13 @@ +import json +from galaxy.tools.deps.commands import which + + +def find_engine(config): + nodejs_path = getattr(config, "nodejs_path", None) + if nodejs_path is None: + nodejs_path = which("nodejs") or which("node") or None + return nodejs_path + + +def jshead(engine_config, root_vars): + return "\n".join(engine_config + ["var %s = %s;" % (k, json.dumps(v)) for k, v in root_vars.items()]) diff --git a/test/unit/tools/test_expression_basics.py b/test/unit/tools/test_expression_basics.py new file mode 100644 index 00000000000..51c7293ac3f --- /dev/null +++ b/test/unit/tools/test_expression_basics.py @@ -0,0 +1,8 @@ +from galaxy.tools.expressions import evaluate + + +def test_evaluate(): + input = { + "script": "{return 5;}" + } + assert evaluate(None, input) == 5 From f5c93e868b7be73f7ced5b7bba8b35b7e41728cc Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 19 Apr 2018 16:08:30 -0400 Subject: [PATCH 02/14] Implement expression tools and non-data tool outputs. PR #6925 introduced a GUI for connecting non-data (e.g. integer, boolean, color, etc..) workflow input parameter to tool input parameters (the backend for this was originally added in #1306). That ideally was just the beginning of work toward using such values in structured ways in workflows. This PR extends tool output handling to allow producing of non-data parameters. These can serve as a source for non-data values in workflows the same work workflow input parameters can. To make such values more easy to produce, this PR also introduces Galaxy expression tools - mirroring functionality regularly used in CWL. These small JavaScript-based tools that consume inputs just like a regular Galaxy tool but that produce dictionary of non-data values. I think these expressions will be maximally useful when paired with format 2 workflows once we allow users to load arbitrary tools (I make the case more in full here https://github.com/galaxyproject/galaxy/pull/7545#issuecomment-473894424), but I outline some potential uses there as well. Because there is always a checklist in my PR descriptions: - Tool definition language and plumbing and datatype for expressing expressions as jobs. - Allow connecting expression tools to parameters in workflows, will delay evaluation of workflow so calculated value - Example test expression tools for testing and demonstration. --- config/datatypes_conf.xml.sample | 1 + lib/galaxy/datatypes/text.py | 24 +++ lib/galaxy/tools/__init__.py | 69 ++++++- lib/galaxy/tools/evaluation.py | 2 +- lib/galaxy/tools/expressions/__init__.py | 8 + lib/galaxy/tools/expressions/script.py | 15 ++ lib/galaxy/tools/parameters/wrapped_json.py | 25 ++- lib/galaxy/tools/parser/interface.py | 5 + lib/galaxy/tools/parser/output_objects.py | 26 ++- lib/galaxy/tools/parser/xml.py | 49 ++++- lib/galaxy/tools/parser/yaml.py | 3 + lib/galaxy/tools/xsd/galaxy.xsd | 194 ++++++++++++------ lib/galaxy/workflow/run.py | 12 ++ lib/galaxy_ext/expressions/__init__.py | 0 lib/galaxy_ext/expressions/handle_job.py | 49 +++++ test/api/test_tools.py | 42 ++++ test/api/test_workflows.py | 23 ++- .../functional/tools/expression_forty_two.xml | 15 ++ .../tools/expression_log_line_count.xml | 14 ++ .../functional/tools/expression_parse_int.xml | 14 ++ test/functional/tools/samples_tool_conf.xml | 4 +- test/unit/jobs/test_expression_run.py | 48 +++++ test/unit/tools/test_parsing.py | 49 +++++ 23 files changed, 623 insertions(+), 68 deletions(-) create mode 100644 lib/galaxy/tools/expressions/script.py create mode 100644 lib/galaxy_ext/expressions/__init__.py create mode 100644 lib/galaxy_ext/expressions/handle_job.py create mode 100644 test/functional/tools/expression_forty_two.xml create mode 100644 test/functional/tools/expression_log_line_count.xml create mode 100644 test/functional/tools/expression_parse_int.xml create mode 100644 test/unit/jobs/test_expression_run.py diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index 76fdb0c7ee5..12f4fae5d2f 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -414,6 +414,7 @@ + diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index c2e71444cf1..c563c2d318c 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -106,6 +106,30 @@ class Json(Text): return "JSON file (%s)" % (nice_size(dataset.get_size())) +class ExpressionJson(Json): + """ Represents the non-data input or output to a tool or workflow. + """ + file_ext = "json" + MetadataElement(name="json_type", default=None, desc="JavaScript or JSON type of expression", readonly=True, visible=True, no_value=None) + + def set_meta(self, dataset, **kwd): + """ + """ + json_type = "null" + with open(dataset.file_name) as f: + obj = json.load(f) + if isinstance(obj, int): + json_type = "int" + elif isinstance(obj, float): + json_type = "float" + elif isinstance(obj, list): + json_type = "list" + elif isinstance(obj, dict): + json_type = "object" + + dataset.metadata.json_type = json_type + + @build_sniff_from_prefix class Ipynb(Json): file_ext = "ipynb" diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 19618bf4ca2..f63f33a94b1 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -63,6 +63,7 @@ from galaxy.tools.parameters.dataset_matcher import ( from galaxy.tools.parameters.grouping import Conditional, ConditionalWhen, Repeat, Section, UploadDataset from galaxy.tools.parameters.input_translation import ToolInputTranslator from galaxy.tools.parameters.meta import expand_meta_parameters +from galaxy.tools.parameters.wrapped_json import json_wrap from galaxy.tools.parser import ( get_tool_source, ToolOutputCollectionPart @@ -2188,6 +2189,72 @@ class OutputParameterJSONTool(Tool): out.close() +class ExpressionTool(Tool): + requires_js_runtime = True + tool_type = 'expression' + EXPRESSION_INPUTS_NAME = "_expression_inputs_.json" + + def parse_command(self, tool_source): + self.command = "cd ../; %s" % expressions.EXPRESSION_SCRIPT_CALL + self.interpreter = None + self._expression = tool_source.parse_expression().strip() + + def parse_outputs(self, tool_source): + # Setup self.outputs and self.output_collections + super(ExpressionTool, self).parse_outputs(tool_source) + + # Validate these outputs for expression tools. + if len(self.output_collections) != 0: + message = "Expression tools may not declare output collections at this time." + raise Exception(message) + for output in self.outputs.values(): + if not hasattr(output, "from_expression"): + message = "Expression tools may not declare output datasets at this time." + raise Exception(message) + + def exec_before_job(self, app, inp_data, out_data, param_dict=None): + super(ExpressionTool, self).exec_before_job(app, inp_data, out_data, param_dict=param_dict) + local_working_directory = param_dict["__local_working_directory__"] + expression_inputs_path = os.path.join(local_working_directory, ExpressionTool.EXPRESSION_INPUTS_NAME) + + outputs = [] + for i, (out_name, data) in enumerate(out_data.iteritems()): + output_def = self.outputs[out_name] + wrapped_data = param_dict.get(out_name) + file_name = str(wrapped_data) + + outputs.append(dict( + name=out_name, + from_expression=output_def.from_expression, + path=file_name, + )) + + if param_dict is None: + raise Exception("Internal error - param_dict is empty.") + + job = {} + json_wrap(self.inputs, param_dict, job, handle_files='OBJECT') + expression_inputs = { + 'job': job, + 'script': self._expression, + 'outputs': outputs, + } + expressions.write_evalute_script(os.path.join(local_working_directory)) + with open(expression_inputs_path, "w") as f: + json.dump(expression_inputs, f) + + def parse_environment_variables(self, tool_source): + """ Setup environment variable for inputs file. + """ + environmnt_variables_raw = super(ExpressionTool, self).parse_environment_variables(tool_source) + expression_script_inputs = dict( + name="GALAXY_EXPRESSION_INPUTS", + template=ExpressionTool.EXPRESSION_INPUTS_NAME, + ) + environmnt_variables_raw.append(expression_script_inputs) + return environmnt_variables_raw + + class DataSourceTool(OutputParameterJSONTool): """ Alternate implementation of Tool for data_source tools -- those that @@ -2934,7 +3001,7 @@ class FilterFromFileTool(DatabaseOperationTool): # Populate tool_type to ToolClass mappings tool_types = {} -for tool_class in [Tool, SetMetadataTool, OutputParameterJSONTool, +for tool_class in [Tool, SetMetadataTool, OutputParameterJSONTool, ExpressionTool, DataManagerTool, DataSourceTool, AsyncDataSourceTool, UnzipCollectionTool, ZipCollectionTool, MergeCollectionTool, RelabelFromFileTool, FilterFromFileTool, BuildListCollectionTool, ExtractDatasetCollectionTool, diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py index b244364ac6e..9e56df5e56c 100644 --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -380,7 +380,7 @@ class ToolEvaluator(object): param_dict['__tool_directory__'] = self.compute_environment.tool_directory() param_dict['__get_data_table_entry__'] = get_data_table_entry - + param_dict['__local_working_directory__'] = self.local_working_directory # We add access to app here, this allows access to app.config, etc param_dict['__app__'] = RawObjectWrapper(self.app) # More convienent access to app.config.new_file_path; we don't need to diff --git a/lib/galaxy/tools/expressions/__init__.py b/lib/galaxy/tools/expressions/__init__.py index b7519e5f57c..b731044825a 100644 --- a/lib/galaxy/tools/expressions/__init__.py +++ b/lib/galaxy/tools/expressions/__init__.py @@ -1,12 +1,20 @@ from .evaluation import evaluate from .sandbox import execjs, interpolate from .util import jshead, find_engine +from .script import ( + write_evalute_script, + EXPRESSION_SCRIPT_CALL, + EXPRESSION_SCRIPT_NAME, +) __all__ = ( 'evaluate', 'execjs', + 'EXPRESSION_SCRIPT_CALL', + 'EXPRESSION_SCRIPT_NAME', 'find_engine', 'interpolate', 'jshead', + 'write_evalute_script', ) diff --git a/lib/galaxy/tools/expressions/script.py b/lib/galaxy/tools/expressions/script.py new file mode 100644 index 00000000000..0cb18ae7464 --- /dev/null +++ b/lib/galaxy/tools/expressions/script.py @@ -0,0 +1,15 @@ +import os + +EXPRESSION_SCRIPT_NAME = "_evaluate_expression_.py" +EXPRESSION_SCRIPT_CALL = "python %s" % EXPRESSION_SCRIPT_NAME + + +def write_evalute_script(in_directory): + """ Responsible for writing the script that evaluates expressions + in Galaxy jobs. + """ + script = os.path.join(in_directory, EXPRESSION_SCRIPT_NAME) + with open(script, "w") as f: + f.write('from galaxy_ext.expressions.handle_job import run; run()') + + return script diff --git a/lib/galaxy/tools/parameters/wrapped_json.py b/lib/galaxy/tools/parameters/wrapped_json.py index c045c9595ef..69549943143 100644 --- a/lib/galaxy/tools/parameters/wrapped_json.py +++ b/lib/galaxy/tools/parameters/wrapped_json.py @@ -62,8 +62,14 @@ def _json_wrap_input(input, value, handle_files="skip"): json_value = _data_input_to_path(value) elif handle_files == "skip": return SKIP_INPUT - else: - raise NotImplementedError() + elif handle_files == "OBJECT": + if value: + if isinstance(value, list): + value = value[0] + return _hda_to_object(value) + else: + return None + raise NotImplementedError() elif input_type == "data_collection": if handle_files == "skip": return SKIP_INPUT @@ -88,6 +94,21 @@ def _json_wrap_input(input, value, handle_files="skip"): return json_value +def _hda_to_object(hda): + hda_dict = hda.to_dict() + metadata_dict = {} + + for key, value in hda_dict.items(): + if key.startswith("metadata_"): + metadata_dict[key[len("metadata_"):]] = value + + return { + 'file_ext': hda_dict['file_ext'], + 'name': hda_dict['name'], + 'metadata': metadata_dict, + } + + def _cast_if_not_none(value, cast_to, empty_to_none=False): # log.debug("value [%s], type[%s]" % (value, type(value))) if value is None or (empty_to_none and str(value) == ''): diff --git a/lib/galaxy/tools/parser/interface.py b/lib/galaxy/tools/parser/interface.py index 2cd31a1aa6f..bdc979dccfc 100644 --- a/lib/galaxy/tools/parser/interface.py +++ b/lib/galaxy/tools/parser/interface.py @@ -88,6 +88,11 @@ class ToolSource(object): """ Return string contianing command to run. """ + def parse_expression(self): + """ Return string contianing command to run. + """ + return None + @abstractmethod def parse_environment_variables(self): """ Return environment variable templates to expose. diff --git a/lib/galaxy/tools/parser/output_objects.py b/lib/galaxy/tools/parser/output_objects.py index bfe51934ff9..805335a8358 100644 --- a/lib/galaxy/tools/parser/output_objects.py +++ b/lib/galaxy/tools/parser/output_objects.py @@ -24,12 +24,13 @@ class ToolOutput(ToolOutputBase): (format, metadata_source, parent) """ - dict_collection_visible_keys = ['name', 'format', 'label', 'hidden'] + dict_collection_visible_keys = ['name', 'format', 'label', 'hidden', 'output_type'] def __init__(self, name, format=None, format_source=None, metadata_source=None, parent=None, label=None, filters=None, actions=None, hidden=False, implicit=False): super(ToolOutput, self).__init__(name, label=label, filters=filters, hidden=hidden) + self.output_type = "data" self.format = format self.format_source = format_source self.metadata_source = metadata_source @@ -70,6 +71,27 @@ class ToolOutput(ToolOutputBase): return as_dict +class ToolExpressionOutput(ToolOutputBase): + dict_collection_visible_keys = ('name', 'format', 'label', 'hidden', 'output_type') + + def __init__(self, name, output_type, from_expression, + label=None, filters=None, actions=None, hidden=False): + super(ToolExpressionOutput, self).__init__(name, label=label, filters=filters, hidden=hidden) + self.output_type = output_type # JSON type... + self.from_expression = from_expression + self.format = "expression.json" # galaxy.datatypes.text.ExpressionJson.file_ext + + self.format_source = None + self.metadata_source = None + self.parent = None + self.actions = actions + + # Initialize default values + self.change_format = [] + self.implicit = False + self.from_work_dir = None + + class ToolOutputCollection(ToolOutputBase): """ Represents a HistoryDatasetCollectionAssociation of output datasets produced @@ -85,6 +107,7 @@ class ToolOutputCollection(ToolOutputBase): """ + dict_collection_visible_keys = ('name', 'format', 'label', 'hidden', 'output_type') dict_collection_visible_keys = ['name', 'default_format', 'label', 'hidden', 'inherit_format', 'inherit_metadata'] @@ -102,6 +125,7 @@ class ToolOutputCollection(ToolOutputBase): inherit_metadata=False ): super(ToolOutputCollection, self).__init__(name, label=label, filters=filters, hidden=hidden) + self.output_type = "collection" self.collection = True self.default_format = default_format self.structure = structure diff --git a/lib/galaxy/tools/parser/xml.py b/lib/galaxy/tools/parser/xml.py index 997eb760406..4d67278c26f 100644 --- a/lib/galaxy/tools/parser/xml.py +++ b/lib/galaxy/tools/parser/xml.py @@ -22,6 +22,7 @@ from .interface import ( from .output_actions import ToolOutputActionGroup from .output_collection_def import dataset_collector_descriptions_from_elem from .output_objects import ( + ToolExpressionOutput, ToolOutput, ToolOutputCollection, ToolOutputCollectionStructure @@ -111,6 +112,12 @@ class XmlToolSource(ToolSource): command_el = self._command_el return ((command_el is not None) and command_el.text) or None + def parse_expression(self): + """ Return string contianing command to run. + """ + expression_el = self.root.find("expression") + return ((expression_el is not None) and expression_el.text) or None + def parse_environment_variables(self): environment_variables_el = self.root.find("environment_variables") if environment_variables_el is None: @@ -256,7 +263,12 @@ class XmlToolSource(ToolSource): for _ in out_elem.findall("data"): _parse(_) - for collection_elem in out_elem.findall("collection"): + def _parse_expression(output_elem, **kwds): + output_def = self._parse_expression_output(output_elem, tool, **kwds) + data_dict[output_def.name] = output_def + return output_def + + def _parse_collection(collection_elem): name = collection_elem.get("name") label = xml_text(collection_elem, "label") default_format = collection_elem.get("format", "data") @@ -312,6 +324,24 @@ class XmlToolSource(ToolSource): output_collection.outputs[output_name] = data output_collections[name] = output_collection + for out_child in out_elem.getchildren(): + if out_child.tag == "data": + _parse(out_child) + elif out_child.tag == "collection": + _parse_collection(out_child) + elif out_child.tag == "output": + output_type = out_child.get("type") + if output_type == "data": + _parse(out_child) + elif output_type == "collection": + out_child.attrib["type"] = out_child.get("collection_type") + out_child.attrib["type_source"] = out_child.get("collection_type_source") + _parse_collection(out_child) + else: + _parse_expression(out_child) + else: + log.warn("Unknown output tag encountered [%s]" % out_child.tag) + for output_def in data_dict.values(): outputs[output_def.name] = output_def return outputs, output_collections @@ -323,6 +353,7 @@ class XmlToolSource(ToolSource): default_format="data", default_format_source=None, default_metadata_source="", + expression_type=None, ): output = ToolOutput(data_elem.get("name")) output_format = data_elem.get("format", default_format) @@ -347,6 +378,22 @@ class XmlToolSource(ToolSource): output.dataset_collector_descriptions = dataset_collector_descriptions_from_elem(data_elem, legacy=self.legacy_defaults) return output + def _parse_expression_output(self, output_elem, tool, **kwds): + output_type = output_elem.get("type") + from_expression = output_elem.get("from") + output = ToolExpressionOutput( + output_elem.get("name"), + output_type, + from_expression, + ) + output.path = output_elem.get("value") + output.label = xml_text(output_elem, "label") + + output.hidden = string_as_bool(output_elem.get("hidden", "")) + output.actions = ToolOutputActionGroup(output, output_elem.find('actions')) + output.dataset_collector_descriptions = [] + return output + def parse_stdio(self): """ parse error handling from command and stdio tag diff --git a/lib/galaxy/tools/parser/yaml.py b/lib/galaxy/tools/parser/yaml.py index 63e642e5648..52ae9d4cb07 100644 --- a/lib/galaxy/tools/parser/yaml.py +++ b/lib/galaxy/tools/parser/yaml.py @@ -54,6 +54,9 @@ class YamlToolSource(ToolSource): def parse_command(self): return self.root_dict.get("command") + def parse_expression(self): + return self.root_dict.get("expression") + def parse_environment_variables(self): return [] diff --git a/lib/galaxy/tools/xsd/galaxy.xsd b/lib/galaxy/tools/xsd/galaxy.xsd index 17e3191df80..a298cc13dd1 100644 --- a/lib/galaxy/tools/xsd/galaxy.xsd +++ b/lib/galaxy/tools/xsd/galaxy.xsd @@ -67,7 +67,8 @@ the tool menu immediately following the hyperlink for the tool (based on the - + + @@ -2781,6 +2782,33 @@ Prior to Galaxy release 19.01 the stdio block has only been used for non-legacy + + + + + + + + + Type of expression defined by this expression block. The only current valid option is emca5.1 - which will evaluate the expression in a sandbox using node. The option still must be specified to allow a different default in the future. + + + + + + + + + + + + + + @@ -3684,6 +3712,7 @@ The default is ``galaxy.json``. + @@ -3725,6 +3754,71 @@ pipes or periods (e.g. ``.``).]]> + + + + + + + + + The short name for the output datatype. +The valid values for format can be found in +[/config/datatypes_conf.xml.sample](https://github.com/galaxyproject/galaxy/blob/dev/config/datatypes_conf.xml.sample) +(e.g. ``format="pdf"`` or ``format="fastqsanger"``). + + + + + Sets the source of element identifier to the specified input. +This only applies to collections that are mapped over a non-collection input and that have equivalent structures. If this references input elements in conditionals, this value should be qualified (e.g. ``cond|input`` instead of ``input`` if ``input`` is in a conditional with ``name="cond"``). + + + + + This copies the metadata information +from the tool's input dataset. This is particularly useful for interval data +types where the order of the columns is not set. + + + + + Relative path to a file produced by the +tool in its working directory. Output's contents are set to this file's +contents. + + + + + Boolean indicating whether to hide +dataset in the history view. (Default is ``false``.) + + + + + + + + This is the name of input collection or +dataset to derive "structure" of the output from (output element count and +identifiers). For instance, if the referenced input has three ordered items with +identifiers ``sample1``, ``sample2``, and ``sample3``. If this references input +elements in conditionals, this value should be qualified (e.g. ``cond|input`` instead +of ``input`` if ``input`` is in a conditional with ``name="cond"``). + + + + + If ``structured_like`` is set, inherit +format of outputs from format of corresponding input. + + + + - - - - - - - - The short name for the output datatype. -The valid values for format can be found in -[/config/datatypes_conf.xml.sample](https://github.com/galaxyproject/galaxy/blob/dev/config/datatypes_conf.xml.sample) -(e.g. ``format="pdf"`` or ``format="fastqsanger"``). - - - - - Sets the source of element identifier to the specified input. -This only applies to collections that are mapped over a non-collection input and that have equivalent structures. If this references input elements in conditionals, this value should be qualified (e.g. ``cond|input`` instead of ``input`` if ``input`` is in a conditional with ``name="cond"``). - - - - - This copies the metadata information -from the tool's input dataset. This is particularly useful for interval data -types where the order of the columns is not set. - - - - - Relative path to a file produced by the -tool in its working directory. Output's contents are set to this file's -contents. - - - - - Boolean indicating whether to hide -dataset in the history view. (Default is ``false``.) - - + @@ -3887,6 +3939,7 @@ Creating collections in tools is covered in-depth in + Collection type for output (e.g. ``paired``, ``list``, or ``list:list``). @@ -3898,24 +3951,44 @@ Creating collections in tools is covered in-depth in derive collection's type (e.g. ``collection_type``) from. - - - This is the name of input collection or -dataset to derive "structure" of the output from (output element count and -identifiers). For instance, if the referenced input has three ordered items with -identifiers ``sample1``, ``sample2``, and ``sample3``. If this references input -elements in conditionals, this value should be qualified (e.g. ``cond|input`` instead -of ``input`` if ``input`` is in a conditional with ``name="cond"``). - - - - - If ``structured_like`` is set, inherit -format of outputs from format of corresponding input. - - + + + + + + + + + + + + + Output type. This could be older more established Galaxy types (e.g. data and collection) - in which case the semantics of this largely reflect the corresponding ``data`` and ``collection`` tags. This could also be newer non-data types such as ``integer`` or ``boolean``. + + + + + In expression tools, use this to specify a dictionary value to populate this output from. The semantics may change for other expression types in the future. + + + + + Collection type for output (e.g. ``paired``, ``list``, or ``list:list``). + + + + + This is the name of input collection to +derive collection's type (e.g. ``collection_type``) from. + + + + + diff --git a/lib/galaxy/workflow/run.py b/lib/galaxy/workflow/run.py index b75bee7ecac..961a673e9f4 100644 --- a/lib/galaxy/workflow/run.py +++ b/lib/galaxy/workflow/run.py @@ -1,3 +1,4 @@ +import json import logging import uuid @@ -367,6 +368,17 @@ class WorkflowProgress(object): delayed_why = "dependent collection [%s] not yet populated with datasets" % replacement.id raise modules.DelayedWorkflowEvaluation(why=delayed_why) + + is_hda = isinstance(replacement, model.HistoryDatasetAssociation) + if not is_data and is_hda: + if replacement.is_ok: + with open(replacement.file_name, 'r') as f: + replacement = json.load(f) + elif replacement.is_pending: + raise modules.DelayedWorkflowEvaluation() + else: + raise modules.CancelWorkflowEvaluation() + return replacement def get_replacement_workflow_output(self, workflow_output): diff --git a/lib/galaxy_ext/expressions/__init__.py b/lib/galaxy_ext/expressions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/galaxy_ext/expressions/handle_job.py b/lib/galaxy_ext/expressions/handle_job.py new file mode 100644 index 00000000000..4cfc3b52dc4 --- /dev/null +++ b/lib/galaxy_ext/expressions/handle_job.py @@ -0,0 +1,49 @@ +""" +Execute an external process to evaluate expressions for Galaxy jobs. + +Galaxy should be importable on sys.path . +""" + +import json +import logging +import os +import sys + +# insert *this* galaxy before all others on sys.path +sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))) + +# ensure supported version +assert sys.version_info[:2] >= (2, 7) and sys.version_info[:2] <= (2, 7), 'Python version must be 2.7, this is: %s' % sys.version + +logging.basicConfig() +log = logging.getLogger(__name__) + +from galaxy.tools.expressions import evaluate + +try: + from cwltool import expression +except ImportError: + expression = None + + +def run(environment_path=None): + if expression is None: + raise Exception("Python library cwltool must available to evaluate expressions.") + + if environment_path is None: + environment_path = os.environ.get("GALAXY_EXPRESSION_INPUTS") + with open(environment_path, "r") as f: + raw_inputs = json.load(f) + + outputs = raw_inputs["outputs"] + inputs = raw_inputs.copy() + del inputs["outputs"] + + result = evaluate(None, inputs) + + for output in outputs: + path = output["path"] + from_expression = "$(" + output["from_expression"] + ")" + output_value = expression.interpolate(from_expression, result) + with open(path, "w") as f: + json.dump(output_value, f) diff --git a/test/api/test_tools.py b/test/api/test_tools.py index d1c1e275a80..d2900e5a8d0 100644 --- a/test/api/test_tools.py +++ b/test/api/test_tools.py @@ -2030,6 +2030,48 @@ class ToolsTestCase(api.ApiTestCase): output_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output) self.assertEqual(output_content.strip(), "123\n456\n456\n0ab") + @skip_without_tool("expression_forty_two") + def test_galaxy_expression_tool_simplest(self): + history_id = self.dataset_populator.new_history() + inputs = { + } + run_response = self._run( + "expression_forty_two", history_id, inputs + ) + self._assert_status_code_is(run_response, 200) + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + output_content = self.dataset_populator.get_history_dataset_content(history_id) + self.assertEqual(output_content, "42") + + @skip_without_tool("expression_parse_int") + def test_galaxy_expression_tool_simple(self): + history_id = self.dataset_populator.new_history() + inputs = { + 'input1': '7', + } + run_response = self._run( + "expression_parse_int", history_id, inputs + ) + self._assert_status_code_is(run_response, 200) + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + output_content = self.dataset_populator.get_history_dataset_content(history_id) + self.assertEqual(output_content, "7") + + @skip_without_tool("expression_log_line_count") + def test_galaxy_expression_metadata(self): + history_id = self.dataset_populator.new_history() + new_dataset1 = self.dataset_populator.new_dataset(history_id, content='1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n11\n12\n13\n14') + inputs = { + 'input1': dataset_to_param(new_dataset1), + } + run_response = self._run( + "expression_log_line_count", history_id, inputs + ) + self._assert_status_code_is(run_response, 200) + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + output_content = self.dataset_populator.get_history_dataset_content(history_id) + self.assertEqual(output_content, "3") + def __build_group_list(self, history_id): response = self.dataset_collection_populator.upload_collection(history_id, "list", elements=[ { diff --git a/test/api/test_workflows.py b/test/api/test_workflows.py index 61f2c558be9..5ab72f3341c 100644 --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -1965,7 +1965,7 @@ text_input: type: raw """, history_id=history_id, wait=True, assert_ok=False) - def test_run_with_text_connection(self): + def test_run_with_text_input_connection(self): with self.dataset_populator.test_history() as history_id: self._run_jobs(""" class: GalaxyWorkflow @@ -1996,6 +1996,27 @@ text_input: content = self.dataset_populator.get_history_dataset_content(history_id) self.assertEqual("chrX\t152691446\t152691471\tCCDS14735.1_cds_0_0_chrX_152691447_f\t0\t+\n", content) + def test_run_with_numeric_input_connection(self): + history_id = self.dataset_populator.new_history() + self._run_jobs(""" +class: GalaxyWorkflow +steps: +- label: forty_two + tool_id: expression_forty_two + state: {} +- label: consume_expression_parameter + tool_id: cheetah_casting + state: + floattest: 3.14 + inttest: + $link: forty_two#out1 +test_data: {} +""", history_id=history_id) + + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + content = self.dataset_populator.get_history_dataset_content(history_id) + self.assertEquals("43\n4.14\n", content) + @skip_without_tool('cat1') def test_workflow_rerun_with_use_cached_job(self): workflow = self.workflow_populator.load_workflow(name="test_for_run") diff --git a/test/functional/tools/expression_forty_two.xml b/test/functional/tools/expression_forty_two.xml new file mode 100644 index 00000000000..c30553f85fa --- /dev/null +++ b/test/functional/tools/expression_forty_two.xml @@ -0,0 +1,15 @@ + + Parse Int + + {return {'output': + 42}; + } + + + + + + + Produces the integer 42. + diff --git a/test/functional/tools/expression_log_line_count.xml b/test/functional/tools/expression_log_line_count.xml new file mode 100644 index 00000000000..26804a97127 --- /dev/null +++ b/test/functional/tools/expression_log_line_count.xml @@ -0,0 +1,14 @@ + + Log Lines + + {return {'output': Math.max(Math.round(Math.log(parseInt($job.input1.metadata.data_lines))), 1)};} + + + + + + + + + diff --git a/test/functional/tools/expression_parse_int.xml b/test/functional/tools/expression_parse_int.xml new file mode 100644 index 00000000000..df2e9b98436 --- /dev/null +++ b/test/functional/tools/expression_parse_int.xml @@ -0,0 +1,14 @@ + + Parse Int + + {return {'output': parseInt($job.input1)};} + + + + + + + + Parse an integer from text. + diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index 2409bd0dd4b..77d4f11b78d 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -147,8 +147,10 @@ + + + - diff --git a/test/unit/jobs/test_expression_run.py b/test/unit/jobs/test_expression_run.py new file mode 100644 index 00000000000..71f07a0b97c --- /dev/null +++ b/test/unit/jobs/test_expression_run.py @@ -0,0 +1,48 @@ +import json +import os +import tempfile +import shutil +import subprocess + +from galaxy.tools import expressions + +THIS_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) +TEST_DIRECTORY = os.path.join(THIS_DIRECTORY, os.path.pardir, os.path.pardir) +ROOT_DIRECTORY = os.path.join(TEST_DIRECTORY, os.path.pardir) +LIB_DIRECTORY = os.path.join(ROOT_DIRECTORY, "lib") + + +def test_run_simple(): + test_directory = tempfile.mkdtemp() + try: + environment_path = os.path.join(test_directory, "env.json") + environment = { + 'job': {'input1': '7'}, + 'outputs': [ + {'name': 'out1', 'from_expression': "output1", 'path': 'moo'} + ], + 'script': "{return {'output1': parseInt($job.input1)};}", + } + with open(environment_path, "w") as f: + json.dump(environment, f) + expressions.write_evalute_script( + test_directory, + ) + new_env = os.environ.copy() + if "PYTHONPATH" in new_env: + new_env['PYTHONPATH'] = "%s:%s" % (LIB_DIRECTORY, new_env["PYTHONPATH"]) + else: + new_env['PYTHONPATH'] = "%s" % (LIB_DIRECTORY) + new_env['GALAXY_EXPRESSION_INPUTS'] = environment_path + p = subprocess.Popen( + args=expressions.EXPRESSION_SCRIPT_CALL, + shell=True, + cwd=test_directory, + env=new_env, + ) + assert p.wait() == 0 + with open(os.path.join(test_directory, 'moo')) as f: + out_content = f.read() + assert out_content == '7', out_content + finally: + shutil.rmtree(test_directory) diff --git a/test/unit/tools/test_parsing.py b/test/unit/tools/test_parsing.py index a87b9734e0d..e442cd5ca21 100644 --- a/test/unit/tools/test_parsing.py +++ b/test/unit/tools/test_parsing.py @@ -92,6 +92,39 @@ tests: compare: sim_size """ +TOOL_EXPRESSION_XML_1 = """ + + Parse Int + + {return {'output': parseInt($job.input1)};} + + + + + + + + Parse an integer from text. + +""" + + +TOOL_EXPRESSION_YAML_1 = """ +class: GalaxyExpressionTool +name: "parse_int" +id: parse_int +version: 1.0.2 +expression: "{return {'output': parseInt($job.input1)};}" +inputs: + - name: input1 + label: Text to parse + type: text +outputs: + out1: + type: integer + from: "#output" +""" + class BaseLoaderTestCase(unittest.TestCase): @@ -119,6 +152,22 @@ class BaseLoaderTestCase(unittest.TestCase): return tool_source +class XmlExpressionLoaderTestCase(BaseLoaderTestCase): + source_file_name = "expression.xml" + source_contents = TOOL_EXPRESSION_XML_1 + + def test_expression(self): + assert self._tool_source.parse_expression().strip() == "{return {'output': parseInt($job.input1)};}" + + def test_tool_type(self): + assert self._tool_source.parse_tool_type() == "expression" + + +class YamlExpressionLoaderTestCase(BaseLoaderTestCase): + source_file_name = "expression.yml" + source_contents = TOOL_EXPRESSION_XML_1 + + class XmlLoaderTestCase(BaseLoaderTestCase): source_file_name = "bwa.xml" source_contents = TOOL_XML_1 From 4659dd8bc63d55e94d13391fc048b7d31aea3981 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 19 Mar 2019 17:21:15 -0400 Subject: [PATCH 03/14] cwltool requirements --- .../dependencies/pipfiles/default/Pipfile | 1 + .../default/pinned-dev-requirements.txt | 2 +- .../pipfiles/default/pinned-requirements.txt | 20 ++++++++++++++++++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile index b6ad3afea6d..abc4b2e76cd 100644 --- a/lib/galaxy/dependencies/pipfiles/default/Pipfile +++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile @@ -40,6 +40,7 @@ uWSGI = "*" pysam = "==0.15.2" bdbag = "==1.4.1" # 1.5.0 requires Python >=2.7.9 bleach = "*" +cwltool = "==1.0.20180721142728" "bz2file" = {version = "*", markers = "python_version < '3.3'"} ipaddress = {version = "*", markers = "python_version < '3.3'"} isa-rwval = "*" diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt index 0dc921a00ac..fef1dbf7b42 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-dev-requirements.txt @@ -23,7 +23,7 @@ more-itertools==5.0.0 ; python_version <= '2.7' nose==1.3.7 nosehtml==0.4.5 packaging==19.0 -pathlib2==2.3.3 ; python_version < '3.6' +pathlib2==2.3.2 ; python_version < '3' pathtools==0.1.2 pbr==5.1.3 pluggy==0.9.0 diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index a5da11c3af7..17d49415785 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -4,6 +4,7 @@ adal==1.2.1 amqp==2.4.2 appdirs==1.4.3 asn1crypto==0.24.0 +avro==1.8.1 ; python_version < '3' azure-common==1.1.14 azure-cosmosdb-nspkg==2.0.2 azure-cosmosdb-table==1.0.4 @@ -31,6 +32,7 @@ botocore==1.12.115 bunch==1.0.1 bx-python==0.8.2 bz2file==0.98 ; python_version < '3.3' +cachecontrol==0.11.7 cachetools==3.1.0 certifi==2019.3.9 cffi==1.12.2 @@ -42,6 +44,7 @@ cloudbridge==2.0.0 cmd2==0.8.9 contextlib2==0.5.5 ; python_version < '3.5' cryptography==2.6.1 +cwltool==1.0.20180721142728 debtcollector==1.21.0 decorator==4.3.2 deprecated==1.2.5 @@ -75,14 +78,18 @@ jsonpointer==2.0 jsonschema==2.6.0 keystoneauth1==3.13.1 kombu==4.4.0 +lockfile==0.12.2 +lxml==4.3.2 mako==1.0.7 markupsafe==1.1.1 mercurial==3.7.3 ; python_version < '3' +mistune==0.8.4 monotonic==1.5 msgpack==0.6.1 msrest==0.5.5 msrestazure==0.5.0 munch==2.3.2 +mypy-extensions==0.4.1 netaddr==0.7.19 netifaces==0.10.9 networkx==1.11 @@ -107,14 +114,17 @@ parsley==1.3 paste==3.0.8 pastedeploy==2.0.1 pastescript==3.1.0 +pathlib2==2.3.2 ; python_version < '3' pbr==5.1.3 prettytable==0.7.2 +prov==1.5.1 psutil==5.6.1 pulsar-galaxy-lib==0.8.3 pyasn1-modules==0.2.4 pyasn1==0.4.5 pycparser==2.19 pycryptodome==3.7.3 +pyeventsystem==0.1.0 pyinotify==0.9.6 ; sys_platform != 'win32' and sys_platform != 'darwin' and sys_platform != 'sunos5' pyjwt==1.7.1 pykwalify==1.7.0 @@ -136,6 +146,8 @@ python-openid==2.2.5 ; python_version < '3.0' python-swiftclient==3.6.0 pytz==2018.9 pyyaml==5.1 +rdflib-jsonld==0.4.0 +rdflib==4.2.2 repoze.lru==0.7 requests-oauthlib==1.2.0 requests-toolbelt==0.9.1 @@ -144,7 +156,12 @@ requestsexceptions==1.4.0 rfc3986==1.2.0 routes==2.4.1 rsa==4.0 +ruamel.ordereddict==0.4.13 ; platform_python_implementation == 'CPython' and python_version <= '2.7' +ruamel.yaml==0.15.89 s3transfer==0.2.0 +scandir==1.10.0 ; python_version < '3.5' +schema-salad==2.7.20181126142424 +shellescape==3.4.1 simplejson==3.16.0 six==1.11.0 social-auth-core[openidconnect]==3.1.0 @@ -157,7 +174,8 @@ subprocess32==3.5.3 ; python_version < '3.0' svgwrite==1.2.1 tempita==0.5.2 tenacity==4.12.0 -typing==3.6.6 ; python_version < '3.5' +typing-extensions==3.7.2 +typing==3.6.6 ; python_version < '3.6' tzlocal==1.5.1 unicodecsv==0.14.1 ; python_version < '3.0' uritemplate==3.0.0 From 5c80323dcb86bbcc952482584c23e232f93fd57d Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 17 Mar 2019 20:29:23 -0400 Subject: [PATCH 04/14] Rollback expression utilities for operation tools. Some extras in here for group and filter tool that never made it out of CWL branch and into Galaxy - clean those up for getting expression tools into Galaxy. --- lib/galaxy/tools/expressions/__init__.py | 6 +- lib/galaxy/tools/expressions/sandbox.py | 151 ----------------------- lib/galaxy/tools/expressions/util.py | 4 - 3 files changed, 1 insertion(+), 160 deletions(-) delete mode 100644 lib/galaxy/tools/expressions/sandbox.py diff --git a/lib/galaxy/tools/expressions/__init__.py b/lib/galaxy/tools/expressions/__init__.py index b731044825a..c44e4437b2a 100644 --- a/lib/galaxy/tools/expressions/__init__.py +++ b/lib/galaxy/tools/expressions/__init__.py @@ -1,6 +1,5 @@ from .evaluation import evaluate -from .sandbox import execjs, interpolate -from .util import jshead, find_engine +from .util import find_engine from .script import ( write_evalute_script, EXPRESSION_SCRIPT_CALL, @@ -10,11 +9,8 @@ from .script import ( __all__ = ( 'evaluate', - 'execjs', 'EXPRESSION_SCRIPT_CALL', 'EXPRESSION_SCRIPT_NAME', 'find_engine', - 'interpolate', - 'jshead', 'write_evalute_script', ) diff --git a/lib/galaxy/tools/expressions/sandbox.py b/lib/galaxy/tools/expressions/sandbox.py deleted file mode 100644 index a3646e100d4..00000000000 --- a/lib/galaxy/tools/expressions/sandbox.py +++ /dev/null @@ -1,151 +0,0 @@ -import subprocess -import json -import threading - -from .util import find_engine - - -class JavascriptException(Exception): - pass - - -def execjs(config, js, jslib): - application = find_engine(config) - try: - nodejs = subprocess.Popen([application], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - except OSError as e: - if e.errno == 2: - nodejs = subprocess.Popen(["docker", "run", - "--attach=STDIN", "--attach=STDOUT", "--attach=STDERR", - "--interactive", - "--rm", - "commonworkflowlanguage/nodejs-engine", "nodejs"], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - else: - raise - - fn = "\"use strict\";%s\n(function()%s)()" % (jslib, js if isinstance(js, basestring) and len(js) > 1 and js[0] == '{' else ("{return (%s);}" % js)) - script = "console.log(JSON.stringify(require(\"vm\").runInNewContext(%s, {})));\n" % json.dumps(fn) - - def term(): - try: - nodejs.terminate() - except OSError: - pass - - # Time out after 5 seconds - tm = threading.Timer(5, term) - tm.start() - - stdoutdata, stderrdata = nodejs.communicate(script) - tm.cancel() - - if nodejs.returncode != 0: - raise JavascriptException("Returncode was: %s\nscript was: %s\nstdout was: '%s'\nstderr was: '%s'\n" % (nodejs.returncode, script, stdoutdata, stderrdata)) - else: - return json.loads(stdoutdata) - - -class SubstitutionError(Exception): - pass - - -DEFAULT = 0 -DOLLAR = 1 -PAREN = 2 -BRACE = 3 -SINGLE_QUOTE = 4 -DOUBLE_QUOTE = 5 -BACKSLASH = 6 - - -def scanner(scan): - - i = 0 - stack = [DEFAULT] - start = 0 - while i < len(scan): - state = stack[-1] - c = scan[i] - - if state == DEFAULT: - if c == '$': - stack.append(DOLLAR) - elif c == '\\': - stack.append(BACKSLASH) - elif state == BACKSLASH: - stack.pop() - if stack[-1] == DEFAULT: - return [i - 1, i + 1] - elif state == DOLLAR: - if c == '(': - start = i - 1 - stack.append(PAREN) - elif c == '{': - start = i - 1 - stack.append(BRACE) - elif state == PAREN: - if c == '(': - stack.append(PAREN) - elif c == ')': - stack.pop() - if stack[-1] == DOLLAR: - return [start, i + 1] - elif c == "'": - stack.append(SINGLE_QUOTE) - elif c == '"': - stack.append(DOUBLE_QUOTE) - elif state == BRACE: - if c == '{': - stack.append(BRACE) - elif c == '}': - stack.pop() - if stack[-1] == DOLLAR: - return [start, i + 1] - elif c == "'": - stack.append(SINGLE_QUOTE) - elif c == '"': - stack.append(DOUBLE_QUOTE) - elif state == SINGLE_QUOTE: - if c == "'": - stack.pop() - elif c == '\\': - stack.append(BACKSLASH) - elif state == DOUBLE_QUOTE: - if c == '"': - stack.pop() - elif c == '\\': - stack.append(BACKSLASH) - i += 1 - - if len(stack) > 1: - raise SubstitutionError("Substitution error, unfinished block starting at position {}: {}".format(start, scan[start:])) - else: - return None - - -def interpolate(scan, jslib): - scan = scan.strip() - parts = [] - w = scanner(scan) - while w: - parts.append(scan[0:w[0]]) - - if scan[w[0]] == '$': - e = execjs(scan[w[0] + 1:w[1]], jslib) - if w[0] == 0 and w[1] == len(scan): - return e - leaf = json.dumps(e, sort_keys=True) - if leaf[0] == '"': - leaf = leaf[1:-1] - parts.append(leaf) - elif scan[w[0]] == '\\': - e = scan[w[1] - 1] - parts.append(e) - - scan = scan[w[1]:] - w = scanner(scan) - parts.append(scan) - return ''.join(parts) diff --git a/lib/galaxy/tools/expressions/util.py b/lib/galaxy/tools/expressions/util.py index b7ea1bf090a..83f954a2ba3 100644 --- a/lib/galaxy/tools/expressions/util.py +++ b/lib/galaxy/tools/expressions/util.py @@ -7,7 +7,3 @@ def find_engine(config): if nodejs_path is None: nodejs_path = which("nodejs") or which("node") or None return nodejs_path - - -def jshead(engine_config, root_vars): - return "\n".join(engine_config + ["var %s = %s;" % (k, json.dumps(v)) for k, v in root_vars.items()]) From c2e67e3270408dfe85c512d3f25f8506c16d61e7 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 17 Mar 2019 20:30:40 -0400 Subject: [PATCH 05/14] flake imports --- lib/galaxy/tools/__init__.py | 2 +- lib/galaxy/tools/expressions/__init__.py | 4 ++-- lib/galaxy/tools/expressions/util.py | 1 - lib/galaxy_ext/expressions/handle_job.py | 14 +++++--------- test/unit/jobs/test_expression_run.py | 2 +- 5 files changed, 9 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index f63f33a94b1..60d6289f937 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -29,6 +29,7 @@ from galaxy.managers.jobs import JobSearch from galaxy.metadata import get_metadata_compute_strategy from galaxy.model.tags import GalaxyTagHandler from galaxy.queue_worker import send_control_task +from galaxy.tools import expressions from galaxy.tools.actions import DefaultToolAction from galaxy.tools.actions.data_manager import DataManagerToolAction from galaxy.tools.actions.data_source import DataSourceToolAction @@ -36,7 +37,6 @@ from galaxy.tools.actions.model_operations import ModelOperationToolAction from galaxy.tools.deps import ( CachedDependencyManager, ) -from galaxy.tools import expressions from galaxy.tools.fetcher import ToolLocationFetcher from galaxy.tools.parameters import ( check_param, diff --git a/lib/galaxy/tools/expressions/__init__.py b/lib/galaxy/tools/expressions/__init__.py index c44e4437b2a..bf20411a392 100644 --- a/lib/galaxy/tools/expressions/__init__.py +++ b/lib/galaxy/tools/expressions/__init__.py @@ -1,10 +1,10 @@ from .evaluation import evaluate -from .util import find_engine from .script import ( - write_evalute_script, EXPRESSION_SCRIPT_CALL, EXPRESSION_SCRIPT_NAME, + write_evalute_script, ) +from .util import find_engine __all__ = ( diff --git a/lib/galaxy/tools/expressions/util.py b/lib/galaxy/tools/expressions/util.py index 83f954a2ba3..93531186fc9 100644 --- a/lib/galaxy/tools/expressions/util.py +++ b/lib/galaxy/tools/expressions/util.py @@ -1,4 +1,3 @@ -import json from galaxy.tools.deps.commands import which diff --git a/lib/galaxy_ext/expressions/handle_job.py b/lib/galaxy_ext/expressions/handle_job.py index 4cfc3b52dc4..29fc11f83bc 100644 --- a/lib/galaxy_ext/expressions/handle_job.py +++ b/lib/galaxy_ext/expressions/handle_job.py @@ -3,7 +3,6 @@ Execute an external process to evaluate expressions for Galaxy jobs. Galaxy should be importable on sys.path . """ - import json import logging import os @@ -12,19 +11,16 @@ import sys # insert *this* galaxy before all others on sys.path sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))) -# ensure supported version -assert sys.version_info[:2] >= (2, 7) and sys.version_info[:2] <= (2, 7), 'Python version must be 2.7, this is: %s' % sys.version - -logging.basicConfig() -log = logging.getLogger(__name__) - -from galaxy.tools.expressions import evaluate - try: from cwltool import expression except ImportError: expression = None +from galaxy.tools.expressions import evaluate + +logging.basicConfig() +log = logging.getLogger(__name__) + def run(environment_path=None): if expression is None: diff --git a/test/unit/jobs/test_expression_run.py b/test/unit/jobs/test_expression_run.py index 71f07a0b97c..c23455d6e07 100644 --- a/test/unit/jobs/test_expression_run.py +++ b/test/unit/jobs/test_expression_run.py @@ -1,8 +1,8 @@ import json import os -import tempfile import shutil import subprocess +import tempfile from galaxy.tools import expressions From f579066655c7ce3bd7198446f97e3214a08228b2 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 19 Mar 2019 17:16:35 -0400 Subject: [PATCH 06/14] Python 3 fixes for galaxy.tools.expressions.evaluation. --- lib/galaxy/tools/expressions/evaluation.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/tools/expressions/evaluation.py b/lib/galaxy/tools/expressions/evaluation.py index 89becc1f862..54066d5094b 100644 --- a/lib/galaxy/tools/expressions/evaluation.py +++ b/lib/galaxy/tools/expressions/evaluation.py @@ -27,11 +27,12 @@ def evaluate(config, input): close_fds=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) - - (stdoutdata, stderrdata) = sp.communicate(json.dumps(new_input) + "\n\n") + input_str = json.dumps(new_input) + "\n\n" + input_bytes = input_str.encode("utf-8") + (stdoutdata, stderrdata) = sp.communicate(input_bytes) if sp.returncode != 0: args = (json.dumps(new_input, indent=4), stdoutdata, stderrdata) message = "Expression engine returned non-zero exit code on evaluation of\n%s%s%s" % args raise Exception(message) - return json.loads(stdoutdata) + return json.loads(stdoutdata.decode("utf-8")) From c4332f4e68480c57472949303daacdcceddb698a Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 19 Mar 2019 17:17:29 -0400 Subject: [PATCH 07/14] Error message clean up thanks to @mvdbeek. --- lib/galaxy/tools/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 60d6289f937..ff24a451eae 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -105,10 +105,7 @@ from .provided_metadata import parse_tool_provided_metadata log = logging.getLogger(__name__) REQUIRES_JS_RUNTIME_MESSAGE = ("The tool [%s] requires a nodejs runtime to execute " - "but node nor nodejs could be found on Galaxy's PATH and " - "no runtime was configured using the nodejs_path option in " - "galaxy.ini.") - + "but node or nodejs could not be found. Please contact the Galaxy adminstrator") HELP_UNINITIALIZED = threading.Lock() MODEL_TOOLS_PATH = os.path.abspath(os.path.dirname(__file__)) From f427a74dd97f28a93f036a8c1ef963fa18e8618c Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 19 Mar 2019 17:19:52 -0400 Subject: [PATCH 08/14] More changes to expression tool commit thanks to @mvdbeek. --- lib/galaxy/tools/parameters/wrapped_json.py | 3 ++- lib/galaxy_ext/expressions/handle_job.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tools/parameters/wrapped_json.py b/lib/galaxy/tools/parameters/wrapped_json.py index 69549943143..f9c90d4abe1 100644 --- a/lib/galaxy/tools/parameters/wrapped_json.py +++ b/lib/galaxy/tools/parameters/wrapped_json.py @@ -69,7 +69,8 @@ def _json_wrap_input(input, value, handle_files="skip"): return _hda_to_object(value) else: return None - raise NotImplementedError() + else: + raise NotImplementedError() elif input_type == "data_collection": if handle_files == "skip": return SKIP_INPUT diff --git a/lib/galaxy_ext/expressions/handle_job.py b/lib/galaxy_ext/expressions/handle_job.py index 29fc11f83bc..680e16fad20 100644 --- a/lib/galaxy_ext/expressions/handle_job.py +++ b/lib/galaxy_ext/expressions/handle_job.py @@ -24,7 +24,7 @@ log = logging.getLogger(__name__) def run(environment_path=None): if expression is None: - raise Exception("Python library cwltool must available to evaluate expressions.") + raise Exception("Python library cwltool must be available to evaluate expressions.") if environment_path is None: environment_path = os.environ.get("GALAXY_EXPRESSION_INPUTS") From 98738d80aa259c0158310a6bbcabec7311e2afb1 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 20 Mar 2019 10:02:18 +0100 Subject: [PATCH 09/14] Raise exception when node is not on PATH --- lib/galaxy/tools/__init__.py | 6 ++++-- lib/galaxy/tools/expressions/util.py | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index ff24a451eae..e5ee2c7dc72 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -753,8 +753,10 @@ class Tool(Dictifiable): mod = __import__(module, globals(), locals(), [cls]) self.tool_action = getattr(mod, cls)() if getattr(self.tool_action, "requires_js_runtime", False): - if expressions.find_engine(self.app.config) is None: - message = REQUIRES_JS_RUNTIME_MESSAGE % self.tool_id + try: + expressions.find_engine(self.app.config) + except Exception: + message = REQUIRES_JS_RUNTIME_MESSAGE % self.tool_id or self.tool_uuid raise Exception(message) # Tests self.__parse_tests(tool_source) diff --git a/lib/galaxy/tools/expressions/util.py b/lib/galaxy/tools/expressions/util.py index 93531186fc9..082ed3c916f 100644 --- a/lib/galaxy/tools/expressions/util.py +++ b/lib/galaxy/tools/expressions/util.py @@ -5,4 +5,6 @@ def find_engine(config): nodejs_path = getattr(config, "nodejs_path", None) if nodejs_path is None: nodejs_path = which("nodejs") or which("node") or None + if nodejs_path is None: + raise Exception("nodejs or node not found on PATH") return nodejs_path From 5e778f4812290cf9c9827b606dcd32fa2f6e2e5a Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 20 Mar 2019 10:02:56 +0100 Subject: [PATCH 10/14] Install node as part of common_startup.sh --- scripts/common_startup.sh | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index f224ed910b1..e5dc5d0935c 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -224,14 +224,23 @@ if [ $SKIP_CLIENT_BUILD -eq 0 ]; then fi fi +# Install node if not installed +if [ -n "$VIRTUAL_ENV" ]; then + if ! in_venv "$(command -v node)" || [ "$(node --version)" != "v$NODE_VERSION" ]; then + echo "Installing node into $VIRTUAL_ENV with nodeenv." + nodeenv -n $NODE_VERSION -p + fi +elif [ -n "$CONDA_DEFAULT_ENV" -a -n "$CONDA_EXE" ]; then + if ! in_conda_env "$(command -v node)"; then + echo "Installing node into '$CONDA_DEFAULT_ENV' Conda environment with conda." + $CONDA_EXE install --yes --override-channels --channel conda-forge --channel defaults --name $CONDA_DEFAULT_ENV node=$NODE_VERSION + fi +fi + # Build client if necessary. if [ $SKIP_CLIENT_BUILD -eq 0 ]; then # Ensure dependencies are installed if [ -n "$VIRTUAL_ENV" ]; then - if ! in_venv "$(command -v node)" || [ "$(node --version)" != "v$NODE_VERSION" ]; then - echo "Installing node into $VIRTUAL_ENV with nodeenv." - nodeenv -n $NODE_VERSION -p - fi if ! in_venv "$(command -v yarn)"; then echo "Installing yarn into $VIRTUAL_ENV with npm." npm install --global yarn From 552942fca9f4b6e74d74a1593a28ec274a24bdfe Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 20 Mar 2019 11:12:51 +0100 Subject: [PATCH 11/14] Fix enumerating over output There's no iteritems in python 3. We could use six.iteritems which calls iteritems on py2 and items on py3, but I don't expect these dicts to be large enough that there would be any noticeable difference. --- lib/galaxy/tools/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index e5ee2c7dc72..aaccb3604fa 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2217,7 +2217,7 @@ class ExpressionTool(Tool): expression_inputs_path = os.path.join(local_working_directory, ExpressionTool.EXPRESSION_INPUTS_NAME) outputs = [] - for i, (out_name, data) in enumerate(out_data.iteritems()): + for i, (out_name, data) in enumerate(out_data.items()): output_def = self.outputs[out_name] wrapped_data = param_dict.get(out_name) file_name = str(wrapped_data) From 5e8cbcd5b4e39809fe0f83547ecf062d6d01c6eb Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 20 Mar 2019 11:14:56 +0100 Subject: [PATCH 12/14] Allow connecting ExpressionTool outputs to inputs of correct type This means we can now connect these tools in the workflow editor. --- lib/galaxy/workflow/modules.py | 4 ++++ test/unit/workflows/test_modules.py | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 06e02a0678b..ba5edbd20ea 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -45,6 +45,7 @@ from galaxy.tools.parameters.basic import ( ) from galaxy.tools.parameters.history_query import HistoryQuery from galaxy.tools.parameters.wrapped import make_dict_copy +from galaxy.tools.parser.output_objects import ToolExpressionOutput from galaxy.util.bunch import Bunch from galaxy.util.json import safe_loads from galaxy.util.odict import odict @@ -937,6 +938,8 @@ class ToolModule(WorkflowModule): if filter_output(tool_output, self.state.inputs): continue extra_kwds = {} + if isinstance(tool_output, ToolExpressionOutput): + extra_kwds['parameter'] = True if tool_output.collection: extra_kwds["collection"] = True collection_type = tool_output.structure.collection_type @@ -973,6 +976,7 @@ class ToolModule(WorkflowModule): dict( name=name, extensions=formats, + type=tool_output.output_type, **extra_kwds ) ) diff --git a/test/unit/workflows/test_modules.py b/test/unit/workflows/test_modules.py index 921596b8d9f..4ad43a127f2 100644 --- a/test/unit/workflows/test_modules.py +++ b/test/unit/workflows/test_modules.py @@ -452,7 +452,8 @@ def __mock_tool( format_source=None, change_format=[], filters=[], - label=None)}, + label=None, + output_type='data')}, params_from_strings=mock.Mock(), check_and_update_param_values=mock.Mock(), to_json=_to_json From c80b80d3cf96b0137e172e460f63326e34ef8591 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 20 Mar 2019 12:25:14 +0100 Subject: [PATCH 13/14] Fix test_workflow_resume_with_mapped_over_input test self.dataset_populator._get_contents_request(history_id=history_id).json() returns the dataset collection as the first element. We do need a HDA here though, so we'd just randomly pick a HDA with the HDCA ID. So if that dataset was deleted by chance the test would fail. Ouch. --- test/api/test_workflows.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/api/test_workflows.py b/test/api/test_workflows.py index 5ab72f3341c..e35e72aba2d 100644 --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -791,12 +791,14 @@ test_data: """, history_id=history_id, assert_ok=False, wait=False) self.wait_for_invocation_and_jobs(history_id, job_summary.workflow_id, job_summary.invocation_id, assert_ok=False) history_contents = self.dataset_populator._get_contents_request(history_id=history_id).json() + first_input = history_contents[1] + assert first_input['history_content_type'] == 'dataset' paused_dataset = history_contents[-1] failed_dataset = self.dataset_populator.get_history_dataset_details(history_id, hid=5, assert_ok=False) assert paused_dataset['state'] == 'paused', paused_dataset assert failed_dataset['state'] == 'error', failed_dataset inputs = {"input1": {'values': [{'src': 'hda', - 'id': history_contents[0]['id']}] + 'id': first_input['id']}] }, "failbool": "false", "rerun_remap_job_id": failed_dataset['creating_job']} From ee078d315c67095eaa68d19b5a1afd5640059189 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 20 Mar 2019 16:56:49 -0400 Subject: [PATCH 14/14] Fix expression tool type parsing and docs. Thanks to @martenson for the catch. --- lib/galaxy/tools/parser/xml.py | 7 ++++++- lib/galaxy/tools/xsd/galaxy.xsd | 4 ++-- test/functional/tools/expression_forty_two.xml | 2 +- test/functional/tools/expression_log_line_count.xml | 2 +- test/functional/tools/expression_parse_int.xml | 2 +- test/unit/tools/test_parsing.py | 2 +- 6 files changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/tools/parser/xml.py b/lib/galaxy/tools/parser/xml.py index 4d67278c26f..c142c97745d 100644 --- a/lib/galaxy/tools/parser/xml.py +++ b/lib/galaxy/tools/parser/xml.py @@ -116,7 +116,12 @@ class XmlToolSource(ToolSource): """ Return string contianing command to run. """ expression_el = self.root.find("expression") - return ((expression_el is not None) and expression_el.text) or None + if expression_el is not None: + expression_type = expression_el.get("type") + if expression_type != "ecma5.1": + raise Exception("Unknown expression type [%s] encountered" % expression_type) + return expression_el.text + return None def parse_environment_variables(self): environment_variables_el = self.root.find("environment_variables") diff --git a/lib/galaxy/tools/xsd/galaxy.xsd b/lib/galaxy/tools/xsd/galaxy.xsd index a298cc13dd1..0f59437410f 100644 --- a/lib/galaxy/tools/xsd/galaxy.xsd +++ b/lib/galaxy/tools/xsd/galaxy.xsd @@ -2794,7 +2794,7 @@ of "type" specified for this expression block. - Type of expression defined by this expression block. The only current valid option is emca5.1 - which will evaluate the expression in a sandbox using node. The option still must be specified to allow a different default in the future. + Type of expression defined by this expression block. The only current valid option is ecma5.1 - which will evaluate the expression in a sandbox using node. The option still must be specified to allow a different default in the future. @@ -2806,7 +2806,7 @@ of "type" specified for this expression block. - + diff --git a/test/functional/tools/expression_forty_two.xml b/test/functional/tools/expression_forty_two.xml index c30553f85fa..af6afd0d997 100644 --- a/test/functional/tools/expression_forty_two.xml +++ b/test/functional/tools/expression_forty_two.xml @@ -1,7 +1,7 @@ Parse Int - + {return {'output': 42}; } diff --git a/test/functional/tools/expression_log_line_count.xml b/test/functional/tools/expression_log_line_count.xml index 26804a97127..b5811769906 100644 --- a/test/functional/tools/expression_log_line_count.xml +++ b/test/functional/tools/expression_log_line_count.xml @@ -1,7 +1,7 @@ Log Lines - + {return {'output': Math.max(Math.round(Math.log(parseInt($job.input1.metadata.data_lines))), 1)};} diff --git a/test/functional/tools/expression_parse_int.xml b/test/functional/tools/expression_parse_int.xml index df2e9b98436..3e2080aac0c 100644 --- a/test/functional/tools/expression_parse_int.xml +++ b/test/functional/tools/expression_parse_int.xml @@ -1,7 +1,7 @@ Parse Int - + {return {'output': parseInt($job.input1)};} diff --git a/test/unit/tools/test_parsing.py b/test/unit/tools/test_parsing.py index e442cd5ca21..772a4b98418 100644 --- a/test/unit/tools/test_parsing.py +++ b/test/unit/tools/test_parsing.py @@ -95,7 +95,7 @@ tests: TOOL_EXPRESSION_XML_1 = """ Parse Int - + {return {'output': parseInt($job.input1)};}