diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index e529fddf706..bb1877b2d83 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -313,6 +313,7 @@ class Configuration(object): log.warning("preserve_python_environment set to unknown value [%s], defaulting to legacy_only") preserve_python_environment = "legacy_only" self.preserve_python_environment = preserve_python_environment + self.nodejs_path = kwargs.get("nodejs_path", None) # Older default container cache path, I don't think anyone is using it anymore and it wasn't documented - we # should probably drop the backward compatiblity to save the path check. self.container_image_cache_path = self.resolve_path(kwargs.get("container_image_cache_path", "database/container_images")) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 0e2c3da4abd..19618bf4ca2 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -36,6 +36,7 @@ from galaxy.tools.actions.model_operations import ModelOperationToolAction from galaxy.tools.deps import ( CachedDependencyManager, ) +from galaxy.tools import expressions from galaxy.tools.fetcher import ToolLocationFetcher from galaxy.tools.parameters import ( check_param, @@ -102,6 +103,12 @@ from .provided_metadata import parse_tool_provided_metadata log = logging.getLogger(__name__) +REQUIRES_JS_RUNTIME_MESSAGE = ("The tool [%s] requires a nodejs runtime to execute " + "but node nor nodejs could be found on Galaxy's PATH and " + "no runtime was configured using the nodejs_path option in " + "galaxy.ini.") + + HELP_UNINITIALIZED = threading.Lock() MODEL_TOOLS_PATH = os.path.abspath(os.path.dirname(__file__)) # Tools that require Galaxy's Python environment to be preserved. @@ -747,6 +754,10 @@ class Tool(Dictifiable): module, cls = action mod = __import__(module, globals(), locals(), [cls]) self.tool_action = getattr(mod, cls)() + if getattr(self.tool_action, "requires_js_runtime", False): + if expressions.find_engine(self.app.config) is None: + message = REQUIRES_JS_RUNTIME_MESSAGE % self.tool_id + raise Exception(message) # Tests self.__parse_tests(tool_source) diff --git a/lib/galaxy/tools/expressions/__init__.py b/lib/galaxy/tools/expressions/__init__.py new file mode 100644 index 00000000000..b7519e5f57c --- /dev/null +++ b/lib/galaxy/tools/expressions/__init__.py @@ -0,0 +1,12 @@ +from .evaluation import evaluate +from .sandbox import execjs, interpolate +from .util import jshead, find_engine + + +__all__ = ( + 'evaluate', + 'execjs', + 'find_engine', + 'interpolate', + 'jshead', +) diff --git a/lib/galaxy/tools/expressions/cwlNodeEngine.js b/lib/galaxy/tools/expressions/cwlNodeEngine.js new file mode 100644 index 00000000000..1129584977f --- /dev/null +++ b/lib/galaxy/tools/expressions/cwlNodeEngine.js @@ -0,0 +1,46 @@ +#!/usr/bin/env nodejs + +"use strict"; + +process.stdin.setEncoding('utf8'); + +var incoming = ""; + +process.stdin.on('readable', function() { + var chunk = process.stdin.read(); + if (chunk !== null) { + incoming += chunk; + } +}); + +process.stdin.on('end', function() { + var j = JSON.parse(incoming); + var exp = "" + + if (j.script[0] == "{") { + exp = "{return function()" + j.script + "();}"; + } + else { + exp = "{return " + j.script + ";}"; + } + + var fn = '"use strict";\n'; + + if (j.engineConfig) { + for (var index = 0; index < j.engineConfig.length; ++index) { + fn += j.engineConfig[index] + "\n"; + } + } + + fn += "var $job = " + JSON.stringify(j.job) + ";\n"; + fn += "var $self = " + JSON.stringify(j.context) + ";\n" + + fn += "var $runtime = " + JSON.stringify(j.runtime) + ";\n" + fn += "var $tmpdir = " + JSON.stringify(j.tmpdir) + ";\n" + fn += "var $outdir = " + JSON.stringify(j.outdir) + ";\n" + + + fn += "(function()" + exp + ")()"; + + process.stdout.write(JSON.stringify(require("vm").runInNewContext(fn, {}))); +}); diff --git a/lib/galaxy/tools/expressions/evaluation.py b/lib/galaxy/tools/expressions/evaluation.py new file mode 100644 index 00000000000..89becc1f862 --- /dev/null +++ b/lib/galaxy/tools/expressions/evaluation.py @@ -0,0 +1,37 @@ +import json +import os +import subprocess + +from .util import find_engine + +FILE_DIRECTORY = os.path.normpath(os.path.dirname(os.path.join(__file__))) +NODE_ENGINE = os.path.join(FILE_DIRECTORY, "cwlNodeEngine.js") + + +def evaluate(config, input): + application = find_engine(config) + + default_context = { + "engineConfig": [], + "job": {}, + "context": None, + "outdir": None, + "tmpdir": None, + } + + new_input = default_context + new_input.update(input) + + sp = subprocess.Popen([application, NODE_ENGINE], + shell=False, + close_fds=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + + (stdoutdata, stderrdata) = sp.communicate(json.dumps(new_input) + "\n\n") + if sp.returncode != 0: + args = (json.dumps(new_input, indent=4), stdoutdata, stderrdata) + message = "Expression engine returned non-zero exit code on evaluation of\n%s%s%s" % args + raise Exception(message) + + return json.loads(stdoutdata) diff --git a/lib/galaxy/tools/expressions/sandbox.py b/lib/galaxy/tools/expressions/sandbox.py new file mode 100644 index 00000000000..a3646e100d4 --- /dev/null +++ b/lib/galaxy/tools/expressions/sandbox.py @@ -0,0 +1,151 @@ +import subprocess +import json +import threading + +from .util import find_engine + + +class JavascriptException(Exception): + pass + + +def execjs(config, js, jslib): + application = find_engine(config) + try: + nodejs = subprocess.Popen([application], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except OSError as e: + if e.errno == 2: + nodejs = subprocess.Popen(["docker", "run", + "--attach=STDIN", "--attach=STDOUT", "--attach=STDERR", + "--interactive", + "--rm", + "commonworkflowlanguage/nodejs-engine", "nodejs"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + else: + raise + + fn = "\"use strict\";%s\n(function()%s)()" % (jslib, js if isinstance(js, basestring) and len(js) > 1 and js[0] == '{' else ("{return (%s);}" % js)) + script = "console.log(JSON.stringify(require(\"vm\").runInNewContext(%s, {})));\n" % json.dumps(fn) + + def term(): + try: + nodejs.terminate() + except OSError: + pass + + # Time out after 5 seconds + tm = threading.Timer(5, term) + tm.start() + + stdoutdata, stderrdata = nodejs.communicate(script) + tm.cancel() + + if nodejs.returncode != 0: + raise JavascriptException("Returncode was: %s\nscript was: %s\nstdout was: '%s'\nstderr was: '%s'\n" % (nodejs.returncode, script, stdoutdata, stderrdata)) + else: + return json.loads(stdoutdata) + + +class SubstitutionError(Exception): + pass + + +DEFAULT = 0 +DOLLAR = 1 +PAREN = 2 +BRACE = 3 +SINGLE_QUOTE = 4 +DOUBLE_QUOTE = 5 +BACKSLASH = 6 + + +def scanner(scan): + + i = 0 + stack = [DEFAULT] + start = 0 + while i < len(scan): + state = stack[-1] + c = scan[i] + + if state == DEFAULT: + if c == '$': + stack.append(DOLLAR) + elif c == '\\': + stack.append(BACKSLASH) + elif state == BACKSLASH: + stack.pop() + if stack[-1] == DEFAULT: + return [i - 1, i + 1] + elif state == DOLLAR: + if c == '(': + start = i - 1 + stack.append(PAREN) + elif c == '{': + start = i - 1 + stack.append(BRACE) + elif state == PAREN: + if c == '(': + stack.append(PAREN) + elif c == ')': + stack.pop() + if stack[-1] == DOLLAR: + return [start, i + 1] + elif c == "'": + stack.append(SINGLE_QUOTE) + elif c == '"': + stack.append(DOUBLE_QUOTE) + elif state == BRACE: + if c == '{': + stack.append(BRACE) + elif c == '}': + stack.pop() + if stack[-1] == DOLLAR: + return [start, i + 1] + elif c == "'": + stack.append(SINGLE_QUOTE) + elif c == '"': + stack.append(DOUBLE_QUOTE) + elif state == SINGLE_QUOTE: + if c == "'": + stack.pop() + elif c == '\\': + stack.append(BACKSLASH) + elif state == DOUBLE_QUOTE: + if c == '"': + stack.pop() + elif c == '\\': + stack.append(BACKSLASH) + i += 1 + + if len(stack) > 1: + raise SubstitutionError("Substitution error, unfinished block starting at position {}: {}".format(start, scan[start:])) + else: + return None + + +def interpolate(scan, jslib): + scan = scan.strip() + parts = [] + w = scanner(scan) + while w: + parts.append(scan[0:w[0]]) + + if scan[w[0]] == '$': + e = execjs(scan[w[0] + 1:w[1]], jslib) + if w[0] == 0 and w[1] == len(scan): + return e + leaf = json.dumps(e, sort_keys=True) + if leaf[0] == '"': + leaf = leaf[1:-1] + parts.append(leaf) + elif scan[w[0]] == '\\': + e = scan[w[1] - 1] + parts.append(e) + + scan = scan[w[1]:] + w = scanner(scan) + parts.append(scan) + return ''.join(parts) diff --git a/lib/galaxy/tools/expressions/util.py b/lib/galaxy/tools/expressions/util.py new file mode 100644 index 00000000000..b7ea1bf090a --- /dev/null +++ b/lib/galaxy/tools/expressions/util.py @@ -0,0 +1,13 @@ +import json +from galaxy.tools.deps.commands import which + + +def find_engine(config): + nodejs_path = getattr(config, "nodejs_path", None) + if nodejs_path is None: + nodejs_path = which("nodejs") or which("node") or None + return nodejs_path + + +def jshead(engine_config, root_vars): + return "\n".join(engine_config + ["var %s = %s;" % (k, json.dumps(v)) for k, v in root_vars.items()]) diff --git a/test/unit/tools/test_expression_basics.py b/test/unit/tools/test_expression_basics.py new file mode 100644 index 00000000000..51c7293ac3f --- /dev/null +++ b/test/unit/tools/test_expression_basics.py @@ -0,0 +1,8 @@ +from galaxy.tools.expressions import evaluate + + +def test_evaluate(): + input = { + "script": "{return 5;}" + } + assert evaluate(None, input) == 5