Library methods for dealing with sandboxed JavaScript expressions.

Based on work by Peter Amstutz in cwltool (https://github.com/common-workflow-language/cwltool).

lib/galaxy/config.py

@@ -313,6 +313,7 @@ class Configuration(object):
             log.warning("preserve_python_environment set to unknown value [%s], defaulting to legacy_only")
             preserve_python_environment = "legacy_only"
         self.preserve_python_environment = preserve_python_environment
+        self.nodejs_path = kwargs.get("nodejs_path", None)
         # Older default container cache path, I don't think anyone is using it anymore and it wasn't documented - we
         # should probably drop the backward compatiblity to save the path check.
         self.container_image_cache_path = self.resolve_path(kwargs.get("container_image_cache_path", "database/container_images"))

lib/galaxy/tools/__init__.py

@@ -36,6 +36,7 @@ from galaxy.tools.actions.model_operations import ModelOperationToolAction
 from galaxy.tools.deps import (
     CachedDependencyManager,
 )
+from galaxy.tools import expressions
 from galaxy.tools.fetcher import ToolLocationFetcher
 from galaxy.tools.parameters import (
     check_param,
@@ -102,6 +103,12 @@ from .provided_metadata import parse_tool_provided_metadata
 
 log = logging.getLogger(__name__)
 
+REQUIRES_JS_RUNTIME_MESSAGE = ("The tool [%s] requires a nodejs runtime to execute "
+                               "but node nor nodejs could be found on Galaxy's PATH and "
+                               "no runtime was configured using the nodejs_path option in "
+                               "galaxy.ini.")
+
+
 HELP_UNINITIALIZED = threading.Lock()
 MODEL_TOOLS_PATH = os.path.abspath(os.path.dirname(__file__))
 # Tools that require Galaxy's Python environment to be preserved.
@@ -747,6 +754,10 @@ class Tool(Dictifiable):
             module, cls = action
             mod = __import__(module, globals(), locals(), [cls])
             self.tool_action = getattr(mod, cls)()
+            if getattr(self.tool_action, "requires_js_runtime", False):
+                if expressions.find_engine(self.app.config) is None:
+                    message = REQUIRES_JS_RUNTIME_MESSAGE % self.tool_id
+                    raise Exception(message)
         # Tests
         self.__parse_tests(tool_source)
 

lib/galaxy/tools/expressions/__init__.py

@@ -0,0 +1,12 @@
+from .evaluation import evaluate
+from .sandbox import execjs, interpolate
+from .util import jshead, find_engine
+
+
+__all__ = (
+    'evaluate',
+    'execjs',
+    'find_engine',
+    'interpolate',
+    'jshead',
+)

lib/galaxy/tools/expressions/cwlNodeEngine.js

@@ -0,0 +1,46 @@
+#!/usr/bin/env nodejs
+
+"use strict";
+
+process.stdin.setEncoding('utf8');
+
+var incoming = "";
+
+process.stdin.on('readable', function() {
+  var chunk = process.stdin.read();
+    if (chunk !== null) {
+        incoming += chunk;
+    }
+});
+
+process.stdin.on('end', function() {
+    var j = JSON.parse(incoming);
+    var exp = ""
+
+    if (j.script[0] == "{") {
+        exp = "{return function()" + j.script + "();}";
+    }
+    else {
+        exp = "{return " + j.script + ";}";
+    }
+
+    var fn = '"use strict";\n';
+
+    if (j.engineConfig) {
+        for (var index = 0; index < j.engineConfig.length; ++index) {
+            fn += j.engineConfig[index] + "\n";
+        }
+    }
+
+    fn += "var $job = " + JSON.stringify(j.job) + ";\n";
+    fn += "var $self = " + JSON.stringify(j.context) + ";\n"
+
+    fn += "var $runtime = " + JSON.stringify(j.runtime) + ";\n"
+    fn += "var $tmpdir = " + JSON.stringify(j.tmpdir) + ";\n"
+    fn += "var $outdir = " + JSON.stringify(j.outdir) + ";\n"
+
+
+    fn += "(function()" + exp + ")()";
+
+    process.stdout.write(JSON.stringify(require("vm").runInNewContext(fn, {})));
+});

lib/galaxy/tools/expressions/evaluation.py

@@ -0,0 +1,37 @@
+import json
+import os
+import subprocess
+
+from .util import find_engine
+
+FILE_DIRECTORY = os.path.normpath(os.path.dirname(os.path.join(__file__)))
+NODE_ENGINE = os.path.join(FILE_DIRECTORY, "cwlNodeEngine.js")
+
+
+def evaluate(config, input):
+    application = find_engine(config)
+
+    default_context = {
+        "engineConfig": [],
+        "job": {},
+        "context": None,
+        "outdir": None,
+        "tmpdir": None,
+    }
+
+    new_input = default_context
+    new_input.update(input)
+
+    sp = subprocess.Popen([application, NODE_ENGINE],
+                          shell=False,
+                          close_fds=True,
+                          stdin=subprocess.PIPE,
+                          stdout=subprocess.PIPE)
+
+    (stdoutdata, stderrdata) = sp.communicate(json.dumps(new_input) + "\n\n")
+    if sp.returncode != 0:
+        args = (json.dumps(new_input, indent=4), stdoutdata, stderrdata)
+        message = "Expression engine returned non-zero exit code on evaluation of\n%s%s%s" % args
+        raise Exception(message)
+
+    return json.loads(stdoutdata)

lib/galaxy/tools/expressions/sandbox.py

@@ -0,0 +1,151 @@
+import subprocess
+import json
+import threading
+
+from .util import find_engine
+
+
+class JavascriptException(Exception):
+    pass
+
+
+def execjs(config, js, jslib):
+    application = find_engine(config)
+    try:
+        nodejs = subprocess.Popen([application], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    except OSError as e:
+        if e.errno == 2:
+            nodejs = subprocess.Popen(["docker", "run",
+                                       "--attach=STDIN", "--attach=STDOUT", "--attach=STDERR",
+                                       "--interactive",
+                                       "--rm",
+                                       "commonworkflowlanguage/nodejs-engine", "nodejs"],
+                                      stdin=subprocess.PIPE,
+                                      stdout=subprocess.PIPE,
+                                      stderr=subprocess.PIPE)
+        else:
+            raise
+
+    fn = "\"use strict\";%s\n(function()%s)()" % (jslib, js if isinstance(js, basestring) and len(js) > 1 and js[0] == '{' else ("{return (%s);}" % js))
+    script = "console.log(JSON.stringify(require(\"vm\").runInNewContext(%s, {})));\n" % json.dumps(fn)
+
+    def term():
+        try:
+            nodejs.terminate()
+        except OSError:
+            pass
+
+    # Time out after 5 seconds
+    tm = threading.Timer(5, term)
+    tm.start()
+
+    stdoutdata, stderrdata = nodejs.communicate(script)
+    tm.cancel()
+
+    if nodejs.returncode != 0:
+        raise JavascriptException("Returncode was: %s\nscript was: %s\nstdout was: '%s'\nstderr was: '%s'\n" % (nodejs.returncode, script, stdoutdata, stderrdata))
+    else:
+        return json.loads(stdoutdata)
+
+
+class SubstitutionError(Exception):
+    pass
+
+
+DEFAULT = 0
+DOLLAR = 1
+PAREN = 2
+BRACE = 3
+SINGLE_QUOTE = 4
+DOUBLE_QUOTE = 5
+BACKSLASH = 6
+
+
+def scanner(scan):
+
+    i = 0
+    stack = [DEFAULT]
+    start = 0
+    while i < len(scan):
+        state = stack[-1]
+        c = scan[i]
+
+        if state == DEFAULT:
+            if c == '$':
+                stack.append(DOLLAR)
+            elif c == '\\':
+                stack.append(BACKSLASH)
+        elif state == BACKSLASH:
+            stack.pop()
+            if stack[-1] == DEFAULT:
+                return [i - 1, i + 1]
+        elif state == DOLLAR:
+            if c == '(':
+                start = i - 1
+                stack.append(PAREN)
+            elif c == '{':
+                start = i - 1
+                stack.append(BRACE)
+        elif state == PAREN:
+            if c == '(':
+                stack.append(PAREN)
+            elif c == ')':
+                stack.pop()
+                if stack[-1] == DOLLAR:
+                    return [start, i + 1]
+            elif c == "'":
+                stack.append(SINGLE_QUOTE)
+            elif c == '"':
+                stack.append(DOUBLE_QUOTE)
+        elif state == BRACE:
+            if c == '{':
+                stack.append(BRACE)
+            elif c == '}':
+                stack.pop()
+                if stack[-1] == DOLLAR:
+                    return [start, i + 1]
+            elif c == "'":
+                stack.append(SINGLE_QUOTE)
+            elif c == '"':
+                stack.append(DOUBLE_QUOTE)
+        elif state == SINGLE_QUOTE:
+            if c == "'":
+                stack.pop()
+            elif c == '\\':
+                stack.append(BACKSLASH)
+        elif state == DOUBLE_QUOTE:
+            if c == '"':
+                stack.pop()
+            elif c == '\\':
+                stack.append(BACKSLASH)
+        i += 1
+
+    if len(stack) > 1:
+        raise SubstitutionError("Substitution error, unfinished block starting at position {}: {}".format(start, scan[start:]))
+    else:
+        return None
+
+
+def interpolate(scan, jslib):
+    scan = scan.strip()
+    parts = []
+    w = scanner(scan)
+    while w:
+        parts.append(scan[0:w[0]])
+
+        if scan[w[0]] == '$':
+            e = execjs(scan[w[0] + 1:w[1]], jslib)
+            if w[0] == 0 and w[1] == len(scan):
+                return e
+            leaf = json.dumps(e, sort_keys=True)
+            if leaf[0] == '"':
+                leaf = leaf[1:-1]
+            parts.append(leaf)
+        elif scan[w[0]] == '\\':
+            e = scan[w[1] - 1]
+            parts.append(e)
+
+        scan = scan[w[1]:]
+        w = scanner(scan)
+    parts.append(scan)
+    return ''.join(parts)

lib/galaxy/tools/expressions/util.py

@@ -0,0 +1,13 @@
+import json
+from galaxy.tools.deps.commands import which
+
+
+def find_engine(config):
+    nodejs_path = getattr(config, "nodejs_path", None)
+    if nodejs_path is None:
+        nodejs_path = which("nodejs") or which("node") or None
+    return nodejs_path
+
+
+def jshead(engine_config, root_vars):
+    return "\n".join(engine_config + ["var %s = %s;" % (k, json.dumps(v)) for k, v in root_vars.items()])

test/unit/tools/test_expression_basics.py

@@ -0,0 +1,8 @@
+from galaxy.tools.expressions import evaluate
+
+
+def test_evaluate():
+    input = {
+        "script": "{return 5;}"
+    }
+    assert evaluate(None, input) == 5