Merge branch 'dev' of github.com:galaxyproject/galaxy into dev

lib/galaxy/datatypes/data.py

@@ -8,11 +8,11 @@ import shutil
 import string
 import tempfile
 import zipfile
-from cgi import escape
 from inspect import isclass
 
 import six
 import webob.exc
+from markupsafe import escape
 
 from galaxy import util
 from galaxy.datatypes.metadata import MetadataElement  # import directly to maintain ease of use in Datatype class definitions

lib/galaxy/datatypes/genetics.py

@@ -15,8 +15,8 @@ import logging
 import os
 import re
 import sys
-from cgi import escape
 
+from markupsafe import escape
 from six.moves.urllib.parse import quote_plus
 
 from galaxy.datatypes import metadata

lib/galaxy/datatypes/isa.py

@@ -14,7 +14,6 @@ import re
 import shutil
 import sys
 import tempfile
-from cgi import escape
 
 # Imports isatab after turning off warnings inside logger settings to avoid pandas warning making uploads fail.
 logging.getLogger("isatools.isatab").setLevel(logging.ERROR)
@@ -22,6 +21,7 @@ from isatools import (
     isajson,
     isatab_meta
 )
+from markupsafe import escape
 
 from galaxy import util
 from galaxy.datatypes import data

lib/galaxy/datatypes/sequence.py

@@ -9,10 +9,10 @@ import re
 import string
 import subprocess
 import sys
-from cgi import escape
 from itertools import islice
 
 import bx.align.maf
+from markupsafe import escape
 
 from galaxy import util
 from galaxy.datatypes import metadata

lib/galaxy/datatypes/tabular.py

@@ -12,10 +12,10 @@ import shutil
 import subprocess
 import sys
 import tempfile
-from cgi import escape
 from json import dumps
 
 import pysam
+from markupsafe import escape
 
 from galaxy import util
 from galaxy.datatypes import binary, data, metadata

lib/galaxy/jobs/__init__.py

@@ -27,7 +27,7 @@ from galaxy import model, util
 from galaxy.datatypes import sniff
 from galaxy.exceptions import ObjectInvalid, ObjectNotFound
 from galaxy.jobs.actions.post import ActionBox
-from galaxy.jobs.mapper import JobRunnerMapper
+from galaxy.jobs.mapper import JobMappingException, JobRunnerMapper
 from galaxy.jobs.runners import BaseJobRunner, JobState
 from galaxy.metadata import get_metadata_compute_strategy
 from galaxy.objectstore import ObjectStorePopulator
@@ -994,6 +994,17 @@ class JobWrapper(HasResourceParameters):
         """
         job = self.get_job()
         self.sa_session.refresh(job)
+
+        # If this fail method is being called because a dynamic rule raised JobMappingException, the call to
+        # self.get_destination_configuration() below accesses self.job_destination and will just cause
+        # JobMappingException to be raised again.
+        try:
+            self.job_destination
+        except JobMappingException as exc:
+            log.debug("(%s) fail(): Job destination raised JobMappingException('%s'), caching fake '__fail__' "
+                      "destination for completion of fail method", self.get_id_tag(), str(exc.failure_message))
+            self.job_runner_mapper.cached_job_destination = JobDestination(id='__fail__')
+
         # if the job was deleted, don't fail it
         if not job.state == job.states.DELETED:
             # Check if the failure is due to an exception

lib/galaxy/jobs/mapper.py

@@ -1,10 +1,10 @@
 import importlib
-import inspect
 import logging
 
 import galaxy.jobs.rules
 from galaxy.jobs import stock_rules
 from galaxy.jobs.dynamic_tool_destination import map_tool_to_destination
+from galaxy.util.getargspec import getfullargspec
 from galaxy.util.submodules import import_submodules
 from .rule_helper import RuleHelper
 
@@ -56,7 +56,7 @@ class JobRunnerMapper(object):
             self.rules_module = importlib.import_module(module_name)
 
     def __invoke_expand_function(self, expand_function, destination):
-        function_arg_names = inspect.getargspec(expand_function).args
+        function_arg_names = getfullargspec(expand_function).args
         app = self.job_wrapper.app
         possible_args = {
             "job_id": self.job_wrapper.job_id,

lib/galaxy/jobs/runners/cli.py

@@ -181,6 +181,8 @@ class ShellJobRunner(AsynchronousJobRunner):
                     # Try to find out the reason for exiting
                     self.__handle_out_of_memory(ajs, external_job_id)
                     self.work_queue.put((self.mark_as_failed, ajs))
+                    # Don't add the job to the watched items once it fails, deals with https://github.com/galaxyproject/galaxy/issues/7820
+                    continue
             if state == model.Job.states.RUNNING and not ajs.running:
                 ajs.running = True
             ajs.old_state = state

lib/galaxy/jobs/splitters/multi.py

@@ -1,9 +1,9 @@
-import inspect
 import logging
 import os
 import shutil
 
 from galaxy import model, util
+from galaxy.util.getargspec import getfullargspec
 
 
 log = logging.getLogger(__name__)
@@ -157,7 +157,7 @@ def do_merge(job_wrapper, task_wrappers):
                                   % (len(output_files), len(task_dirs), output_file_name))
                     # First two args to merge always output_files and path of dataset. More
                     # complicated merge methods may require more parameters. Set those up here.
-                    extra_merge_arg_names = inspect.getargspec(output_type.merge).args[2:]
+                    extra_merge_arg_names = getfullargspec(output_type.merge).args[2:]
                     extra_merge_args = {}
                     if "output_dataset" in extra_merge_arg_names:
                         extra_merge_args["output_dataset"] = output_dataset

lib/galaxy/tools/errors.py

@@ -1,9 +1,10 @@
 """
 Functionality for dealing with tool errors.
 """
-import cgi
 import string
 
+import markupsafe
+
 from galaxy import (
     model,
     util,
@@ -219,7 +220,7 @@ class ErrorReporter(object):
         # Escape all of the content  for use in the HTML report
         for parameter in report_variables.keys():
             if report_variables[parameter] is not None:
-                report_variables[parameter] = cgi.escape(unicodify(report_variables[parameter]))
+                report_variables[parameter] = markupsafe.escape(unicodify(report_variables[parameter]))
 
         self.html_report = string.Template(error_report_template_html).safe_substitute(report_variables)
 

lib/galaxy/tools/lint.py

@@ -4,6 +4,7 @@ from __future__ import print_function
 import inspect
 
 from galaxy.util import submodules
+from galaxy.util.getargspec import getfullargspec
 from .parser import get_tool_source
 
 
@@ -42,7 +43,7 @@ def lint_tool_source_with(lint_context, tool_source, extra_modules=[]):
                 # Look at the first argument to the linter to decide
                 # if we should lint the XML description or the abstract
                 # tool parser object.
-                first_arg = inspect.getargspec(value).args[0]
+                first_arg = getfullargspec(value).args[0]
                 if first_arg == "tool_xml":
                     if tool_xml is None:
                         # XML linter and non-XML tool, skip for now

lib/galaxy/tools/verify/asserts/__init__.py

@@ -1,8 +1,8 @@
-import inspect
 import logging
 import sys
 
 from galaxy.util import unicodify
+from galaxy.util.getargspec import getfullargspec
 
 log = logging.getLogger(__name__)
 
@@ -43,7 +43,7 @@ def verify_assertion(data, assertion_description):
         errmsg = "Unable to find test function associated with XML tag '%s'. Check your tool file syntax." % tag
         raise AssertionError(errmsg)
 
-    assert_function_args = inspect.getargspec(assert_function).args
+    assert_function_args = getfullargspec(assert_function).args
     args = {}
     for attribute, value in assertion_description["attributes"].items():
         if attribute in assert_function_args:

lib/galaxy/util/getargspec.py

@@ -0,0 +1,9 @@
+import inspect
+
+
+def getfullargspec(func):
+    try:
+        return inspect.getfullargspec(func)
+    except AttributeError:
+        # on python 2
+        return inspect.getargspec(func)

lib/galaxy/util/pastescript/loadwsgi.py

@@ -12,6 +12,7 @@ import pkg_resources
 from six import iteritems
 from six.moves.urllib.parse import unquote
 
+from galaxy.util.getargspec import getfullargspec
 from galaxy.util.properties import NicerConfigParser
 
 
@@ -58,11 +59,11 @@ def fix_type_error(exc_info, callable, varargs, kwargs):
     if exc_info is None:
         exc_info = sys.exc_info()
     if (exc_info[0] != TypeError or
-            str(exc_info[1]).find('arguments') == -1 or
+            str(exc_info[1]).find('argument') == -1 or
             getattr(exc_info[1], '_type_error_fixed', False)):
         return exc_info
     exc_info[1]._type_error_fixed = True
-    argspec = inspect.formatargspec(*inspect.getargspec(callable))
+    argspec = inspect.formatargspec(*getfullargspec(callable))
     args = ', '.join(map(_short_repr, varargs))
     if kwargs and args:
         args += ', '

lib/galaxy/util/xml_macros.py

@@ -83,21 +83,13 @@ def _macros_of_type(root, type, el_func):
     return macro_dict
 
 
-def expand_nested_tokens(tokens, restarts=10):
-    token_copy = tokens.copy()
-    token_changed = False
-    for token_name in token_copy.keys():
-        for current_token_name, current_token_value in token_copy.items():
+def expand_nested_tokens(tokens):
+    for token_name in tokens.keys():
+        for current_token_name, current_token_value in tokens.items():
             if token_name in current_token_value:
-                current_token_value = current_token_value.replace(token_name, token_copy[token_name])
-                tokens[current_token_name] = current_token_value
-                # We changed a token, so we need to restart
-                token_changed = True
-    if token_changed:
-        if restarts > 0:
-            expand_nested_tokens(tokens, restarts=restarts - 1)
-        else:
-            raise Exception("Tokens are nested too deep")
+                if token_name == current_token_name:
+                    raise Exception("Token '%s' cannot contain itself" % token_name)
+                tokens[current_token_name] = current_token_value.replace(token_name, tokens[token_name])
     return tokens
 
 
@@ -122,11 +114,11 @@ def _expand_tokens_for_el(element, tokens):
     _expand_tokens(list(element), tokens)
 
 
-def _expand_tokens_str(str, tokens):
+def _expand_tokens_str(s, tokens):
     for key, value in tokens.items():
-        if str.find(key) > -1:
-            str = str.replace(key, value)
-    return str
+        if key in s:
+            s = s.replace(key, value)
+    return s
 
 
 def _expand_macros(elements, macros, tokens):
@@ -158,7 +150,7 @@ def _expand_macro(element, expand_el, macros, tokens):
 
     # HACK for elementtree, newer implementations (etree/lxml) won't
     # require this parent_map data structure but elementtree does not
-    # track parents or recongnize .find('..').
+    # track parents or recognize .find('..').
     # TODO fix this now that we're not using elementtree
     parent_map = dict((c, p) for p in element.iter() for c in p)
     _xml_replace(expand_el, expanded_elements, parent_map)

lib/galaxy/version.py

@@ -1,3 +1,3 @@
-VERSION_MAJOR = "19.05"
+VERSION_MAJOR = "19.09"
 VERSION_MINOR = "dev"
 VERSION = VERSION_MAJOR + ('.' + VERSION_MINOR if VERSION_MINOR else '')

lib/galaxy/web/framework/decorators.py

@@ -1,4 +1,3 @@
-import inspect
 import logging
 from functools import wraps
 from json import loads
@@ -12,6 +11,7 @@ from galaxy.util import (
     parse_non_hex_float,
     unicodify
 )
+from galaxy.util.getargspec import getfullargspec
 from galaxy.util.json import safe_dumps
 from galaxy.web.framework import url_for
 
@@ -176,7 +176,7 @@ def __extract_payload_from_request(trans, func, kwargs):
         # in the payload. Therefore, the decorated method's formal arguments are discovered through reflection and removed from
         # the payload dictionary. This helps to prevent duplicate argument conflicts in downstream methods.
         payload = kwargs.copy()
-        named_args, _, _, _ = inspect.getargspec(func)
+        named_args = getfullargspec(func).args
         for arg in named_args:
             payload.pop(arg, None)
         for k, v in payload.items():

lib/galaxy/web/framework/middleware/error.py

@@ -9,10 +9,10 @@ Error handler middleware
 When an exception is thrown from the wrapper application, this logs
 the exception and displays an error page.
 """
-import cgi
 import sys
 import traceback
 
+import markupsafe
 import six
 from paste import (
     request,
@@ -449,7 +449,7 @@ def send_report(rep, exc_data, html=True):
 
             <pre>%s</pre>
             </p>""" % (
-                cgi.escape(str(rep)), output.getvalue())
+                markupsafe.escape(str(rep)), output.getvalue())
         else:
             return (
                 "Additionally an error occurred while sending the "

lib/galaxy/web/framework/middleware/profile.py

@@ -2,11 +2,11 @@
 Middleware that profiles the request with cProfile and displays profiling
 information at the bottom of each page.
 """
-import cgi
 import cProfile
 import pstats
 import threading
 
+import markupsafe
 from paste import response
 
 
@@ -93,9 +93,9 @@ def pstats_as_html(stats, *sel_list):
     funcs, order_message, select_message = get_func_list(stats, sel_list)
     # Deal with any ordering or selection messages
     if order_message:
-        rval.append("<div>%s</div>" % cgi.escape(order_message))
+        rval.append("<div>%s</div>" % markupsafe.escape(order_message))
     if select_message:
-        rval.append("<div>%s</div>" % cgi.escape(select_message))
+        rval.append("<div>%s</div>" % markupsafe.escape(select_message))
     # Build a table for the functions
     if list:
         rval.append("<table>")
@@ -114,7 +114,7 @@ def pstats_as_html(stats, *sel_list):
             ncalls = str(nc)
             if nc != cc:
                 ncalls = ncalls + '/' + str(cc)
-            rval.append("<td>%s</td>" % cgi.escape(ncalls))
+            rval.append("<td>%s</td>" % markupsafe.escape(ncalls))
             # tottime
             rval.append("<td>%0.8f</td>" % tt)
             # percall
@@ -122,7 +122,7 @@ def pstats_as_html(stats, *sel_list):
                 percall = ""
             else:
                 percall = "%0.8f" % (tt / nc)
-            rval.append("<td>%s</td>" % cgi.escape(percall))
+            rval.append("<td>%s</td>" % markupsafe.escape(percall))
             # cumtime
             rval.append("<td>%0.8f</td>" % ct)
             # ctpercall
@@ -130,9 +130,9 @@ def pstats_as_html(stats, *sel_list):
                 ctpercall = ""
             else:
                 ctpercall = "%0.8f" % (ct / cc)
-            rval.append("<td>%s</td>" % cgi.escape(ctpercall))
+            rval.append("<td>%s</td>" % markupsafe.escape(ctpercall))
             # location
-            rval.append("<td>%s</td>" % cgi.escape(func_std_string(func)))
+            rval.append("<td>%s</td>" % markupsafe.escape(func_std_string(func)))
             # row complete
             rval.append("</tr>")
         rval.append("</table>")

templates/tagging_common.mako

@@ -1,6 +1,6 @@
 <%namespace file="/display_common.mako" import="get_controller_name" />
 <%!
-from cgi import escape
+from markupsafe import escape
 %>
 
 ## Render a tagging element if there is a tagged_item.

templates/webapps/galaxy/workflow/display.mako

@@ -2,9 +2,9 @@
 <%namespace file="/display_common.mako" import="render_message" />
 
 <%!
+    import markupsafe
     from galaxy.tools.parameters.basic import DataCollectionToolParameter, DataToolParameter, RuntimeValue
     from galaxy.web import form_builder
-    import cgi
 %>
 
 <%def name="stylesheets()">
@@ -64,7 +64,7 @@
                     <i>select at runtime</i>
                 %endif
             %else:
-                ${cgi.escape( param.value_to_display_text( value ) or 'Unavailable.' )}
+                ${markupsafe.escape( param.value_to_display_text( value ) or 'Unavailable.' )}
             %endif
         </div>
         %if hasattr( step, 'upgrade_messages' ) and step.upgrade_messages and param.name in step.upgrade_messages:

test/unit/test_fix_type_error.py

@@ -0,0 +1,23 @@
+import six
+
+from galaxy.util.pastescript.loadwsgi import fix_type_error
+
+if six.PY2:
+    argtype = ' keyword'
+else:
+    argtype = ''
+EXP_STRING = "a_func() got multiple values for%s argument 'kwarg'; got (1, 2, kwarg=...), wanted (argone, kwarg=True)" % argtype
+
+
+def a_func(argone, kwarg=True):
+    pass
+
+
+def test_fix_type_error():
+    args = [1, 2]
+    kwargs = {'kwarg': False}
+    try:
+        a_func(*args, **kwargs)
+    except TypeError:
+        exc_info = fix_type_error(None, a_func, args, kwargs)
+    assert exc_info[1].args[0] == EXP_STRING

test/unit/tools/test_parsing.py

@@ -61,6 +61,22 @@ TOOL_WITH_TOKEN = r"""
 </tool>
 """
 
+TOOL_WITH_RECURSIVE_TOKEN = r"""
+<tool id="tool_with_recursive_token" name="Token" version="1">
+    <macros>
+        <token name="@NESTED_TOKEN@">
+<![CDATA[
+    before
+    @NESTED_TOKEN@
+    after
+        ]]></token>
+    </macros>
+    <command>
+@NESTED_TOKEN@
+    </command>
+</tool>
+"""
+
 TOOL_YAML_1 = """
 name: "Bowtie Mapper"
 class: GalaxyTool
@@ -328,6 +344,10 @@ class XmlLoaderTestCase(BaseLoaderTestCase):
         assert command
         assert '@' not in command
 
+    def test_recursive_token(self):
+        with self.assertRaises(Exception):
+            self._get_tool_source(source_contents=TOOL_WITH_RECURSIVE_TOKEN)
+
 
 class YamlLoaderTestCase(BaseLoaderTestCase):
     source_file_name = "bwa.yml"