Merge pull request #7867 from mvdbeek/fix_scope_issue_for_py3_list_comprehensions

[19.05] Run futurize for python2 cheetah templates, fix scope issue for py3 comprehensions

lib/galaxy/tools/__init__.py

@@ -464,6 +464,7 @@ class Tool(Dictifiable):
         self.guid = guid
         self.old_id = None
         self.version = None
+        self.python_template_version = None
         self._lineage = None
         self.dependencies = []
         # populate toolshed repository info, if available
@@ -643,6 +644,14 @@ class Tool(Dictifiable):
             message = template % (self.id, self.profile)
             raise Exception(message)
 
+        self.python_template_version = tool_source.parse_python_template_version()
+        if self.python_template_version is None:
+            # If python_template_version not specified we assume tools with profile versions >= 19.05 are python 3 ready
+            if self.profile >= 19.05:
+                self.python_template_version = '3.5'
+            else:
+                self.python_template_version = '2.7'
+
         # Get the (user visible) name of the tool
         self.name = tool_source.parse_name()
         if not self.name and dynamic:

lib/galaxy/tools/actions/__init__.py

@@ -377,7 +377,8 @@ class DefaultToolAction(object):
                     wrapped_params.params,
                     inp_data,
                     inp_dataset_collections,
-                    input_ext
+                    input_ext,
+                    python_template_version=tool.python_template_version,
                 )
                 create_datasets = True
                 dataset = None
@@ -744,7 +745,7 @@ class DefaultToolAction(object):
         if output.label:
             params['tool'] = tool
             params['on_string'] = on_text
-            return fill_template(output.label, context=params)
+            return fill_template(output.label, context=params, python_template_version=tool.python_template_version)
         else:
             return self._get_default_data_name(dataset, tool, on_text=on_text, trans=trans, incoming=incoming, history=history, params=params, job_params=job_params)
 
@@ -761,7 +762,7 @@ class DefaultToolAction(object):
         if output.actions:
             for action in output.actions.actions:
                 if action.tag == "metadata" and action.default:
-                    metadata_new_value = fill_template(action.default, context=params).split(",")
+                    metadata_new_value = fill_template(action.default, context=params, python_template_version=tool.python_template_version).split(",")
                     dataset.metadata.__setattr__(str(action.name), metadata_new_value)
 
     def _get_default_data_name(self, dataset, tool, on_text=None, trans=None, incoming=None, history=None, params=None, job_params=None, **kwd):
@@ -924,7 +925,7 @@ def get_ext_or_implicit_ext(hda):
     return hda.ext
 
 
-def determine_output_format(output, parameter_context, input_datasets, input_dataset_collections, random_input_ext):
+def determine_output_format(output, parameter_context, input_datasets, input_dataset_collections, random_input_ext, python_template_version='3'):
     """ Determines the output format for a dataset based on an abstract
     description of the output (galaxy.tools.parser.ToolOutput), the parameter
     wrappers, a map of the input datasets (name => HDA), and the last input
@@ -986,7 +987,7 @@ def determine_output_format(output, parameter_context, input_datasets, input_dat
                         if '$' not in check:
                             # allow a simple name or more complex specifications
                             check = '${%s}' % check
-                        if str(fill_template(check, context=parameter_context)) == when_elem.get('value', None):
+                        if fill_template(check, context=parameter_context, python_template_version=python_template_version) == when_elem.get('value', None):
                             ext = when_elem.get('format', ext)
                     except Exception:  # bad tag input value; possibly referencing a param within a different conditional when block or other nonexistent grouping construct
                         continue

lib/galaxy/tools/evaluation.py

@@ -469,7 +469,7 @@ class ToolEvaluator(object):
             return
         try:
             # Substituting parameters into the command
-            command_line = fill_template(command, context=param_dict)
+            command_line = fill_template(command, context=param_dict, python_template_version=self.tool.python_template_version)
             cleaned_command_line = []
             # Remove leading and trailing whitespace from each line for readability.
             for line in command_line.split('\n'):
@@ -575,7 +575,7 @@ class ToolEvaluator(object):
 
     def __write_workdir_file(self, config_filename, content, context, is_template=True):
         if is_template:
-            value = fill_template(content, context=context)
+            value = fill_template(content, context=context, python_template_version=self.tool.python_template_version)
         else:
             value = unicodify(content)
         with io.open(config_filename, "w", encoding='utf-8') as f:

lib/galaxy/tools/parser/cwl.py

@@ -150,6 +150,9 @@ class CwlToolSource(ToolSource):
     def parse_profile(self):
         return "16.04"
 
+    def parse_python_template_version(self):
+        return '3.5'
+
 
 class CwlPageSource(PageSource):
 

lib/galaxy/tools/parser/interface.py

@@ -218,6 +218,12 @@ class ToolSource(object):
         """ Return tool profile version as Galaxy major e.g. 16.01 or 16.04.
         """
 
+    @abstractmethod
+    def parse_python_template_version(self):
+        """
+        Return minimum python version that the tool template has been developed against.
+        """
+
     def macro_paths(self):
         return []
 

lib/galaxy/tools/parser/xml.py

@@ -5,6 +5,8 @@ import traceback
 import uuid
 from math import isinf
 
+import packaging.version
+
 from galaxy.tools.deps import requirements
 from galaxy.util import string_as_bool, xml_text, xml_to_string
 from galaxy.util.odict import odict
@@ -483,6 +485,12 @@ class XmlToolSource(ToolSource):
         # - Enable buggy interpreter attribute.
         return self.root.get("profile", "16.01")
 
+    def parse_python_template_version(self):
+        python_template_version = self.root.get("python_template_version", None)
+        if python_template_version is not None:
+            python_template_version = packaging.version.parse(python_template_version)
+        return python_template_version
+
 
 def _test_elem_to_dict(test_elem, i):
     rval = dict(

lib/galaxy/tools/parser/yaml.py

@@ -1,3 +1,5 @@
+import packaging.version
+
 from galaxy.tools.deps import requirements
 from galaxy.util.odict import odict
 from .interface import InputSource
@@ -162,6 +164,12 @@ class YamlToolSource(ToolSource):
     def parse_profile(self):
         return self.root_dict.get("profile", "16.04")
 
+    def parse_python_template_version(self):
+        python_template_version = self.root_dict.get("python_template_version", None)
+        if python_template_version is not None:
+            python_template_version = packaging.version.parse(python_template_version)
+        return python_template_version
+
 
 def _parse_test(i, test_dict):
     inputs = test_dict["inputs"]

lib/galaxy/tools/xsd/galaxy.xsd

@@ -175,13 +175,22 @@ set this to have values such as ``data_source`` or ``manage_data``.</xs:document
       </xs:attribute>
       <xs:attribute name="profile" type="xs:string">
         <xs:annotation>
-          <xs:documentation xml:lang="en">This string specified the minimum Galaxy
+          <xs:documentation xml:lang="en">This string specifies the minimum Galaxy
 version that should be required to run this tool. Certain legacy behaviors such
 as using standard error content to detect errors instead of exit code are disabled
 automatically if profile is set to any version newer than ``16.01``, such as
 ``16.04``.</xs:documentation>
         </xs:annotation>
       </xs:attribute>
+      <xs:attribute name="python_template_version" type="xs:float">
+        <xs:annotation>
+          <xs:documentation xml:lang="en">This string specifies the minimum Python
+version that is able to fill the Cheetah sections of the tool. If unset defaults
+to 2.7 if the profile is older than 19.01, otherwise defaults to 3.5. Galaxy will 
+attempt to convert Python statements in Cheetah sections using [future](http://python-future.org/)
+if Galaxy is run on Python 3 and ``python_template_version`` is below 3.</xs:documentation>
+        </xs:annotation>
+      </xs:attribute>
       <xs:attribute name="workflow_compatible" type="xs:boolean" default="true">
         <xs:annotation>
           <xs:documentation xml:lang="en">This attribute indicates if

lib/galaxy/util/template.py

@@ -1,10 +1,51 @@
 """Entry point for the usage of Cheetah templating within Galaxy."""
+from __future__ import absolute_import
+
+import sys
+import traceback
+from lib2to3.refactor import RefactoringTool
+
+import packaging.version
+from Cheetah.Compiler import Compiler
+from Cheetah.NameMapper import NotFound
 from Cheetah.Template import Template
+from past.translation import myfixes
 
 from . import unicodify
 
+# Skip libpasteurize fixers, which make sure code is py2 and py3 compatible.
+# This is not needed, we only translate code on py3.
+myfixes = [f for f in myfixes if not f.startswith('libpasteurize')]
+refactoring_tool = RefactoringTool(myfixes, {'print_function': True})
 
-def fill_template(template_text, context=None, **kwargs):
+
+class FixedModuleCodeCompiler(Compiler):
+
+    module_code = None
+
+    def getModuleCode(self):
+        self._moduleDef = self.module_code
+        return self._moduleDef
+
+
+def create_compiler_class(module_code):
+
+    class CustomCompilerClass(FixedModuleCodeCompiler):
+        pass
+
+    setattr(CustomCompilerClass, 'module_code', module_code)
+
+    return CustomCompilerClass
+
+
+def fill_template(template_text,
+                  context=None,
+                  retry=10,
+                  compiler_class=Compiler,
+                  first_exception=None,
+                  futurized=False,
+                  python_template_version='3',
+                  **kwargs):
     """Fill a cheetah template out for specified context.
 
     If template_text is None, an exception will be thrown, if context
@@ -15,4 +56,62 @@ def fill_template(template_text, context=None, **kwargs):
         raise TypeError("Template text specified as None to fill_template.")
     if not context:
         context = kwargs
-    return unicodify(Template(source=template_text, searchList=[context]))
+    if isinstance(python_template_version, str):
+        python_template_version = packaging.version.parse(python_template_version)
+    klass = Template.compile(source=template_text, compilerClass=compiler_class)
+    t = klass(searchList=[context])
+    try:
+        return unicodify(t)
+    except NotFound as e:
+        if first_exception is None:
+            first_exception = e
+        if sys.version_info.major > 2 and python_template_version.release[0] < 3 and retry > 0:
+            tb = e.__traceback__
+            last_stack = traceback.extract_tb(tb)[-1]
+            if last_stack.name == '<listcomp>':
+                # On python 3 list, dict and set comprehensions as well as generator expressions
+                # have their own local scope, which prevents accessing frame variables in cheetah.
+                # We can work around this by replacing `$var` with `var`, but we only do this for
+                # list comprehensions, as this has never worked for dict or set comprehensions or
+                # generator expressions in Cheetah.
+                var_not_found = e.args[0].split("'")[1]
+                replace_str = 'VFFSL(SL,"%s",True)' % var_not_found
+                lineno = last_stack.lineno - 1
+                module_code = t._CHEETAH_generatedModuleCode.splitlines()
+                module_code[lineno] = module_code[lineno].replace(replace_str, var_not_found)
+                module_code = "\n".join(module_code)
+                compiler_class = create_compiler_class(module_code)
+                return fill_template(template_text=template_text,
+                                     context=context,
+                                     retry=retry - 1,
+                                     compiler_class=compiler_class,
+                                     first_exception=first_exception,
+                                     python_template_version=python_template_version,
+                                     )
+        raise first_exception or e
+    except Exception as e:
+        if first_exception is None:
+            first_exception = e
+        if sys.version_info.major > 2 and python_template_version.release[0] < 3 and not futurized:
+            # Possibly an error caused by attempting to run python 2
+            # template code on python 3. Run the generated module code
+            # through futurize and hope for the best.
+            module_code = t._CHEETAH_generatedModuleCode
+            module_code = futurize_preprocessor(module_code)
+            compiler_class = create_compiler_class(module_code)
+            return fill_template(template_text=template_text,
+                                 context=context,
+                                 retry=retry,
+                                 compiler_class=compiler_class,
+                                 first_exception=first_exception,
+                                 futurized=True,
+                                 python_template_version=python_template_version,
+                                 )
+        raise first_exception or e
+
+
+def futurize_preprocessor(source):
+    source = str(refactoring_tool.refactor_string(source, name='auto_translate_cheetah'))
+    # libfuturize.fixes.fix_unicode_keep_u' breaks from Cheetah.compat import unicode
+    source = source.replace('from Cheetah.compat import str', 'from Cheetah.compat import unicode')
+    return source

lib/galaxy/webapps/galaxy/controllers/tag.py

@@ -90,7 +90,6 @@ class TagsController(BaseUIController, UsesTagsMixin):
         user = trans.user
         item_class = self.get_class(item_class)
         q = '' if q is None else q
-        q = q.encode('utf-8')
         if q.find(":") == -1:
             return self._get_tag_autocomplete_names(trans, q, limit, timestamp, user, item, item_class)
         else:

lib/galaxy/workflow/modules.py

@@ -993,7 +993,9 @@ class ToolModule(WorkflowModule):
                         params = make_dict_copy(self.state.inputs)
                         params['on_string'] = 'input dataset(s)'
                         params['tool'] = self.tool
-                        extra_kwds['label'] = fill_template(tool_output.label, context=params)
+                        extra_kwds['label'] = fill_template(tool_output.label,
+                                                            context=params,
+                                                            python_template_version=self.tool.python_template_version)
                     except Exception:
                         pass
                 data_outputs.append(

test/base/api.py

@@ -1,3 +1,4 @@
+import os
 from contextlib import contextmanager
 
 from six.moves.urllib.parse import urlencode
@@ -22,6 +23,12 @@ from .testcase import FunctionalTestCase
 
 class UsesApiTestCaseMixin(object):
 
+    def tearDown(self):
+        if os.environ.get('GALAXY_TEST_EXTERNAL') is None:
+            # Only kill running jobs after test for managed test instances
+            for job in self.galaxy_interactor.get('jobs?state=running&?user_details=true').json():
+                self._delete("jobs/%s" % job['id'])
+
     def _api_url(self, path, params=None, use_key=None, use_admin_key=None):
         if not params:
             params = {}

test/unit/tools/test_evaluation.py

@@ -266,6 +266,7 @@ class MockTool(object):
 
     def __init__(self, app):
         self.profile = 16.01
+        self.python_template_version = '2.7'
         self.app = app
         self.hooks_called = []
         self.environment_variables = []

test/unit/tools/test_fill_template.py

@@ -0,0 +1,77 @@
+import sys
+
+import pytest
+from Cheetah.NameMapper import NotFound
+
+from galaxy.util.template import fill_template
+
+SIMPLE_TEMPLATE = """#for item in $a_list:
+    echo $item
+    #end for
+"""
+FILLED_SIMPLE_TEMPLATE = """    echo 1
+    echo 2
+"""
+LIST_COMPREHENSION_TEMPLATE = """#for $i in [1]:
+#set $v = "".join([str(_) for _ in [1] if _ == $i])
+echo $v
+#end for
+"""
+DICT_COMPREHENSION_TEMPLATE = """#for $i in [1]:
+#set $v = {_:_ for _ in [1] if _ == $i}
+echo $v[$i]
+#end for
+"""
+SET_COMPR_TEMPLATE = """#for $i in [1]:
+#set $v = {_ for _ in [1] if _ == $i}
+echo $v.pop()
+#end for
+"""
+GEN_EXPR_TEMPLATE = """#for $i in [1]:
+#set $v = list((_ for _ in [1] if _ == $i))
+echo $v[0]
+#end for
+"""
+TWO_TO_THREE_TEMPLATE = """#set $a = [x for x in {'a': '1'}.iterkeys()][0]
+#set $b = [x for x in {'a': '1'}.iteritems()][0][0]
+#set $c = [x for x in {'a': '1'}.itervalues()][0]
+$a $b $c"""
+
+
+def test_fill_simple_template():
+    template_str = str(fill_template(SIMPLE_TEMPLATE, {'a_list': [1, 2]}))
+    assert template_str == FILLED_SIMPLE_TEMPLATE
+
+
+def test_fill_list_comprehension_template():
+    if sys.version_info.major > 2:
+        with pytest.raises(NotFound):
+            fill_template(LIST_COMPREHENSION_TEMPLATE, retry=0)
+    else:
+        template_str = fill_template(LIST_COMPREHENSION_TEMPLATE, retry=0)
+        assert template_str == 'echo 1\n'
+
+
+def test_fill_list_comprehension_template_2():
+    template_str = fill_template(LIST_COMPREHENSION_TEMPLATE, python_template_version='2', retry=1)
+    assert template_str == 'echo 1\n'
+
+
+def test_fill_dict_comprehension():
+    with pytest.raises(NotFound):
+        fill_template(DICT_COMPREHENSION_TEMPLATE, retry=1)
+
+
+def test_set_comprehension():
+    with pytest.raises(NotFound):
+        fill_template(SET_COMPR_TEMPLATE, retry=1)
+
+
+def test_gen_expr():
+    with pytest.raises(NotFound):
+        fill_template(GEN_EXPR_TEMPLATE, retry=1)
+
+
+def test_fix_template_two_to_three():
+    template_str = fill_template(TWO_TO_THREE_TEMPLATE, python_template_version='2', retry=1)
+    assert template_str == 'a a 1'