diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index c1f7c980723..02ea6f558c1 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -464,6 +464,7 @@ class Tool(Dictifiable): self.guid = guid self.old_id = None self.version = None + self.python_template_version = None self._lineage = None self.dependencies = [] # populate toolshed repository info, if available @@ -643,6 +644,14 @@ class Tool(Dictifiable): message = template % (self.id, self.profile) raise Exception(message) + self.python_template_version = tool_source.parse_python_template_version() + if self.python_template_version is None: + # If python_template_version not specified we assume tools with profile versions >= 19.05 are python 3 ready + if self.profile >= 19.05: + self.python_template_version = '3.5' + else: + self.python_template_version = '2.7' + # Get the (user visible) name of the tool self.name = tool_source.parse_name() if not self.name and dynamic: diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index c5f6ce5f31d..06c54d9dbc6 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -377,7 +377,8 @@ class DefaultToolAction(object): wrapped_params.params, inp_data, inp_dataset_collections, - input_ext + input_ext, + python_template_version=tool.python_template_version, ) create_datasets = True dataset = None @@ -744,7 +745,7 @@ class DefaultToolAction(object): if output.label: params['tool'] = tool params['on_string'] = on_text - return fill_template(output.label, context=params) + return fill_template(output.label, context=params, python_template_version=tool.python_template_version) else: return self._get_default_data_name(dataset, tool, on_text=on_text, trans=trans, incoming=incoming, history=history, params=params, job_params=job_params) @@ -761,7 +762,7 @@ class DefaultToolAction(object): if output.actions: for action in output.actions.actions: if action.tag == "metadata" and action.default: - metadata_new_value = fill_template(action.default, context=params).split(",") + metadata_new_value = fill_template(action.default, context=params, python_template_version=tool.python_template_version).split(",") dataset.metadata.__setattr__(str(action.name), metadata_new_value) def _get_default_data_name(self, dataset, tool, on_text=None, trans=None, incoming=None, history=None, params=None, job_params=None, **kwd): @@ -924,7 +925,7 @@ def get_ext_or_implicit_ext(hda): return hda.ext -def determine_output_format(output, parameter_context, input_datasets, input_dataset_collections, random_input_ext): +def determine_output_format(output, parameter_context, input_datasets, input_dataset_collections, random_input_ext, python_template_version='3'): """ Determines the output format for a dataset based on an abstract description of the output (galaxy.tools.parser.ToolOutput), the parameter wrappers, a map of the input datasets (name => HDA), and the last input @@ -986,7 +987,7 @@ def determine_output_format(output, parameter_context, input_datasets, input_dat if '$' not in check: # allow a simple name or more complex specifications check = '${%s}' % check - if str(fill_template(check, context=parameter_context)) == when_elem.get('value', None): + if fill_template(check, context=parameter_context, python_template_version=python_template_version) == when_elem.get('value', None): ext = when_elem.get('format', ext) except Exception: # bad tag input value; possibly referencing a param within a different conditional when block or other nonexistent grouping construct continue diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py index 1b741b7dc97..162f0c906d1 100644 --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -469,7 +469,7 @@ class ToolEvaluator(object): return try: # Substituting parameters into the command - command_line = fill_template(command, context=param_dict) + command_line = fill_template(command, context=param_dict, python_template_version=self.tool.python_template_version) cleaned_command_line = [] # Remove leading and trailing whitespace from each line for readability. for line in command_line.split('\n'): @@ -575,7 +575,7 @@ class ToolEvaluator(object): def __write_workdir_file(self, config_filename, content, context, is_template=True): if is_template: - value = fill_template(content, context=context) + value = fill_template(content, context=context, python_template_version=self.tool.python_template_version) else: value = unicodify(content) with io.open(config_filename, "w", encoding='utf-8') as f: diff --git a/lib/galaxy/tools/parser/cwl.py b/lib/galaxy/tools/parser/cwl.py index 576023848b5..edf744acbb1 100644 --- a/lib/galaxy/tools/parser/cwl.py +++ b/lib/galaxy/tools/parser/cwl.py @@ -150,6 +150,9 @@ class CwlToolSource(ToolSource): def parse_profile(self): return "16.04" + def parse_python_template_version(self): + return '3.5' + class CwlPageSource(PageSource): diff --git a/lib/galaxy/tools/parser/interface.py b/lib/galaxy/tools/parser/interface.py index aa6ade42b69..f63e51ce8e5 100644 --- a/lib/galaxy/tools/parser/interface.py +++ b/lib/galaxy/tools/parser/interface.py @@ -218,6 +218,12 @@ class ToolSource(object): """ Return tool profile version as Galaxy major e.g. 16.01 or 16.04. """ + @abstractmethod + def parse_python_template_version(self): + """ + Return minimum python version that the tool template has been developed against. + """ + def macro_paths(self): return [] diff --git a/lib/galaxy/tools/parser/xml.py b/lib/galaxy/tools/parser/xml.py index 1f46c406dc9..6c7f32af29c 100644 --- a/lib/galaxy/tools/parser/xml.py +++ b/lib/galaxy/tools/parser/xml.py @@ -5,6 +5,8 @@ import traceback import uuid from math import isinf +import packaging.version + from galaxy.tools.deps import requirements from galaxy.util import string_as_bool, xml_text, xml_to_string from galaxy.util.odict import odict @@ -483,6 +485,12 @@ class XmlToolSource(ToolSource): # - Enable buggy interpreter attribute. return self.root.get("profile", "16.01") + def parse_python_template_version(self): + python_template_version = self.root.get("python_template_version", None) + if python_template_version is not None: + python_template_version = packaging.version.parse(python_template_version) + return python_template_version + def _test_elem_to_dict(test_elem, i): rval = dict( diff --git a/lib/galaxy/tools/parser/yaml.py b/lib/galaxy/tools/parser/yaml.py index 65ce7747644..88500de805c 100644 --- a/lib/galaxy/tools/parser/yaml.py +++ b/lib/galaxy/tools/parser/yaml.py @@ -1,3 +1,5 @@ +import packaging.version + from galaxy.tools.deps import requirements from galaxy.util.odict import odict from .interface import InputSource @@ -162,6 +164,12 @@ class YamlToolSource(ToolSource): def parse_profile(self): return self.root_dict.get("profile", "16.04") + def parse_python_template_version(self): + python_template_version = self.root_dict.get("python_template_version", None) + if python_template_version is not None: + python_template_version = packaging.version.parse(python_template_version) + return python_template_version + def _parse_test(i, test_dict): inputs = test_dict["inputs"] diff --git a/lib/galaxy/tools/xsd/galaxy.xsd b/lib/galaxy/tools/xsd/galaxy.xsd index 86eb7bbec48..cd80d4a0149 100644 --- a/lib/galaxy/tools/xsd/galaxy.xsd +++ b/lib/galaxy/tools/xsd/galaxy.xsd @@ -175,13 +175,22 @@ set this to have values such as ``data_source`` or ``manage_data``. - This string specified the minimum Galaxy + This string specifies the minimum Galaxy version that should be required to run this tool. Certain legacy behaviors such as using standard error content to detect errors instead of exit code are disabled automatically if profile is set to any version newer than ``16.01``, such as ``16.04``. + + + This string specifies the minimum Python +version that is able to fill the Cheetah sections of the tool. If unset defaults +to 2.7 if the profile is older than 19.01, otherwise defaults to 3.5. Galaxy will +attempt to convert Python statements in Cheetah sections using [future](http://python-future.org/) +if Galaxy is run on Python 3 and ``python_template_version`` is below 3. + + This attribute indicates if diff --git a/lib/galaxy/util/template.py b/lib/galaxy/util/template.py index 1fb97f6ed59..c5d22066664 100644 --- a/lib/galaxy/util/template.py +++ b/lib/galaxy/util/template.py @@ -1,10 +1,51 @@ """Entry point for the usage of Cheetah templating within Galaxy.""" +from __future__ import absolute_import + +import sys +import traceback +from lib2to3.refactor import RefactoringTool + +import packaging.version +from Cheetah.Compiler import Compiler +from Cheetah.NameMapper import NotFound from Cheetah.Template import Template +from past.translation import myfixes from . import unicodify +# Skip libpasteurize fixers, which make sure code is py2 and py3 compatible. +# This is not needed, we only translate code on py3. +myfixes = [f for f in myfixes if not f.startswith('libpasteurize')] +refactoring_tool = RefactoringTool(myfixes, {'print_function': True}) -def fill_template(template_text, context=None, **kwargs): + +class FixedModuleCodeCompiler(Compiler): + + module_code = None + + def getModuleCode(self): + self._moduleDef = self.module_code + return self._moduleDef + + +def create_compiler_class(module_code): + + class CustomCompilerClass(FixedModuleCodeCompiler): + pass + + setattr(CustomCompilerClass, 'module_code', module_code) + + return CustomCompilerClass + + +def fill_template(template_text, + context=None, + retry=10, + compiler_class=Compiler, + first_exception=None, + futurized=False, + python_template_version='3', + **kwargs): """Fill a cheetah template out for specified context. If template_text is None, an exception will be thrown, if context @@ -15,4 +56,62 @@ def fill_template(template_text, context=None, **kwargs): raise TypeError("Template text specified as None to fill_template.") if not context: context = kwargs - return unicodify(Template(source=template_text, searchList=[context])) + if isinstance(python_template_version, str): + python_template_version = packaging.version.parse(python_template_version) + klass = Template.compile(source=template_text, compilerClass=compiler_class) + t = klass(searchList=[context]) + try: + return unicodify(t) + except NotFound as e: + if first_exception is None: + first_exception = e + if sys.version_info.major > 2 and python_template_version.release[0] < 3 and retry > 0: + tb = e.__traceback__ + last_stack = traceback.extract_tb(tb)[-1] + if last_stack.name == '': + # On python 3 list, dict and set comprehensions as well as generator expressions + # have their own local scope, which prevents accessing frame variables in cheetah. + # We can work around this by replacing `$var` with `var`, but we only do this for + # list comprehensions, as this has never worked for dict or set comprehensions or + # generator expressions in Cheetah. + var_not_found = e.args[0].split("'")[1] + replace_str = 'VFFSL(SL,"%s",True)' % var_not_found + lineno = last_stack.lineno - 1 + module_code = t._CHEETAH_generatedModuleCode.splitlines() + module_code[lineno] = module_code[lineno].replace(replace_str, var_not_found) + module_code = "\n".join(module_code) + compiler_class = create_compiler_class(module_code) + return fill_template(template_text=template_text, + context=context, + retry=retry - 1, + compiler_class=compiler_class, + first_exception=first_exception, + python_template_version=python_template_version, + ) + raise first_exception or e + except Exception as e: + if first_exception is None: + first_exception = e + if sys.version_info.major > 2 and python_template_version.release[0] < 3 and not futurized: + # Possibly an error caused by attempting to run python 2 + # template code on python 3. Run the generated module code + # through futurize and hope for the best. + module_code = t._CHEETAH_generatedModuleCode + module_code = futurize_preprocessor(module_code) + compiler_class = create_compiler_class(module_code) + return fill_template(template_text=template_text, + context=context, + retry=retry, + compiler_class=compiler_class, + first_exception=first_exception, + futurized=True, + python_template_version=python_template_version, + ) + raise first_exception or e + + +def futurize_preprocessor(source): + source = str(refactoring_tool.refactor_string(source, name='auto_translate_cheetah')) + # libfuturize.fixes.fix_unicode_keep_u' breaks from Cheetah.compat import unicode + source = source.replace('from Cheetah.compat import str', 'from Cheetah.compat import unicode') + return source diff --git a/lib/galaxy/webapps/galaxy/controllers/tag.py b/lib/galaxy/webapps/galaxy/controllers/tag.py index 25e74fdf576..0320f2be72e 100644 --- a/lib/galaxy/webapps/galaxy/controllers/tag.py +++ b/lib/galaxy/webapps/galaxy/controllers/tag.py @@ -90,7 +90,6 @@ class TagsController(BaseUIController, UsesTagsMixin): user = trans.user item_class = self.get_class(item_class) q = '' if q is None else q - q = q.encode('utf-8') if q.find(":") == -1: return self._get_tag_autocomplete_names(trans, q, limit, timestamp, user, item, item_class) else: diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 7386e2c2f84..d7089226c5e 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -993,7 +993,9 @@ class ToolModule(WorkflowModule): params = make_dict_copy(self.state.inputs) params['on_string'] = 'input dataset(s)' params['tool'] = self.tool - extra_kwds['label'] = fill_template(tool_output.label, context=params) + extra_kwds['label'] = fill_template(tool_output.label, + context=params, + python_template_version=self.tool.python_template_version) except Exception: pass data_outputs.append( diff --git a/test/base/api.py b/test/base/api.py index 8b98da97abf..45988fb9bc6 100644 --- a/test/base/api.py +++ b/test/base/api.py @@ -1,3 +1,4 @@ +import os from contextlib import contextmanager from six.moves.urllib.parse import urlencode @@ -22,6 +23,12 @@ from .testcase import FunctionalTestCase class UsesApiTestCaseMixin(object): + def tearDown(self): + if os.environ.get('GALAXY_TEST_EXTERNAL') is None: + # Only kill running jobs after test for managed test instances + for job in self.galaxy_interactor.get('jobs?state=running&?user_details=true').json(): + self._delete("jobs/%s" % job['id']) + def _api_url(self, path, params=None, use_key=None, use_admin_key=None): if not params: params = {} diff --git a/test/unit/tools/test_evaluation.py b/test/unit/tools/test_evaluation.py index e79eb235ef8..f85ceb227ba 100644 --- a/test/unit/tools/test_evaluation.py +++ b/test/unit/tools/test_evaluation.py @@ -266,6 +266,7 @@ class MockTool(object): def __init__(self, app): self.profile = 16.01 + self.python_template_version = '2.7' self.app = app self.hooks_called = [] self.environment_variables = [] diff --git a/test/unit/tools/test_fill_template.py b/test/unit/tools/test_fill_template.py new file mode 100644 index 00000000000..48d280e27dc --- /dev/null +++ b/test/unit/tools/test_fill_template.py @@ -0,0 +1,77 @@ +import sys + +import pytest +from Cheetah.NameMapper import NotFound + +from galaxy.util.template import fill_template + +SIMPLE_TEMPLATE = """#for item in $a_list: + echo $item + #end for +""" +FILLED_SIMPLE_TEMPLATE = """ echo 1 + echo 2 +""" +LIST_COMPREHENSION_TEMPLATE = """#for $i in [1]: +#set $v = "".join([str(_) for _ in [1] if _ == $i]) +echo $v +#end for +""" +DICT_COMPREHENSION_TEMPLATE = """#for $i in [1]: +#set $v = {_:_ for _ in [1] if _ == $i} +echo $v[$i] +#end for +""" +SET_COMPR_TEMPLATE = """#for $i in [1]: +#set $v = {_ for _ in [1] if _ == $i} +echo $v.pop() +#end for +""" +GEN_EXPR_TEMPLATE = """#for $i in [1]: +#set $v = list((_ for _ in [1] if _ == $i)) +echo $v[0] +#end for +""" +TWO_TO_THREE_TEMPLATE = """#set $a = [x for x in {'a': '1'}.iterkeys()][0] +#set $b = [x for x in {'a': '1'}.iteritems()][0][0] +#set $c = [x for x in {'a': '1'}.itervalues()][0] +$a $b $c""" + + +def test_fill_simple_template(): + template_str = str(fill_template(SIMPLE_TEMPLATE, {'a_list': [1, 2]})) + assert template_str == FILLED_SIMPLE_TEMPLATE + + +def test_fill_list_comprehension_template(): + if sys.version_info.major > 2: + with pytest.raises(NotFound): + fill_template(LIST_COMPREHENSION_TEMPLATE, retry=0) + else: + template_str = fill_template(LIST_COMPREHENSION_TEMPLATE, retry=0) + assert template_str == 'echo 1\n' + + +def test_fill_list_comprehension_template_2(): + template_str = fill_template(LIST_COMPREHENSION_TEMPLATE, python_template_version='2', retry=1) + assert template_str == 'echo 1\n' + + +def test_fill_dict_comprehension(): + with pytest.raises(NotFound): + fill_template(DICT_COMPREHENSION_TEMPLATE, retry=1) + + +def test_set_comprehension(): + with pytest.raises(NotFound): + fill_template(SET_COMPR_TEMPLATE, retry=1) + + +def test_gen_expr(): + with pytest.raises(NotFound): + fill_template(GEN_EXPR_TEMPLATE, retry=1) + + +def test_fix_template_two_to_three(): + template_str = fill_template(TWO_TO_THREE_TEMPLATE, python_template_version='2', retry=1) + assert template_str == 'a a 1'