Merge pull request #8097 from bgruening/fix_macro_backport_19.01

[19.01] Fix nested macro/token expansion

lib/galaxy/util/xml_macros.py

@@ -19,6 +19,7 @@ def load_with_references(path):
 
     # Collect tokens
     tokens = _macros_of_type(root, 'token', lambda el: el.text or '')
+    tokens = expand_nested_tokens(tokens)
 
     # Expand xml macros
     macro_dict = _macros_of_type(root, 'xml', lambda el: XmlMacroDef(el))
@@ -82,8 +83,18 @@ def _macros_of_type(root, type, el_func):
     return macro_dict
 
 
+def expand_nested_tokens(tokens):
+    for token_name in tokens.keys():
+        for current_token_name, current_token_value in tokens.items():
+            if token_name in current_token_value:
+                if token_name == current_token_name:
+                    raise Exception("Token '%s' cannot contain itself" % token_name)
+                tokens[current_token_name] = current_token_value.replace(token_name, tokens[token_name])
+    return tokens
+
+
 def _expand_tokens(elements, tokens):
-    if not tokens or not elements:
+    if not tokens or elements is None:
         return
 
     for element in elements:
@@ -103,11 +114,11 @@ def _expand_tokens_for_el(element, tokens):
     _expand_tokens(list(element), tokens)
 
 
-def _expand_tokens_str(str, tokens):
+def _expand_tokens_str(s, tokens):
     for key, value in tokens.items():
-        if str.find(key) > -1:
-            str = str.replace(key, value)
-    return str
+        if key in s:
+            s = s.replace(key, value)
+    return s
 
 
 def _expand_macros(elements, macros, tokens):
@@ -139,7 +150,7 @@ def _expand_macro(element, expand_el, macros, tokens):
 
     # HACK for elementtree, newer implementations (etree/lxml) won't
     # require this parent_map data structure but elementtree does not
-    # track parents or recongnize .find('..').
+    # track parents or recognize .find('..').
     # TODO fix this now that we're not using elementtree
     parent_map = dict((c, p) for p in element.iter() for c in p)
     _xml_replace(expand_el, expanded_elements, parent_map)

test/unit/tools/test_parsing.py

@@ -41,6 +41,42 @@ TOOL_XML_1 = """
 </tool>
 """
 
+TOOL_WITH_TOKEN = r"""
+<tool id="tool_with_token" name="Token" version="1">
+    <macros>
+        <token name="@ESCAPE_IDENTIFIER@">
+<![CDATA[
+#set identifier = re.sub('[^\s\w\-]', '_', str($file.element_identifier))
+        ]]></token>
+        <token name="@NESTED_TOKEN@">
+<![CDATA[
+    before
+    @ESCAPE_IDENTIFIER@
+    after
+        ]]></token>
+    </macros>
+    <command>
+@NESTED_TOKEN@
+    </command>
+</tool>
+"""
+
+TOOL_WITH_RECURSIVE_TOKEN = r"""
+<tool id="tool_with_recursive_token" name="Token" version="1">
+    <macros>
+        <token name="@NESTED_TOKEN@">
+<![CDATA[
+    before
+    @NESTED_TOKEN@
+    after
+        ]]></token>
+    </macros>
+    <command>
+@NESTED_TOKEN@
+    </command>
+</tool>
+"""
+
 TOOL_YAML_1 = """
 name: "Bowtie Mapper"
 class: GalaxyTool
@@ -253,6 +289,16 @@ class XmlLoaderTestCase(BaseLoaderTestCase):
     def test_refresh_option(self):
         assert self._tool_source.parse_refresh() is False
 
+    def test_nested_token(self):
+        tool_source = self._get_tool_source(source_contents=TOOL_WITH_TOKEN)
+        command = tool_source.parse_command()
+        assert command
+        assert '@' not in command
+
+    def test_recursive_token(self):
+        with self.assertRaises(Exception):
+            self._get_tool_source(source_contents=TOOL_WITH_RECURSIVE_TOKEN)
+
 
 class YamlLoaderTestCase(BaseLoaderTestCase):
     source_file_name = "bwa.yml"