Fix nested macro/token expansion on python 3

I noticed that in multiqc @ESCAPE_IDENTIFIER@ wasn't being replaced.
It's surprising that this didn't cause any issues on python 2,
but might have to do with the iteration order changes.

lib/galaxy/util/xml_macros.py

@@ -19,6 +19,7 @@ def load_with_references(path):
 
     # Collect tokens
     tokens = _macros_of_type(root, 'token', lambda el: el.text or '')
+    tokens = expand_nested_tokens(tokens)
 
     # Expand xml macros
     macro_dict = _macros_of_type(root, 'xml', lambda el: XmlMacroDef(el))
@@ -82,8 +83,26 @@ def _macros_of_type(root, type, el_func):
     return macro_dict
 
 
+def expand_nested_tokens(tokens, restarts=10):
+    token_copy = tokens.copy()
+    token_changed = False
+    for token_name in token_copy.keys():
+        for current_token_name, current_token_value in token_copy.items():
+            if token_name in current_token_value:
+                current_token_value = current_token_value.replace(token_name, token_copy[token_name])
+                tokens[current_token_name] = current_token_value
+                # We changed a token, so we need to restart
+                token_changed = True
+    if token_changed:
+        if restarts > 0:
+            expand_nested_tokens(tokens, restarts=restarts - 1)
+        else:
+            raise Exception("Tokens are nested too deep")
+    return tokens
+
+
 def _expand_tokens(elements, tokens):
-    if not tokens or not elements:
+    if not tokens or elements is None:
         return
 
     for element in elements:

test/unit/tools/test_parsing.py

@@ -41,6 +41,26 @@ TOOL_XML_1 = """
 </tool>
 """
 
+TOOL_WITH_TOKEN = r"""
+<tool id="tool_with_token" name="Token" version="1">
+    <macros>
+        <token name="@ESCAPE_IDENTIFIER@">
+<![CDATA[
+#set identifier = re.sub('[^\s\w\-]', '_', str($file.element_identifier))
+        ]]></token>
+        <token name="@NESTED_TOKEN@">
+<![CDATA[
+    before
+    @ESCAPE_IDENTIFIER@
+    after
+        ]]></token>
+    </macros>
+    <command>
+@NESTED_TOKEN@
+    </command>
+</tool>
+"""
+
 TOOL_YAML_1 = """
 name: "Bowtie Mapper"
 class: GalaxyTool
@@ -253,6 +273,12 @@ class XmlLoaderTestCase(BaseLoaderTestCase):
     def test_refresh_option(self):
         assert self._tool_source.parse_refresh() is False
 
+    def test_nested_token(self):
+        tool_source = self._get_tool_source(source_contents=TOOL_WITH_TOKEN)
+        command = tool_source.parse_command()
+        assert command
+        assert '@' not in command
+
 
 class YamlLoaderTestCase(BaseLoaderTestCase):
     source_file_name = "bwa.yml"