py3: fix installing and running data managers

lib/galaxy/tools/data/__init__.py

@@ -185,10 +185,11 @@ class ToolDataTableManager(object):
         # add new elems
         out_elems.extend(new_elems)
         out_path_is_new = not os.path.exists(full_path)
-        with RenamedTemporaryFile(full_path) as out:
+        with RenamedTemporaryFile(full_path, mode='w') as out:
             out.write('<?xml version="1.0"?>\n<tables>\n')
             for elem in out_elems:
-                out.write(util.xml_to_string(elem, pretty=True))
+                elem = util.xml_to_string(elem, pretty=True)
+                out.write(elem)
             out.write('</tables>\n')
         os.chmod(full_path, 0o644)
         if out_path_is_new:
@@ -626,9 +627,10 @@ class TabularToolDataTable(ToolDataTable, Dictifiable):
                     # ensure last existing line ends with new line
                     data_table_fh.seek(-1, 2)  # last char in file
                     last_char = data_table_fh.read(1)
-                    if last_char not in ['\n', '\r']:
-                        data_table_fh.write('\n')
-                data_table_fh.write("%s\n" % (self.separator.join(fields)))
+                    if last_char not in [b'\n', b'\r']:
+                        data_table_fh.write(b'\n')
+                    fields = "%s\n" % self.separator.join(fields)
+                data_table_fh.write(fields.encode('utf-8'))
         return not is_error
 
     def _remove_entry(self, values):

lib/galaxy/util/renamed_temporary_file.py

@@ -10,6 +10,11 @@ class RenamedTemporaryFile(object):
     path on exit.
     """
     def __init__(self, final_path, **kwargs):
+        """
+        >>> dir = tempfile.mkdtemp()
+        >>> with RenamedTemporaryFile(os.path.join(dir, 'test.txt'), mode="w") as out:
+        ...     _ = out.write('bla')
+        """
         tmpfile_dir = kwargs.pop('dir', None)
 
         # Put temporary file in the same directory as the location for the
@@ -18,7 +23,7 @@ class RenamedTemporaryFile(object):
         if tmpfile_dir is None:
             tmpfile_dir = os.path.dirname(final_path)
 
-        self.tmpfile = tempfile.NamedTemporaryFile(dir=tmpfile_dir, **kwargs)
+        self.tmpfile = tempfile.NamedTemporaryFile(dir=tmpfile_dir, delete=False, **kwargs)
         self.final_path = final_path
 
     def __getattr__(self, attr):
@@ -33,7 +38,7 @@ class RenamedTemporaryFile(object):
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         if exc_type is None:
-            self.tmpfile.delete = False
+            self.tmpfile.flush()
             result = self.tmpfile.__exit__(exc_type, exc_val, exc_tb)
             os.rename(self.tmpfile.name, self.final_path)
         else:

lib/tool_shed/galaxy_install/install_manager.py

@@ -470,7 +470,7 @@ class InstallRepositoryManager(object):
             # dictionaries, a dictionary defining the Repository, a dictionary defining the
             # Repository revision (RepositoryMetadata), and a dictionary including the additional
             # information required to install the repository.
-            items = json.loads(raw_text)
+            items = json.loads(util.unicodify(raw_text))
             repository_revision_dict = items[1]
             repo_info_dict = items[2]
         else:

lib/tool_shed/galaxy_install/tools/data_manager.py

@@ -3,6 +3,7 @@ import os
 import threading
 import time
 
+from galaxy.util.renamed_temporary_file import RenamedTemporaryFile
 from tool_shed.galaxy_install.tools import tool_panel_manager
 from tool_shed.util import xml_util
 
@@ -31,15 +32,14 @@ class DataManagerHandler(object):
         lock = threading.Lock()
         lock.acquire(True)
         try:
-            fh = open(config_filename, 'wb')
-            if data_managers_path is not None:
-                fh.write('<?xml version="1.0"?>\n<data_managers tool_path="%s">\n    ' % data_managers_path)
-            else:
-                fh.write('<?xml version="1.0"?>\n<data_managers>\n    ')
-            for elem in config_elems:
-                fh.write(xml_util.xml_to_string(elem))
-            fh.write('</data_managers>\n')
-            fh.close()
+            with RenamedTemporaryFile(config_filename, mode='w') as fh:
+                if data_managers_path is not None:
+                    fh.write('<?xml version="1.0"?>\n<data_managers tool_path="%s">\n    ' % data_managers_path)
+                else:
+                    fh.write('<?xml version="1.0"?>\n<data_managers>\n    ')
+                for elem in config_elems:
+                    fh.write(xml_util.xml_to_string(elem))
+                fh.write('</data_managers>\n')
         except Exception:
             log.exception("Exception in DataManagerHandler.data_manager_config_elems_to_xml_file")
         finally:

test/integration/test_data_manager_table_reload.py

@@ -47,7 +47,7 @@ class DataManagerIntegrationTestCase(integration_util.IntegrationTestCase, UsesS
         Test that we can install data managers, create a new dbkey, and use that dbkey in a downstream data manager.
         """
         self.install_repository("devteam", "data_manager_fetch_genome_dbkeys_all_fasta", "b1bc53e9bbc5")
-        self.install_repository("devteam", "data_manager_sam_fasta_index_builder", "1865e693d8b2")
+        self.install_repository("devteam", "data_manager_sam_fasta_index_builder", "406896e00d0e", 'https://testtoolshed.g2.bx.psu.edu')
         with self._different_user(email="%s@galaxy.org" % self.username):
             with self.dataset_populator.test_history() as history_id:
                 run_response = self.dataset_populator.run_tool(tool_id=FETCH_TOOL_ID,

test/integration/tool_sheds_conf.xml

@@ -1,4 +1,5 @@
 <?xml version="1.0"?>
 <tool_sheds>
     <tool_shed name="Galaxy Main Tool Shed" url="https://toolshed.g2.bx.psu.edu/"/>
+    <tool_shed name="Galaxy Test Tool Shed" url="https://testtoolshed.g2.bx.psu.edu/"/>
 </tool_sheds>

test/integration/uses_shed.py

@@ -48,9 +48,9 @@ class UsesShed(object):
         with open(config["shed_tool_data_table_config"], 'w') as shed_data_table_config:
             shed_data_table_config.write(SHED_DATA_TABLES)
 
-    def install_repository(self, owner, name, changeset):
+    def install_repository(self, owner, name, changeset, tool_shed_url='https://toolshed.g2.bx.psu.edu'):
         payload = {
-            'tool_shed_url': 'https://toolshed.g2.bx.psu.edu',
+            'tool_shed_url': tool_shed_url,
             'name': name,
             'owner': owner,
             'changeset_revision': changeset