Write out registry once per job

lib/galaxy/datatypes/registry.py

@@ -6,7 +6,6 @@ from __future__ import absolute_import
 import imp
 import logging
 import os
-import tempfile
 from collections import OrderedDict as odict
 from xml.etree.ElementTree import Element
 
@@ -672,7 +671,6 @@ class Registry(object):
         # We need to be able to add a job to the queue to set metadata. The queue will currently only accept jobs with an associated
         # tool.  We'll load a special tool to be used for Auto-Detecting metadata; this is less than ideal, but effective
         # Properly building a tool without relying on parsing an XML file is near difficult...so we bundle with Galaxy.
-        self.to_xml_file()
         set_meta_tool = toolbox.load_hidden_lib_tool("galaxy/datatypes/set_metadata_tool.xml")
         self.set_external_metadata_tool = set_meta_tool
         self.log.debug("Loaded external metadata tool: %s", self.set_external_metadata_tool.id)
@@ -876,16 +874,8 @@ class Registry(object):
             return self.xml_filename
         return self.xml_filename
 
-    def to_xml_file(self):
-        if self.xml_filename is not None:
-            # If persisted previously, attempt to remove the temporary file in which we were written.
-            try:
-                os.unlink(self.xml_filename)
-            except:
-                pass
-            self.xml_filename = None
-        fd, filename = tempfile.mkstemp()
-        self.xml_filename = os.path.abspath(filename)
+    def to_xml_file(self, path=None):
+        path = os.path.abspath(path)
         if self.converters_path_attr:
             converters_path_str = ' converters_path="%s"' % self.converters_path_attr
         else:
@@ -894,19 +884,19 @@ class Registry(object):
             display_path_str = ' display_path="%s"' % self.display_path_attr
         else:
             display_path_str = ''
-        os.write(fd, '<?xml version="1.0"?>\n')
-        os.write(fd, '<datatypes>\n')
-        os.write(fd, '<registration%s%s>\n' % (converters_path_str, display_path_str))
-        for elem in self.datatype_elems:
-            os.write(fd, '%s' % galaxy.util.xml_to_string(elem))
-        os.write(fd, '</registration>\n')
-        os.write(fd, '<sniffers>\n')
-        for elem in self.sniffer_elems:
-            os.write(fd, '%s' % galaxy.util.xml_to_string(elem))
-        os.write(fd, '</sniffers>\n')
-        os.write(fd, '</datatypes>\n')
-        os.close(fd)
-        os.chmod(self.xml_filename, 0o644)
+        with open(path, 'w') as registry_xml:
+            os.chmod(path, 0o644)
+            registry_xml.write('<?xml version="1.0"?>\n')
+            registry_xml.write('<datatypes>\n')
+            registry_xml.write('<registration%s%s>\n' % (converters_path_str, display_path_str))
+            for elem in self.datatype_elems:
+                registry_xml.write('%s' % galaxy.util.xml_to_string(elem))
+            registry_xml.write('</registration>\n')
+            registry_xml.write('<sniffers>\n')
+            for elem in self.sniffer_elems:
+                registry_xml.write('%s' % galaxy.util.xml_to_string(elem))
+            registry_xml.write('</sniffers>\n')
+            registry_xml.write('</datatypes>\n')
 
     def get_extension(self, elem):
         """

lib/galaxy/jobs/__init__.py

@@ -1645,8 +1645,7 @@ class JobWrapper(object, HasResourceParameters):
 
     def setup_external_metadata(self, exec_dir=None, tmp_dir=None,
                                 dataset_files_path=None, config_root=None,
-                                config_file=None, datatypes_config=None,
-                                resolve_metadata_dependencies=False,
+                                config_file=None, resolve_metadata_dependencies=False,
                                 set_extension=True, **kwds):
         # extension could still be 'auto' if this is the upload tool.
         job = self.get_job()
@@ -1665,8 +1664,8 @@ class JobWrapper(object, HasResourceParameters):
             config_root = self.app.config.root
         if config_file is None:
             config_file = self.app.config.config_file
-        if datatypes_config is None:
-            datatypes_config = self.app.datatypes_registry.integrated_datatypes_configs
+        datatypes_config = os.path.join(self.working_directory, 'registry.xml')
+        self.app.datatypes_registry.to_xml_file(path=datatypes_config)
         command = self.external_output_metadata.setup_external_metadata([output_dataset_assoc.dataset for
                                                                          output_dataset_assoc in
                                                                          job.output_datasets + job.output_library_datasets],
@@ -1677,7 +1676,7 @@ class JobWrapper(object, HasResourceParameters):
                                                                         config_root=config_root,
                                                                         config_file=config_file,
                                                                         datatypes_config=datatypes_config,
-                                                                        job_metadata=os.path.join(self.tool_working_directory, self.tool.provided_metadata_file),
+                                                                        job_metadata=os.path.join(self.tool_working_directory, TOOL_PROVIDED_JOB_METADATA_FILE),
                                                                         max_metadata_value_size=self.app.config.max_metadata_value_size,
                                                                         **kwds)
         if resolve_metadata_dependencies:
@@ -1975,7 +1974,7 @@ class TaskWrapper(JobWrapper):
         pass
 
     def setup_external_metadata(self, exec_dir=None, tmp_dir=None, dataset_files_path=None,
-                                config_root=None, config_file=None, datatypes_config=None,
+                                config_root=None, config_file=None,
                                 set_extension=True, **kwds):
         # There is no metadata setting for tasks.  This is handled after the merge, at the job level.
         return ""

lib/galaxy/jobs/command_factory.py

@@ -198,7 +198,6 @@ def __handle_metadata(commands_builder, job_wrapper, runner, remote_command_para
     output_fnames = metadata_kwds.get('output_fnames', job_wrapper.get_output_fnames())
     config_root = metadata_kwds.get('config_root', None)
     config_file = metadata_kwds.get('config_file', None)
-    datatypes_config = metadata_kwds.get('datatypes_config', None)
     compute_tmp_dir = metadata_kwds.get('compute_tmp_dir', None)
     resolve_metadata_dependencies = job_wrapper.commands_in_new_shell
     metadata_command = job_wrapper.setup_external_metadata(
@@ -209,7 +208,6 @@ def __handle_metadata(commands_builder, job_wrapper, runner, remote_command_para
         set_extension=False,
         config_root=config_root,
         config_file=config_file,
-        datatypes_config=datatypes_config,
         compute_tmp_dir=compute_tmp_dir,
         resolve_metadata_dependencies=resolve_metadata_dependencies,
         kwds={'overwrite': False}

lib/galaxy/tools/evaluation.py

@@ -134,6 +134,7 @@ class ToolEvaluator(object):
             raise SyntaxError("Unbound variable input.")  # Don't let $input hang Python evaluation process.
 
         param_dict["input"] = input
+        param_dict['__datatypes_config__'] = param_dict['GALAXY_DATATYPES_CONF_FILE'] = os.path.join(job_working_directory, 'registry.xml')
 
         param_dict.update(self.tool.template_macro_params)
         # All parameters go into the param_dict
@@ -391,7 +392,6 @@ class ToolEvaluator(object):
         # For the upload tool, we need to know the root directory and the
         # datatypes conf path, so we can load the datatypes registry
         param_dict['__root_dir__'] = param_dict['GALAXY_ROOT_DIR'] = os.path.abspath(self.app.config.root)
-        param_dict['__datatypes_config__'] = param_dict['GALAXY_DATATYPES_CONF_FILE'] = self.app.datatypes_registry.integrated_datatypes_configs
         param_dict['__admin_users__'] = self.app.config.admin_users
         param_dict['__user__'] = RawObjectWrapper(param_dict.get('__user__', None))