Allow loading dependency resolvers and job metrics from YAML instead of XML.

Mostly for downstream Pulsar configuration improvements - but I don't see a reason not to allow this in Galaxy as well.

Besides simply being more pleasant to write - having everything configurable by Python dictionary should enable easier automation and programatic configuration of these job running systems.

lib/galaxy/jobs/metrics/__init__.py

@@ -1,8 +1,6 @@
 import collections
 import os
 
-from xml.etree import ElementTree
-
 from galaxy.util import plugin_config
 from galaxy import util
 
@@ -37,7 +35,7 @@ class JobMetrics( object ):
         self.set_destination_instrumenter( destination_id, instrumenter )
 
     def set_destination_conf_element( self, destination_id, element ):
-        instrumenter = JobInstrumenter( self.plugin_classes, element )
+        instrumenter = JobInstrumenter( self.plugin_classes, ('xml', element) )
         self.set_destination_instrumenter( destination_id, instrumenter )
 
     def set_destination_instrumenter( self, destination_id, job_instrumenter=None ):
@@ -69,10 +67,10 @@ NULL_JOB_INSTRUMENTER = NullJobInstrumenter()
 
 class JobInstrumenter( object ):
 
-    def __init__( self, plugin_classes, metrics_element, **kwargs ):
+    def __init__( self, plugin_classes, plugins_source, **kwargs ):
         self.extra_kwargs = kwargs
         self.plugin_classes = plugin_classes
-        self.plugins = self.__plugins_for_element( metrics_element )
+        self.plugins = self.__plugins_from_source( plugins_source )
 
     def pre_execute_commands( self, job_directory ):
         commands = []
@@ -107,12 +105,12 @@ class JobInstrumenter( object ):
                 log.exception( "Failed to collect job properties for plugin %s" % plugin )
         return per_plugin_properites
 
-    def __plugins_for_element( self, plugins_element ):
-        return plugin_config.load_plugins_from_element(self.plugin_classes, plugins_element, self.extra_kwargs)
+    def __plugins_from_source( self, plugins_source ):
+        return plugin_config.load_plugins(self.plugin_classes, plugins_source, self.extra_kwargs)
 
     @staticmethod
     def from_file( plugin_classes, conf_file, **kwargs ):
         if not conf_file or not os.path.exists( conf_file ):
             return NULL_JOB_INSTRUMENTER
-        plugins_element = ElementTree.parse( conf_file ).getroot()
-        return JobInstrumenter( plugin_classes, plugins_element, **kwargs )
+        plugins_source = plugin_config.plugin_source_from_path( conf_file )
+        return JobInstrumenter( plugin_classes, plugins_source, **kwargs )

lib/galaxy/tools/deps/__init__.py

@@ -7,8 +7,6 @@ import os.path
 import logging
 log = logging.getLogger( __name__ )
 
-from xml.etree import ElementTree
-
 from .resolvers import INDETERMINATE_DEPENDENCY
 from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver
 from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver
@@ -94,8 +92,8 @@ class DependencyManager( object ):
     def __build_dependency_resolvers( self, conf_file ):
         if not conf_file or not os.path.exists( conf_file ):
             return self.__default_dependency_resolvers()
-        root = ElementTree.parse( conf_file ).getroot()
-        return self.__parse_resolver_conf_xml( root )
+        plugin_source = plugin_config.plugin_source_from_path( conf_file )
+        return self.__parse_resolver_conf_xml( plugin_source )
 
     def __default_dependency_resolvers( self ):
         return [
@@ -104,14 +102,11 @@ class DependencyManager( object ):
             GalaxyPackageDependencyResolver(self, versionless=True),
         ]
 
-    def __parse_resolver_conf_xml(self, root):
+    def __parse_resolver_conf_xml(self, plugin_source):
         """
-
-        :param root: Object representing the root ``<dependency_resolvers>`` object in the file.
-        :type root: ``xml.etree.ElementTree.Element``
         """
         extra_kwds = dict( dependency_manager=self )
-        return plugin_config.load_plugins_from_element( self.resolver_classes, root, extra_kwds )
+        return plugin_config.load_plugins( self.resolver_classes, plugin_source, extra_kwds )
 
     def __resolvers_dict( self ):
         import galaxy.tools.deps.resolvers

lib/galaxy/util/plugin_config.py

@@ -1,3 +1,16 @@
+from xml.etree import ElementTree
+
+try:
+    from galaxy import eggs
+    eggs.require('PyYAML')
+except Exception:
+    # If not in Galaxy, ignore this.
+    pass
+try:
+    import yaml
+except ImportError:
+    yaml = None
+
 from galaxy.util.submodules import submodules
 
 
@@ -19,7 +32,15 @@ def plugins_dict(module, plugin_type_identifier):
     return plugin_dict
 
 
-def load_plugins_from_element(plugins_dict, plugins_element, extra_kwds={}):
+def load_plugins(plugins_dict, plugin_source, extra_kwds={}):
+    source_type, source = plugin_source
+    if source_type == "xml":
+        return __load_plugins_from_element(plugins_dict, source, extra_kwds)
+    else:
+        return __load_plugins_from_dicts(plugins_dict, source, extra_kwds)
+
+
+def __load_plugins_from_element(plugins_dict, plugins_element, extra_kwds):
     plugins = []
 
     for plugin_element in plugins_element.getchildren():
@@ -30,3 +51,31 @@ def load_plugins_from_element(plugins_dict, plugins_element, extra_kwds={}):
         plugins.append( plugin )
 
     return plugins
+
+
+def __load_plugins_from_dicts(plugins_dict, configs, extra_kwds):
+    plugins = []
+
+    for config in configs:
+        plugin_type = config[ "type" ]
+        plugin_kwds = config
+        plugin_kwds.update( extra_kwds )
+        plugin = plugins_dict[ plugin_type ]( **plugin_kwds )
+        plugins.append( plugin )
+
+    return plugins
+
+
+def plugin_source_from_path(path):
+    if path.endswith(".yaml") or path.endswith(".yml"):
+        return ('dict', __read_yaml(path))
+    else:
+        return ('xml', ElementTree.parse( path ).getroot())
+
+
+def __read_yaml(path):
+    if yaml is None:
+        raise ImportError("Attempting to read YAML configuration file - but PyYAML dependency unavailable.")
+
+    with open(path, "rb") as f:
+        return yaml.load(f)