Merge pull request #3503 from jmchilton/datatypes_build_sites

Improvements to build sites configuration

config/build_sites.yml.sample

@@ -0,0 +1,22 @@
+# Build sites define the builds (dbkeys) available at sites used by display
+# applications and the URL to those sites.
+
+# The `display` attributes on the `ucsc` and `gbrowse` sites replace the
+# `ucsc_display_sites` and `gbrowse_display_sites` options in galaxy.ini.
+# Because these are used by "old-style" display applications, their types
+# cannot change if you want the old-style display links for these sites to
+# work.
+- type: ucsc
+  file: tool-data/shared/ucsc/ucsc_build_sites.txt
+  display: [main,test,archaea,ucla]
+- type: gbrowse
+  file: tool-data/shared/gbrowse/gbrowse_build_sites.txt
+  display: [modencode,sgd_yeast,tair,wormbase,wormbase_ws120,wormbase_ws140,wormbase_ws170,wormbase_ws180,wormbase_ws190,wormbase_ws200,wormbase_ws204,wormbase_ws210,wormbase_ws220,wormbase_ws225]
+- type: ensembl
+  file: tool-data/shared/ensembl/ensembl_sites.txt
+- type: ensembl_data_url
+  file: tool-data/shared/ensembl/ensembl_sites_data_URL.txt
+- type: igv
+  file: tool-data/shared/igv/igv_build_sites.txt
+- type: rviewer
+  file: tool-data/shared/rviewer/rviewer_build_sites.txt

config/datatypes_conf.xml.sample

@@ -736,22 +736,4 @@
     <sniffer type="galaxy.datatypes.binary:OxliSubset"/>
     <sniffer type="galaxy.datatypes.binary:OxliGraphLabels"/>
   </sniffers>
-  <build_sites>
-      <!--
-      Build sites define the builds (dbkeys) available at sites used by display
-      applications and the URL to those sites.
-
-      The `display` attributes on the `ucsc` and `gbrowse` sites replace the
-      `ucsc_display_sites` and `gbrowse_display_sites` options in galaxy.ini.
-      Because these are used by "old-style" display applications, their types
-      cannot change if you want the old-style display links for these sites to
-      work.
-      -->
-      <site type="ucsc" file="tool-data/shared/ucsc/ucsc_build_sites.txt" display="main,test,archaea,ucla"/>
-      <site type="gbrowse" file="tool-data/shared/gbrowse/gbrowse_build_sites.txt" display="modencode,sgd_yeast,tair,wormbase,wormbase_ws120,wormbase_ws140,wormbase_ws170,wormbase_ws180,wormbase_ws190,wormbase_ws200,wormbase_ws204,wormbase_ws210,wormbase_ws220,wormbase_ws225"/>
-      <site type="ensembl" file="tool-data/shared/ensembl/ensembl_sites.txt"/>
-      <site type="ensembl_data_url" file="tool-data/shared/ensembl/ensembl_sites_data_URL.txt"/>
-      <site type="igv" file="tool-data/shared/igv/igv_build_sites.txt"/>
-      <site type="rviewer" file="tool-data/shared/rviewer/rviewer_build_sites.txt"/>
-  </build_sites>
 </datatypes>

lib/galaxy/config.py

@@ -36,6 +36,7 @@ PATH_DEFAULTS = dict(
     auth_config_file=['config/auth_conf.xml', 'config/auth_conf.xml.sample'],
     data_manager_config_file=['config/data_manager_conf.xml', 'data_manager_conf.xml', 'config/data_manager_conf.xml.sample'],
     datatypes_config_file=['config/datatypes_conf.xml', 'datatypes_conf.xml', 'config/datatypes_conf.xml.sample'],
+    build_sites_config_file=['config/build_sites.yml', 'config/build_sites.yml.sample'],
     external_service_type_config_file=['config/external_service_types_conf.xml', 'external_service_types_conf.xml', 'config/external_service_types_conf.xml.sample'],
     job_config_file=['config/job_conf.xml', 'job_conf.xml'],
     tool_destinations_config_file=['config/tool_destinations.yml', 'config/tool_destinations.yml.sample'],
@@ -961,7 +962,7 @@ class ConfiguresGalaxyMixin:
     def _configure_datatypes_registry( self, installed_repository_manager=None ):
         from galaxy.datatypes import registry
         # Create an empty datatypes registry.
-        self.datatypes_registry = registry.Registry()
+        self.datatypes_registry = registry.Registry( self.config )
         if installed_repository_manager:
             # Load proprietary datatypes defined in datatypes_conf.xml files in all installed tool shed repositories.  We
             # load proprietary datatypes before datatypes in the distribution because Galaxy's default sniffers include some

lib/galaxy/datatypes/registry.py

@@ -4,9 +4,14 @@ Provides mapping between extensions and datatypes, mime-types, etc.
 from __future__ import absolute_import
 
 import os
-import tempfile
-import logging
 import imp
+import logging
+import tempfile
+
+import yaml
+
+import galaxy.util
+
 from . import data
 from . import tabular
 from . import interval
@@ -18,7 +23,6 @@ from . import coverage
 from . import tracks
 from . import binary
 from . import text
-import galaxy.util
 from galaxy.util.odict import odict
 from .display_applications.application import DisplayApplication
 
@@ -29,9 +33,10 @@ class ConfigurationError( Exception ):
 
 class Registry( object ):
 
-    def __init__( self ):
+    def __init__( self, config=None ):
         self.log = logging.getLogger(__name__)
         self.log.addHandler( logging.NullHandler() )
+        self.config = config
         self.datatypes_by_extension = {}
         self.mimetypes_by_extension = {}
         self.datatype_converters = odict()
@@ -295,7 +300,7 @@ class Registry( object ):
                                          override=override )
             self.upload_file_formats.sort()
             # Load build sites
-            self.load_build_sites( root )
+            self._load_build_sites( root )
             # Persist the xml form of the registry into a temporary file so that it can be loaded from the command line by tools and
             # set_metadata processing.
             self.to_xml_file()
@@ -314,23 +319,50 @@ class Registry( object ):
                     self.sniff_order.append( datatype )
         append_to_sniff_order()
 
-    def load_build_sites( self, root ):
+    def _load_build_sites( self, root ):
+
+        def load_build_site( build_site_config ):
+            # Take in either an XML element or simple dictionary from YAML and add build site for this.
+            if not (build_site_config.get( 'type' ) and build_site_config.get( 'file' )):
+                self.log.exception( "Site is missing required 'type' and 'file' attributes: %s" )
+                return
+
+            site_type = build_site_config.get( 'type' )
+            path = build_site_config.get( 'file' )
+            if not os.path.exists( path ):
+                sample_path = "%s.sample" % path
+                if os.path.exists( sample_path ):
+                    self.log.debug( "Build site file [%s] not found using sample [%s]." % ( path, sample_path ) )
+                    path = sample_path
+
+            self.build_sites[site_type] = path
+            if site_type in ('ucsc', 'gbrowse'):
+                self.legacy_build_sites[site_type] = galaxy.util.read_build_sites( path )
+            if build_site_config.get( 'display', None ):
+                display = build_site_config.get( 'display' )
+                if not isinstance( display, list ):
+                    display = [ x.strip() for x in display.lower().split( ',' ) ]
+                self.display_sites[site_type] = display
+                self.log.debug( "Loaded build site '%s': %s with display sites: %s", site_type, path, display )
+            else:
+                self.log.debug( "Loaded build site '%s': %s", site_type, path )
+
         if root.find( 'build_sites' ) is not None:
             for elem in root.find( 'build_sites' ).findall( 'site' ):
-                if not (elem.get( 'type' ) and elem.get( 'file' )):
-                    self.log.exception( "Site is missing required 'type' and 'file' attributes: %s" )
-                else:
-                    site_type = elem.get( 'type' )
-                    file = elem.get( 'file' )
-                    self.build_sites[site_type] = file
-                    if site_type in ('ucsc', 'gbrowse'):
-                        self.legacy_build_sites[site_type] = galaxy.util.read_build_sites( file )
-                    if elem.get( 'display', None ):
-                        display = elem.get( 'display' )
-                        self.display_sites[site_type] = [ x.strip() for x in display.lower().split( ',' ) ]
-                        self.log.debug( "Loaded build site '%s': %s with display sites: %s", site_type, file, display )
-                    else:
-                        self.log.debug( "Loaded build site '%s': %s", site_type, file )
+                load_build_site( elem )
+        else:
+            build_sites_config_file = getattr( self.config, "build_sites_config_file", None  )
+            if build_sites_config_file and os.path.exists( build_sites_config_file ):
+                with open( build_sites_config_file, "r" ) as f:
+                    build_sites_config = yaml.load( f )
+                if not isinstance( build_sites_config, list ):
+                    self.log.exception( "Build sites configuration YAML file does not declare list of sites." )
+                    return
+
+                for build_site_config in build_sites_config:
+                    load_build_site( build_site_config )
+            else:
+                self.log.debug("No build sites source located.")
 
     def get_legacy_sites_by_build( self, site_type, build ):
         sites = []

scripts/common_startup.sh

@@ -38,9 +38,6 @@ SAMPLES="
     lib/tool_shed/scripts/bootstrap_tool_shed/user_info.xml.sample
     tool-data/shared/ucsc/builds.txt.sample
     tool-data/shared/ucsc/manual_builds.txt.sample
-    tool-data/shared/ucsc/ucsc_build_sites.txt.sample
-    tool-data/shared/igv/igv_build_sites.txt.sample
-    tool-data/shared/rviewer/rviewer_build_sites.txt.sample
     static/welcome.html.sample
 "