Handle the following files for "installed" galaxy:

- config/data_manager_conf.xml - config/external_service_types_conf.xml - config/shed_data_manager_conf.xml - config/shed_tool_data_table_conf.xml - config/shed_data_manager_conf.xml - config/tool_data_table_conf.xml - config/workflow_schedulers_conf.xml - tool-data (for shed-installed data tables) shed_* are now created on demand when shed things are installed. Installed Galaxy can now be started from outside the Galaxy root with just a galaxy.ini. Some features will still be broken (static dbkeys, display build sites). Some (dependency resolvers) are broken just because I haven't gotten to fixing paths yet and will be easy to fix, others (dbkeys) are more complicated because of complex paths and relative-to-galaxy-root defaults (in datatypes_conf.xml).
2026-09-24 16:30:27 +08:00 · 2016-07-07 13:49:27 -04:00
parent 04f4584e59
commit 404dcbc104
16 changed files with 148 additions and 50 deletions
@@ -1,3 +0,0 @@
-<?xml version="1.0"?>
-<data_managers>
-</data_managers>
@@ -1,3 +0,0 @@
-<?xml version="1.0"?>
-<data_managers>
-</data_managers>
@@ -1,3 +0,0 @@
-<?xml version="1.0"?>
-<tables>
-</tables>
@@ -2,6 +2,7 @@
 Contains implementations of the authentication logic.
 """

+import errno
 import logging
 import xml.etree.ElementTree
 from collections import namedtuple
@@ -34,8 +35,8 @@ class AuthManager(object):
        try:
            ct = xml.etree.ElementTree.parse(auth_config_file)
            conf_root = ct.getroot()
-        except (OSError, IOError):
-            if not app.config.auth_config_file_set:
+        except (OSError, IOError) as exc:
+            if exc.errno == errno.ENOENT and not app.config.auth_config_file_set:
                conf_root = xml.etree.ElementTree.fromstring(AUTH_CONF_XML)
            else:
                raise
@@ -4,6 +4,7 @@ Universe configuration builder.
 # absolute_import needed for tool_shed package.
 from __future__ import absolute_import

+import errno
 import logging
 import logging.config
 import os
@@ -97,6 +98,9 @@ class Configuration( object ):

        # Configs no longer read from samples
        self.migrated_tools_config = resolve_path( kwargs.get( 'migrated_tools_conf', 'migrated_tools_conf.xml' ), self.mutable_config_dir )
+        self.shed_tool_conf = resolve_path( kwargs.get( 'shed_tool_conf', 'shed_tool_conf.xml' ), self.mutable_config_dir )
+        for name in ( 'migrated_tools_config', 'shed_tool_conf' ):
+            setattr( self, name + '_set', kwargs.get( name, None ) is not None )

        # Resolve paths of other config files
        self.__parse_config_file_options( kwargs )
@@ -134,6 +138,8 @@ class Configuration( object ):
        self.enable_unique_workflow_defaults = string_as_bool( kwargs.get( 'enable_unique_workflow_defaults', False ) )
        self.tool_path = resolve_path( kwargs.get( "tool_path", "tools" ), self.root )
        self.tool_data_path = resolve_path( kwargs.get( "tool_data_path", "tool-data" ), os.getcwd() )
+        if not self.running_from_source and kwargs.get( "tool_data_path", None ) is None:
+            self.tool_data_path = resolve_path( "tool-data", self.data_dir )
        self.builds_file_path = resolve_path( kwargs.get( "builds_file_path", os.path.join( self.tool_data_path, 'shared', 'ucsc', 'builds.txt') ), self.root )
        self.len_file_path = resolve_path( kwargs.get( "len_file_path", os.path.join( self.tool_data_path, 'shared', 'ucsc', 'chrom') ), self.root )
        # The value of migrated_tools_config is the file reserved for containing only those tools that have been eliminated from the distribution
@@ -585,18 +591,18 @@ class Configuration( object ):
        """
        defaults = dict(
            auth_config_file=[ self.__in_config_dir( 'auth_conf.xml' ) ],
-            data_manager_config_file=[ 'config/data_manager_conf.xml', 'data_manager_conf.xml', 'config/data_manager_conf.xml.sample' ],
+            data_manager_config_file=[ self.__in_config_dir( 'data_manager_conf.xml' ) ],
            datatypes_config_file=[ self.__in_config_dir( 'datatypes_conf.xml' ), self.__in_sample_dir( 'datatypes_conf.xml.sample' ) ],
-            external_service_type_config_file=[ 'config/external_service_types_conf.xml', 'external_service_types_conf.xml', 'config/external_service_types_conf.xml.sample' ],
+            external_service_type_config_file=[ self.__in_config_dir( 'external_service_types_conf.xml' ) ],
            job_config_file=[ 'config/job_conf.xml', 'job_conf.xml' ],
            job_metrics_config_file=[ 'config/job_metrics_conf.xml', 'job_metrics_conf.xml', 'config/job_metrics_conf.xml.sample' ],
            dependency_resolvers_config_file=[ 'config/dependency_resolvers_conf.xml', 'dependency_resolvers_conf.xml' ],
            job_resource_params_file=[ 'config/job_resource_params_conf.xml', 'job_resource_params_conf.xml' ],
            object_store_config_file=[ 'config/object_store_conf.xml', 'object_store_conf.xml' ],
            openid_config_file=[ 'config/openid_conf.xml', 'openid_conf.xml', 'config/openid_conf.xml.sample' ],
-            shed_data_manager_config_file=[ 'shed_data_manager_conf.xml', 'config/shed_data_manager_conf.xml' ],
-            shed_tool_data_table_config=[ 'shed_tool_data_table_conf.xml', 'config/shed_tool_data_table_conf.xml' ],
-            workflow_schedulers_config_file=['config/workflow_schedulers_conf.xml', 'config/workflow_schedulers_conf.xml.sample'],
+            shed_data_manager_config_file=[ self.__in_mutable_config_dir( 'shed_data_manager_conf.xml' ) ],
+            shed_tool_data_table_config=[ self.__in_mutable_config_dir( 'shed_tool_data_table_conf.xml' ) ],
+            workflow_schedulers_config_file=[ self.__in_config_dir( 'config/workflow_schedulers_conf.xml' ) ],
        )

        listify_defaults = dict(
@@ -617,6 +623,14 @@ class Configuration( object ):
                               'config/tool_conf.xml.sample,config/shed_tool_conf.xml' ]
        )

+        if not self.running_from_source:
+            listify_defaults['tool_data_table_config_path'] = [
+                    self.__in_config_dir( 'tool_data_table_conf.xml' ),
+                    self.__in_sample_dir( 'tool_data_table_conf.xml.sample' ) ]
+            listify_defaults['tool_config_file'] = [
+                    self.__in_config_dir( 'tool_conf.xml' ),
+                    self.__in_sample_dir( 'tool_conf.xml.sample' ) ]
+
        for var, defaults in defaults.items():
            if kwargs.get( var, None ) is not None:
                path = kwargs.get( var )
@@ -647,6 +661,12 @@ class Configuration( object ):
                    paths = listify( defaults[-1] )
            setattr( self, var, [ resolve_path( x, self.root ) for x in paths ] )

+        # If the user has configured a shed tool config in tool_config_file
+        # this would add a second, but since we're not parsing them yet we
+        # don't know if that's the case.
+        if not self.running_from_source and self.shed_tool_conf not in self.tool_config_file:
+            self.tool_config_file.append( self.shed_tool_conf )
+
        # Backwards compatibility for names used in too many places to fix
        self.datatypes_config = self.datatypes_config_file
        self.tool_configs = self.tool_config_file
@@ -730,7 +750,7 @@ class Configuration( object ):
        if self.migrated_tools_config not in tool_configs and os.path.exists( self.migrated_tools_config ):
            tool_configs.append( self.migrated_tools_config )
        for path in tool_configs:
-            if not os.path.exists( path ):
+            if not os.path.exists( path ) and path != self.shed_tool_conf:
                raise ConfigurationError("Tool config file not found: %s" % path )
        for datatypes_config in listify( self.datatypes_config ):
            if not os.path.isfile( datatypes_config ):
@@ -915,9 +935,14 @@ class ConfiguresGalaxyMixin:
        self.tool_data_tables = ToolDataTableManager( tool_data_path=self.config.tool_data_path,
                                                      config_filename=self.config.tool_data_table_config_path )
        # Load additional entries defined by self.config.shed_tool_data_table_config into tool data tables.
-        self.tool_data_tables.load_from_config_file( config_filename=self.config.shed_tool_data_table_config,
-                                                     tool_data_path=self.tool_data_tables.tool_data_path,
-                                                     from_shed_config=from_shed_config )
+        try:
+            self.tool_data_tables.load_from_config_file( config_filename=self.config.shed_tool_data_table_config,
+                                                         tool_data_path=self.tool_data_tables.tool_data_path,
+                                                         from_shed_config=from_shed_config )
+        except (OSError, IOError) as exc:
+            # Missing shed_tool_data_table_config is okay if it's the default
+            if exc.errno != errno.ENOENT or self.config.shed_tool_data_table_config_set:
+                raise

    def _configure_datatypes_registry( self, installed_repository_manager=None ):
        from galaxy.datatypes import registry
@@ -1,69 +1,69 @@
-<!-- Use the file tool_data_table_conf.xml.oldlocstyle if you don't want to update your loc files as changed in revision 4550:535d276c92bc-->
+<!-- Paths are relative to the value of `tool_data_path` in galaxy.ini -->
 <tables>
    <!-- Locations of all fasta files under genome directory -->
    <table name="all_fasta" comment_char="#">
        <columns>value, dbkey, name, path</columns>
-        <file path="tool-data/all_fasta.loc" />
+        <file path="all_fasta.loc" />
    </table>
    <!-- Locations of indexes in the BFAST mapper format -->
    <table name="bfast_indexes" comment_char="#">
        <columns>value, dbkey, formats, name, path</columns>
-        <file path="tool-data/bfast_indexes.loc" />
+        <file path="bfast_indexes.loc" />
    </table>
    <!-- Locations of protein (mega)blast databases -->
    <table name="blastdb_p" comment_char="#">
        <columns>value, name, path</columns>
-        <file path="tool-data/blastdb_p.loc" />
+        <file path="blastdb_p.loc" />
    </table>
    <!-- Locations of indexes in the BWA mapper format -->
    <table name="bwa_indexes" comment_char="#">
        <columns>value, dbkey, name, path</columns>
-        <file path="tool-data/bwa_index.loc" />
+        <file path="bwa_index.loc" />
    </table>
    <!-- Locations of indexes in the BWA color-space mapper format -->
    <table name="bwa_indexes_color" comment_char="#">
        <columns>value, dbkey, name, path</columns>
-        <file path="tool-data/bwa_index_color.loc" />
+        <file path="bwa_index_color.loc" />
    </table>
    <!-- Locations of MAF files that have been indexed with bx-python -->
    <table name="indexed_maf_files">
        <columns>name, value, dbkey, species</columns>
-        <file path="tool-data/maf_index.loc" />
+        <file path="maf_index.loc" />
    </table>
    <!-- Locations of fasta files appropriate for NGS simulation -->
    <table name="ngs_sim_fasta" comment_char="#">
        <columns>value, dbkey, name, path</columns>
-        <file path="tool-data/ngs_sim_fasta.loc" />
+        <file path="ngs_sim_fasta.loc" />
    </table>
    <!-- Locations of PerM base index files -->
    <table name="perm_base_indexes" comment_char="#">
        <columns>value, name, path</columns>
-        <file path="tool-data/perm_base_index.loc" />
+        <file path="perm_base_index.loc" />
    </table>
    <!-- Locations of PerM color-space index files -->
    <table name="perm_color_indexes" comment_char="#">
        <columns>value, name, path</columns>
-        <file path="tool-data/perm_color_index.loc" />
+        <file path="perm_color_index.loc" />
    </table>
    <!-- Location of Picard dict file and other files -->
    <table name="picard_indexes" comment_char="#">
        <columns>value, dbkey, name, path</columns>
-        <file path="tool-data/picard_index.loc" />
+        <file path="picard_index.loc" />
    </table>
    <!-- Location of SRMA dict file and other files -->
    <table name="srma_indexes" comment_char="#">
        <columns>value, dbkey, name, path</columns>
-        <file path="tool-data/picard_index.loc" />
+        <file path="picard_index.loc" />
    </table>
    <!-- Location of Mosaik files -->
    <table name="mosaik_indexes" comment_char="#">
        <columns>value, dbkey, name, path</columns>
-        <file path="tool-data/mosaik_index.loc" />
+        <file path="mosaik_index.loc" />
    </table>
    <!-- Locations of indexes in the 2bit format -->
    <table name="twobit" comment_char="#">
        <columns>value, path</columns>
-        <file path="tool-data/twobit.loc" />
+        <file path="twobit.loc" />
    </table>
    <!-- Available IGV builds, loaded from URL -->
    <table name="igv_broad_genomes" comment_char="#">
@@ -73,21 +73,21 @@
    <!-- Available liftOver chain file -->
    <table name="liftOver" comment_char="#">
        <columns>dbkey, name, value</columns>
-        <file path="tool-data/liftOver.loc" />
+        <file path="liftOver.loc" />
    </table>
    <!-- iobio bam servers -->
    <table name="bam_iobio" comment_char="#">
        <columns>value, name, url</columns>
-        <file path="tool-data/bam_iobio.loc" />
+        <file path="bam_iobio.loc" />
    </table>
    <!-- iobio vcf servers -->
    <table name="vcf_iobio" comment_char="#">
        <columns>value, name, url</columns>
-        <file path="tool-data/vcf_iobio.loc" />
+        <file path="vcf_iobio.loc" />
    </table>
    <!-- simple biom servers -->
    <table name="biom_simple_display" comment_char="#">
        <columns>value, name, url</columns>
-        <file path="tool-data/biom_simple_display.loc" />
+        <file path="biom_simple_display.loc" />
    </table>
 </tables>
@@ -1,10 +1,14 @@
+import errno
 import os
 import logging
+
 from galaxy.util.odict import odict
 from galaxy import util, model
 from galaxy.forms.forms import form_factory
 from galaxy.external_services.service import ExternalServiceActionsGroup
 from galaxy.sample_tracking.data_transfer import data_transfer_factories
+
+
 log = logging.getLogger( __name__ )


@@ -25,8 +29,13 @@ class ExternalServiceTypesCollection( object ):

    def load_all( self, config_filename ):
        self.visible_external_service_types = []
-        tree = util.parse_xml( config_filename )
-        root = tree.getroot()
+        try:
+            tree = util.parse_xml( config_filename )
+            root = tree.getroot()
+        except (OSError, IOError) as exc:
+            if exc.errno != errno.ENOENT or self.app.config.external_service_type_config_file_set:
+                raise
+            root = []
        for elem in root:
            try:
                if elem.tag == 'external_service_type':
@@ -6,6 +6,7 @@ users to configure data tables for a local Galaxy instance without needing
 to modify the tool configurations.
 """

+import errno
 import logging
 import os
 import os.path
@@ -26,11 +27,16 @@ log = logging.getLogger( __name__ )

 DEFAULT_TABLE_TYPE = 'tabular'

+TOOL_DATA_TABLE_CONF_XML = """<?xml version="1.0"?>
+<tables>
+</tables>
+"""
+

 class ToolDataTableManager( object ):
    """Manages a collection of tool data tables"""

-    def __init__( self, tool_data_path, config_filename=None ):
+    def __init__( self, tool_data_path, config_filename=None, tool_data_table_config_path_set=None ):
        self.tool_data_path = tool_data_path
        # This stores all defined data table entries from both the tool_data_table_conf.xml file and the shed_tool_data_table_conf.xml file
        # at server startup. If tool shed repositories are installed that contain a valid file named tool_data_table_conf.xml.sample, entries
@@ -141,12 +147,20 @@ class ToolDataTableManager( object ):
        full_path = os.path.abspath( shed_tool_data_table_config )
        # FIXME: we should lock changing this file by other threads / head nodes
        try:
-            tree = util.parse_xml( full_path )
+            try:
+                tree = util.parse_xml( full_path )
+            except (OSError, IOError) as exc:
+                if exc.errno == errno.ENOENT:
+                    with open( full_path, 'w' ) as fh:
+                        fh.write( TOOL_DATA_TABLE_CONF_XML )
+                    tree = util.parse_xml( full_path )
+                else:
+                    raise
            root = tree.getroot()
            out_elems = [ elem for elem in root ]
-        except Exception as e:
+        except Exception as exc:
            out_elems = []
-            log.debug( 'Could not parse existing tool data table config, assume no existing elements: %s', e )
+            log.debug( 'Could not parse existing tool data table config, assume no existing elements: %s', exc )
        for elem in remove_elems:
            # handle multiple occurrences of remove elem in existing elems
            while elem in out_elems:
@@ -340,6 +354,12 @@ class TabularToolDataTable( ToolDataTable, Dictifiable ):
                self._update_version()
            else:
                self.missing_index_file = filename
+                # TODO: some data tables need to exist (even if they are empty)
+                # for tools to load. In an installed Galaxy environment and the
+                # default tool_data_table_conf.xml, this will emit spurious
+                # warnings about missing location files that would otherwise be
+                # empty and we don't care about unless the admin chooses to
+                # populate them.
                log.warning( "Cannot find index file '%s' for tool data table '%s'" % ( filename, self.name ) )

            if filename not in self.filenames or not self.filenames[ filename ][ 'found' ]:
@@ -37,6 +37,10 @@ class DataManagers( object ):
    def load_from_xml( self, xml_filename, store_tool_path=True, replace_existing=False ):
        try:
            tree = util.parse_xml( xml_filename )
+        except (IOError, OSError) as e:
+            if e.errno != errno.ENOENT or self.app.config.data_manager_config_file_set:
+                raise
+            return  # default config option and it doesn't exist, which is fine
        except Exception as e:
            log.error( 'There was an error parsing your Data Manager config file "%s": %s' % ( xml_filename, e ) )
            return  # we are not able to load any data managers
@@ -1,3 +1,4 @@
+import errno
 import logging
 import os
 import string
@@ -39,6 +40,11 @@ from galaxy.tools.deps import build_dependency_manager

 log = logging.getLogger( __name__ )

+SHED_TOOL_CONF_XML = """<?xml version="1.0"?>
+<toolbox tool_path="{shed_tools_dir}">
+</toolbox>
+"""
+

 class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
    """
@@ -124,7 +130,21 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):

        """
        log.info( "Parsing the tool configuration %s" % config_filename )
-        tool_conf_source = get_toolbox_parser(config_filename)
+        try:
+            tool_conf_source = get_toolbox_parser(config_filename)
+        except (OSError, IOError) as exc:
+            for opt in ('shed_tool_conf', 'migrated_tools_config'):
+                if (config_filename == getattr( self.app.config, opt )
+                        and not getattr( self.app.config, opt + '_set' )
+                        and exc.errno == errno.ENOENT):
+                    log.debug( "Skipping loading missing default config file: %s", config_filename )
+                    stcd = dict( config_filename=config_filename,
+                                 tool_path=self.app.config.shed_tools_dir,
+                                 config_elems=[],
+                                 create=SHED_TOOL_CONF_XML.format(shed_tools_dir=self.app.config.shed_tools_dir) )
+                    self._dynamic_tool_confs.append( stcd )
+                    return
+            raise
        tool_path = tool_conf_source.parse_tool_path()
        parsing_shed_tool_conf = tool_conf_source.is_shed_tool_conf()
        if parsing_shed_tool_conf:
@@ -93,8 +93,8 @@ class WorkflowSchedulingManager( object ):
    def __init_schedulers( self ):
        config_file = self.app.config.workflow_schedulers_config_file
        use_default_scheduler = False
-        if not config_file:
-            log.info( "Not workflow schedulers plugin config file defined, using default scheduler." )
+        if not config_file or (not os.path.exists( config_file ) and not self.app.config.workflow_schedulers_config_file_set):
+            log.info( "No workflow schedulers plugin config file defined, using default scheduler." )
            use_default_scheduler = True
        elif not os.path.exists( config_file ):
            log.info( "Cannot find workflow schedulers plugin config file '%s', using default scheduler." % config_file )
@@ -1,3 +1,4 @@
+import errno
 import logging
 import os
 import threading
@@ -7,6 +8,11 @@ from tool_shed.util import xml_util

 log = logging.getLogger( __name__ )

+SHED_DATA_MANAGER_CONF_XML = """<?xml version="1.0"?>
+<data_managers>
+</data_managers>
+"""
+

 class DataManagerHandler( object ):

@@ -41,7 +47,15 @@ class DataManagerHandler( object ):
            for tool_tup in repository_tools_tups:
                repository_tools_by_guid[ tool_tup[ 1 ] ] = dict( tool_config_filename=tool_tup[ 0 ], tool=tool_tup[ 2 ] )
            # Load existing data managers.
-            tree, error_message = xml_util.parse_xml( shed_data_manager_conf_filename )
+            try:
+                tree, error_message = xml_util.parse_xml( shed_data_manager_conf_filename )
+            except (OSError, IOError) as exc:
+                if exc.errno == errno.ENOENT:
+                    with open( shed_data_manager_conf_filename, 'w' ) as fh:
+                        fh.write( SHED_DATA_MANAGER_CONF_XML )
+                    tree, error_message = xml_util.parse_xml( shed_data_manager_conf_filename )
+                else:
+                    raise
            if tree is None:
                return rval
            config_elems = [ elem for elem in tree.getroot() ]
@@ -1,3 +1,4 @@
+import errno
 import logging
 import threading

@@ -26,7 +27,15 @@ class ToolPanelManager( object ):
        shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ]
        tool_path = shed_tool_conf_dict[ 'tool_path' ]
        config_elems = []
-        tree, error_message = xml_util.parse_xml( shed_tool_conf )
+        try:
+            tree, error_message = xml_util.parse_xml( shed_tool_conf )
+        except (OSError, IOError) as exc:
+            if (exc.errno == errno.ENOENT and shed_tool_conf_dict.get( 'create', None ) is not None ):
+                with open( shed_tool_conf, 'w' ) as fh:
+                    fh.write( shed_tool_conf_dict['create'] )
+                tree, error_message = xml_util.parse_xml( shed_tool_conf )
+            else:
+                raise
        if tree:
            root = tree.getroot()
            for elem in root:
@@ -144,7 +144,12 @@ def get_non_shed_tool_panel_configs( app ):
    for config_filename in app.config.tool_configs:
        # Any config file that includes a tool_path attribute in the root tag set like the following is shed-related.
        # <toolbox tool_path="../shed_tools">
-        tree, error_message = xml_util.parse_xml( config_filename )
+        try:
+            tree, error_message = xml_util.parse_xml( config_filename )
+        except (OSError, IOError) as exc:
+            if config_filename == app.config.shed_tool_conf and not app.config.shed_tool_conf_set:
+                continue
+            raise
        if tree is None:
            continue
        root = tree.getroot()