From 404dcbc1040ae301f480adb7d1999f0af0bd99a9 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 13 Jun 2016 15:51:16 -0400 Subject: [PATCH] Handle the following files for "installed" galaxy: - config/data_manager_conf.xml - config/external_service_types_conf.xml - config/shed_data_manager_conf.xml - config/shed_tool_data_table_conf.xml - config/shed_data_manager_conf.xml - config/tool_data_table_conf.xml - config/workflow_schedulers_conf.xml - tool-data (for shed-installed data tables) shed_* are now created on demand when shed things are installed. Installed Galaxy can now be started from outside the Galaxy root with just a galaxy.ini. Some features will still be broken (static dbkeys, display build sites). Some (dependency resolvers) are broken just because I haven't gotten to fixing paths yet and will be easy to fix, others (dbkeys) are more complicated because of complex paths and relative-to-galaxy-root defaults (in datatypes_conf.xml). --- config/data_manager_conf.xml.sample | 3 -- config/shed_data_manager_conf.xml.sample | 3 -- config/shed_tool_data_table_conf.xml.sample | 3 -- lib/galaxy/auth/__init__.py | 5 ++- lib/galaxy/config/__init__.py | 43 +++++++++++++++---- .../external_service_types_conf.xml.sample | 0 .../sample}/tool_data_table_conf.xml.sample | 36 ++++++++-------- .../workflow_schedulers_conf.xml.sample | 0 .../sample_tracking/external_service_types.py | 13 +++++- lib/galaxy/tools/data/__init__.py | 28 ++++++++++-- lib/galaxy/tools/data_manager/manager.py | 4 ++ lib/galaxy/tools/toolbox/base.py | 22 +++++++++- lib/galaxy/workflow/scheduling_manager.py | 4 +- .../galaxy_install/tools/data_manager.py | 16 ++++++- .../tools/tool_panel_manager.py | 11 ++++- lib/tool_shed/util/common_util.py | 7 ++- 16 files changed, 148 insertions(+), 50 deletions(-) delete mode 100644 config/data_manager_conf.xml.sample delete mode 100644 config/shed_data_manager_conf.xml.sample delete mode 100644 config/shed_tool_data_table_conf.xml.sample rename {config => lib/galaxy/config/sample}/external_service_types_conf.xml.sample (100%) rename {config => lib/galaxy/config/sample}/tool_data_table_conf.xml.sample (75%) rename {config => lib/galaxy/config/sample}/workflow_schedulers_conf.xml.sample (100%) diff --git a/config/data_manager_conf.xml.sample b/config/data_manager_conf.xml.sample deleted file mode 100644 index 04b4637c390..00000000000 --- a/config/data_manager_conf.xml.sample +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/config/shed_data_manager_conf.xml.sample b/config/shed_data_manager_conf.xml.sample deleted file mode 100644 index 04b4637c390..00000000000 --- a/config/shed_data_manager_conf.xml.sample +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/config/shed_tool_data_table_conf.xml.sample b/config/shed_tool_data_table_conf.xml.sample deleted file mode 100644 index fd376e4ed3b..00000000000 --- a/config/shed_tool_data_table_conf.xml.sample +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/lib/galaxy/auth/__init__.py b/lib/galaxy/auth/__init__.py index 80a802377d4..822ab1ad505 100644 --- a/lib/galaxy/auth/__init__.py +++ b/lib/galaxy/auth/__init__.py @@ -2,6 +2,7 @@ Contains implementations of the authentication logic. """ +import errno import logging import xml.etree.ElementTree from collections import namedtuple @@ -34,8 +35,8 @@ class AuthManager(object): try: ct = xml.etree.ElementTree.parse(auth_config_file) conf_root = ct.getroot() - except (OSError, IOError): - if not app.config.auth_config_file_set: + except (OSError, IOError) as exc: + if exc.errno == errno.ENOENT and not app.config.auth_config_file_set: conf_root = xml.etree.ElementTree.fromstring(AUTH_CONF_XML) else: raise diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 2d20b226b05..8de20500e2e 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -4,6 +4,7 @@ Universe configuration builder. # absolute_import needed for tool_shed package. from __future__ import absolute_import +import errno import logging import logging.config import os @@ -97,6 +98,9 @@ class Configuration( object ): # Configs no longer read from samples self.migrated_tools_config = resolve_path( kwargs.get( 'migrated_tools_conf', 'migrated_tools_conf.xml' ), self.mutable_config_dir ) + self.shed_tool_conf = resolve_path( kwargs.get( 'shed_tool_conf', 'shed_tool_conf.xml' ), self.mutable_config_dir ) + for name in ( 'migrated_tools_config', 'shed_tool_conf' ): + setattr( self, name + '_set', kwargs.get( name, None ) is not None ) # Resolve paths of other config files self.__parse_config_file_options( kwargs ) @@ -134,6 +138,8 @@ class Configuration( object ): self.enable_unique_workflow_defaults = string_as_bool( kwargs.get( 'enable_unique_workflow_defaults', False ) ) self.tool_path = resolve_path( kwargs.get( "tool_path", "tools" ), self.root ) self.tool_data_path = resolve_path( kwargs.get( "tool_data_path", "tool-data" ), os.getcwd() ) + if not self.running_from_source and kwargs.get( "tool_data_path", None ) is None: + self.tool_data_path = resolve_path( "tool-data", self.data_dir ) self.builds_file_path = resolve_path( kwargs.get( "builds_file_path", os.path.join( self.tool_data_path, 'shared', 'ucsc', 'builds.txt') ), self.root ) self.len_file_path = resolve_path( kwargs.get( "len_file_path", os.path.join( self.tool_data_path, 'shared', 'ucsc', 'chrom') ), self.root ) # The value of migrated_tools_config is the file reserved for containing only those tools that have been eliminated from the distribution @@ -585,18 +591,18 @@ class Configuration( object ): """ defaults = dict( auth_config_file=[ self.__in_config_dir( 'auth_conf.xml' ) ], - data_manager_config_file=[ 'config/data_manager_conf.xml', 'data_manager_conf.xml', 'config/data_manager_conf.xml.sample' ], + data_manager_config_file=[ self.__in_config_dir( 'data_manager_conf.xml' ) ], datatypes_config_file=[ self.__in_config_dir( 'datatypes_conf.xml' ), self.__in_sample_dir( 'datatypes_conf.xml.sample' ) ], - external_service_type_config_file=[ 'config/external_service_types_conf.xml', 'external_service_types_conf.xml', 'config/external_service_types_conf.xml.sample' ], + external_service_type_config_file=[ self.__in_config_dir( 'external_service_types_conf.xml' ) ], job_config_file=[ 'config/job_conf.xml', 'job_conf.xml' ], job_metrics_config_file=[ 'config/job_metrics_conf.xml', 'job_metrics_conf.xml', 'config/job_metrics_conf.xml.sample' ], dependency_resolvers_config_file=[ 'config/dependency_resolvers_conf.xml', 'dependency_resolvers_conf.xml' ], job_resource_params_file=[ 'config/job_resource_params_conf.xml', 'job_resource_params_conf.xml' ], object_store_config_file=[ 'config/object_store_conf.xml', 'object_store_conf.xml' ], openid_config_file=[ 'config/openid_conf.xml', 'openid_conf.xml', 'config/openid_conf.xml.sample' ], - shed_data_manager_config_file=[ 'shed_data_manager_conf.xml', 'config/shed_data_manager_conf.xml' ], - shed_tool_data_table_config=[ 'shed_tool_data_table_conf.xml', 'config/shed_tool_data_table_conf.xml' ], - workflow_schedulers_config_file=['config/workflow_schedulers_conf.xml', 'config/workflow_schedulers_conf.xml.sample'], + shed_data_manager_config_file=[ self.__in_mutable_config_dir( 'shed_data_manager_conf.xml' ) ], + shed_tool_data_table_config=[ self.__in_mutable_config_dir( 'shed_tool_data_table_conf.xml' ) ], + workflow_schedulers_config_file=[ self.__in_config_dir( 'config/workflow_schedulers_conf.xml' ) ], ) listify_defaults = dict( @@ -617,6 +623,14 @@ class Configuration( object ): 'config/tool_conf.xml.sample,config/shed_tool_conf.xml' ] ) + if not self.running_from_source: + listify_defaults['tool_data_table_config_path'] = [ + self.__in_config_dir( 'tool_data_table_conf.xml' ), + self.__in_sample_dir( 'tool_data_table_conf.xml.sample' ) ] + listify_defaults['tool_config_file'] = [ + self.__in_config_dir( 'tool_conf.xml' ), + self.__in_sample_dir( 'tool_conf.xml.sample' ) ] + for var, defaults in defaults.items(): if kwargs.get( var, None ) is not None: path = kwargs.get( var ) @@ -647,6 +661,12 @@ class Configuration( object ): paths = listify( defaults[-1] ) setattr( self, var, [ resolve_path( x, self.root ) for x in paths ] ) + # If the user has configured a shed tool config in tool_config_file + # this would add a second, but since we're not parsing them yet we + # don't know if that's the case. + if not self.running_from_source and self.shed_tool_conf not in self.tool_config_file: + self.tool_config_file.append( self.shed_tool_conf ) + # Backwards compatibility for names used in too many places to fix self.datatypes_config = self.datatypes_config_file self.tool_configs = self.tool_config_file @@ -730,7 +750,7 @@ class Configuration( object ): if self.migrated_tools_config not in tool_configs and os.path.exists( self.migrated_tools_config ): tool_configs.append( self.migrated_tools_config ) for path in tool_configs: - if not os.path.exists( path ): + if not os.path.exists( path ) and path != self.shed_tool_conf: raise ConfigurationError("Tool config file not found: %s" % path ) for datatypes_config in listify( self.datatypes_config ): if not os.path.isfile( datatypes_config ): @@ -915,9 +935,14 @@ class ConfiguresGalaxyMixin: self.tool_data_tables = ToolDataTableManager( tool_data_path=self.config.tool_data_path, config_filename=self.config.tool_data_table_config_path ) # Load additional entries defined by self.config.shed_tool_data_table_config into tool data tables. - self.tool_data_tables.load_from_config_file( config_filename=self.config.shed_tool_data_table_config, - tool_data_path=self.tool_data_tables.tool_data_path, - from_shed_config=from_shed_config ) + try: + self.tool_data_tables.load_from_config_file( config_filename=self.config.shed_tool_data_table_config, + tool_data_path=self.tool_data_tables.tool_data_path, + from_shed_config=from_shed_config ) + except (OSError, IOError) as exc: + # Missing shed_tool_data_table_config is okay if it's the default + if exc.errno != errno.ENOENT or self.config.shed_tool_data_table_config_set: + raise def _configure_datatypes_registry( self, installed_repository_manager=None ): from galaxy.datatypes import registry diff --git a/config/external_service_types_conf.xml.sample b/lib/galaxy/config/sample/external_service_types_conf.xml.sample similarity index 100% rename from config/external_service_types_conf.xml.sample rename to lib/galaxy/config/sample/external_service_types_conf.xml.sample diff --git a/config/tool_data_table_conf.xml.sample b/lib/galaxy/config/sample/tool_data_table_conf.xml.sample similarity index 75% rename from config/tool_data_table_conf.xml.sample rename to lib/galaxy/config/sample/tool_data_table_conf.xml.sample index 62917970e41..a25afdf41c3 100644 --- a/config/tool_data_table_conf.xml.sample +++ b/lib/galaxy/config/sample/tool_data_table_conf.xml.sample @@ -1,69 +1,69 @@ - + value, dbkey, name, path - +
value, dbkey, formats, name, path - +
value, name, path - +
value, dbkey, name, path - +
value, dbkey, name, path - +
name, value, dbkey, species - +
value, dbkey, name, path - +
value, name, path - +
value, name, path - +
value, dbkey, name, path - +
value, dbkey, name, path - +
value, dbkey, name, path - +
value, path - +
@@ -73,21 +73,21 @@
dbkey, name, value - +
value, name, url - +
value, name, url - +
value, name, url - +
diff --git a/config/workflow_schedulers_conf.xml.sample b/lib/galaxy/config/sample/workflow_schedulers_conf.xml.sample similarity index 100% rename from config/workflow_schedulers_conf.xml.sample rename to lib/galaxy/config/sample/workflow_schedulers_conf.xml.sample diff --git a/lib/galaxy/sample_tracking/external_service_types.py b/lib/galaxy/sample_tracking/external_service_types.py index 1e52efc20d0..956b4b7418b 100644 --- a/lib/galaxy/sample_tracking/external_service_types.py +++ b/lib/galaxy/sample_tracking/external_service_types.py @@ -1,10 +1,14 @@ +import errno import os import logging + from galaxy.util.odict import odict from galaxy import util, model from galaxy.forms.forms import form_factory from galaxy.external_services.service import ExternalServiceActionsGroup from galaxy.sample_tracking.data_transfer import data_transfer_factories + + log = logging.getLogger( __name__ ) @@ -25,8 +29,13 @@ class ExternalServiceTypesCollection( object ): def load_all( self, config_filename ): self.visible_external_service_types = [] - tree = util.parse_xml( config_filename ) - root = tree.getroot() + try: + tree = util.parse_xml( config_filename ) + root = tree.getroot() + except (OSError, IOError) as exc: + if exc.errno != errno.ENOENT or self.app.config.external_service_type_config_file_set: + raise + root = [] for elem in root: try: if elem.tag == 'external_service_type': diff --git a/lib/galaxy/tools/data/__init__.py b/lib/galaxy/tools/data/__init__.py index cd287a84a32..661a0c72521 100644 --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -6,6 +6,7 @@ users to configure data tables for a local Galaxy instance without needing to modify the tool configurations. """ +import errno import logging import os import os.path @@ -26,11 +27,16 @@ log = logging.getLogger( __name__ ) DEFAULT_TABLE_TYPE = 'tabular' +TOOL_DATA_TABLE_CONF_XML = """ + + +""" + class ToolDataTableManager( object ): """Manages a collection of tool data tables""" - def __init__( self, tool_data_path, config_filename=None ): + def __init__( self, tool_data_path, config_filename=None, tool_data_table_config_path_set=None ): self.tool_data_path = tool_data_path # This stores all defined data table entries from both the tool_data_table_conf.xml file and the shed_tool_data_table_conf.xml file # at server startup. If tool shed repositories are installed that contain a valid file named tool_data_table_conf.xml.sample, entries @@ -141,12 +147,20 @@ class ToolDataTableManager( object ): full_path = os.path.abspath( shed_tool_data_table_config ) # FIXME: we should lock changing this file by other threads / head nodes try: - tree = util.parse_xml( full_path ) + try: + tree = util.parse_xml( full_path ) + except (OSError, IOError) as exc: + if exc.errno == errno.ENOENT: + with open( full_path, 'w' ) as fh: + fh.write( TOOL_DATA_TABLE_CONF_XML ) + tree = util.parse_xml( full_path ) + else: + raise root = tree.getroot() out_elems = [ elem for elem in root ] - except Exception as e: + except Exception as exc: out_elems = [] - log.debug( 'Could not parse existing tool data table config, assume no existing elements: %s', e ) + log.debug( 'Could not parse existing tool data table config, assume no existing elements: %s', exc ) for elem in remove_elems: # handle multiple occurrences of remove elem in existing elems while elem in out_elems: @@ -340,6 +354,12 @@ class TabularToolDataTable( ToolDataTable, Dictifiable ): self._update_version() else: self.missing_index_file = filename + # TODO: some data tables need to exist (even if they are empty) + # for tools to load. In an installed Galaxy environment and the + # default tool_data_table_conf.xml, this will emit spurious + # warnings about missing location files that would otherwise be + # empty and we don't care about unless the admin chooses to + # populate them. log.warning( "Cannot find index file '%s' for tool data table '%s'" % ( filename, self.name ) ) if filename not in self.filenames or not self.filenames[ filename ][ 'found' ]: diff --git a/lib/galaxy/tools/data_manager/manager.py b/lib/galaxy/tools/data_manager/manager.py index 086065beca0..354de5b5b99 100644 --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -37,6 +37,10 @@ class DataManagers( object ): def load_from_xml( self, xml_filename, store_tool_path=True, replace_existing=False ): try: tree = util.parse_xml( xml_filename ) + except (IOError, OSError) as e: + if e.errno != errno.ENOENT or self.app.config.data_manager_config_file_set: + raise + return # default config option and it doesn't exist, which is fine except Exception as e: log.error( 'There was an error parsing your Data Manager config file "%s": %s' % ( xml_filename, e ) ) return # we are not able to load any data managers diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index 2a2b998b253..659b2f19e2a 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -1,3 +1,4 @@ +import errno import logging import os import string @@ -39,6 +40,11 @@ from galaxy.tools.deps import build_dependency_manager log = logging.getLogger( __name__ ) +SHED_TOOL_CONF_XML = """ + + +""" + class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): """ @@ -124,7 +130,21 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): """ log.info( "Parsing the tool configuration %s" % config_filename ) - tool_conf_source = get_toolbox_parser(config_filename) + try: + tool_conf_source = get_toolbox_parser(config_filename) + except (OSError, IOError) as exc: + for opt in ('shed_tool_conf', 'migrated_tools_config'): + if (config_filename == getattr( self.app.config, opt ) + and not getattr( self.app.config, opt + '_set' ) + and exc.errno == errno.ENOENT): + log.debug( "Skipping loading missing default config file: %s", config_filename ) + stcd = dict( config_filename=config_filename, + tool_path=self.app.config.shed_tools_dir, + config_elems=[], + create=SHED_TOOL_CONF_XML.format(shed_tools_dir=self.app.config.shed_tools_dir) ) + self._dynamic_tool_confs.append( stcd ) + return + raise tool_path = tool_conf_source.parse_tool_path() parsing_shed_tool_conf = tool_conf_source.is_shed_tool_conf() if parsing_shed_tool_conf: diff --git a/lib/galaxy/workflow/scheduling_manager.py b/lib/galaxy/workflow/scheduling_manager.py index 222b878c22f..71ec00df326 100644 --- a/lib/galaxy/workflow/scheduling_manager.py +++ b/lib/galaxy/workflow/scheduling_manager.py @@ -93,8 +93,8 @@ class WorkflowSchedulingManager( object ): def __init_schedulers( self ): config_file = self.app.config.workflow_schedulers_config_file use_default_scheduler = False - if not config_file: - log.info( "Not workflow schedulers plugin config file defined, using default scheduler." ) + if not config_file or (not os.path.exists( config_file ) and not self.app.config.workflow_schedulers_config_file_set): + log.info( "No workflow schedulers plugin config file defined, using default scheduler." ) use_default_scheduler = True elif not os.path.exists( config_file ): log.info( "Cannot find workflow schedulers plugin config file '%s', using default scheduler." % config_file ) diff --git a/lib/tool_shed/galaxy_install/tools/data_manager.py b/lib/tool_shed/galaxy_install/tools/data_manager.py index 7acf0bec5ca..65a685bba57 100644 --- a/lib/tool_shed/galaxy_install/tools/data_manager.py +++ b/lib/tool_shed/galaxy_install/tools/data_manager.py @@ -1,3 +1,4 @@ +import errno import logging import os import threading @@ -7,6 +8,11 @@ from tool_shed.util import xml_util log = logging.getLogger( __name__ ) +SHED_DATA_MANAGER_CONF_XML = """ + + +""" + class DataManagerHandler( object ): @@ -41,7 +47,15 @@ class DataManagerHandler( object ): for tool_tup in repository_tools_tups: repository_tools_by_guid[ tool_tup[ 1 ] ] = dict( tool_config_filename=tool_tup[ 0 ], tool=tool_tup[ 2 ] ) # Load existing data managers. - tree, error_message = xml_util.parse_xml( shed_data_manager_conf_filename ) + try: + tree, error_message = xml_util.parse_xml( shed_data_manager_conf_filename ) + except (OSError, IOError) as exc: + if exc.errno == errno.ENOENT: + with open( shed_data_manager_conf_filename, 'w' ) as fh: + fh.write( SHED_DATA_MANAGER_CONF_XML ) + tree, error_message = xml_util.parse_xml( shed_data_manager_conf_filename ) + else: + raise if tree is None: return rval config_elems = [ elem for elem in tree.getroot() ] diff --git a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py index 204ed0f2749..ac197e5ca2c 100644 --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -1,3 +1,4 @@ +import errno import logging import threading @@ -26,7 +27,15 @@ class ToolPanelManager( object ): shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ] tool_path = shed_tool_conf_dict[ 'tool_path' ] config_elems = [] - tree, error_message = xml_util.parse_xml( shed_tool_conf ) + try: + tree, error_message = xml_util.parse_xml( shed_tool_conf ) + except (OSError, IOError) as exc: + if (exc.errno == errno.ENOENT and shed_tool_conf_dict.get( 'create', None ) is not None ): + with open( shed_tool_conf, 'w' ) as fh: + fh.write( shed_tool_conf_dict['create'] ) + tree, error_message = xml_util.parse_xml( shed_tool_conf ) + else: + raise if tree: root = tree.getroot() for elem in root: diff --git a/lib/tool_shed/util/common_util.py b/lib/tool_shed/util/common_util.py index b213823c51f..7779e7c96b3 100644 --- a/lib/tool_shed/util/common_util.py +++ b/lib/tool_shed/util/common_util.py @@ -144,7 +144,12 @@ def get_non_shed_tool_panel_configs( app ): for config_filename in app.config.tool_configs: # Any config file that includes a tool_path attribute in the root tag set like the following is shed-related. # - tree, error_message = xml_util.parse_xml( config_filename ) + try: + tree, error_message = xml_util.parse_xml( config_filename ) + except (OSError, IOError) as exc: + if config_filename == app.config.shed_tool_conf and not app.config.shed_tool_conf_set: + continue + raise if tree is None: continue root = tree.getroot()