diff --git a/config/data_manager_conf.xml.sample b/config/data_manager_conf.xml.sample
deleted file mode 100644
index 04b4637c390..00000000000
--- a/config/data_manager_conf.xml.sample
+++ /dev/null
@@ -1,3 +0,0 @@
-
-
-
diff --git a/config/shed_data_manager_conf.xml.sample b/config/shed_data_manager_conf.xml.sample
deleted file mode 100644
index 04b4637c390..00000000000
--- a/config/shed_data_manager_conf.xml.sample
+++ /dev/null
@@ -1,3 +0,0 @@
-
-
-
diff --git a/config/shed_tool_data_table_conf.xml.sample b/config/shed_tool_data_table_conf.xml.sample
deleted file mode 100644
index fd376e4ed3b..00000000000
--- a/config/shed_tool_data_table_conf.xml.sample
+++ /dev/null
@@ -1,3 +0,0 @@
-
-
-
diff --git a/lib/galaxy/auth/__init__.py b/lib/galaxy/auth/__init__.py
index 80a802377d4..822ab1ad505 100644
--- a/lib/galaxy/auth/__init__.py
+++ b/lib/galaxy/auth/__init__.py
@@ -2,6 +2,7 @@
Contains implementations of the authentication logic.
"""
+import errno
import logging
import xml.etree.ElementTree
from collections import namedtuple
@@ -34,8 +35,8 @@ class AuthManager(object):
try:
ct = xml.etree.ElementTree.parse(auth_config_file)
conf_root = ct.getroot()
- except (OSError, IOError):
- if not app.config.auth_config_file_set:
+ except (OSError, IOError) as exc:
+ if exc.errno == errno.ENOENT and not app.config.auth_config_file_set:
conf_root = xml.etree.ElementTree.fromstring(AUTH_CONF_XML)
else:
raise
diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py
index 2d20b226b05..8de20500e2e 100644
--- a/lib/galaxy/config/__init__.py
+++ b/lib/galaxy/config/__init__.py
@@ -4,6 +4,7 @@ Universe configuration builder.
# absolute_import needed for tool_shed package.
from __future__ import absolute_import
+import errno
import logging
import logging.config
import os
@@ -97,6 +98,9 @@ class Configuration( object ):
# Configs no longer read from samples
self.migrated_tools_config = resolve_path( kwargs.get( 'migrated_tools_conf', 'migrated_tools_conf.xml' ), self.mutable_config_dir )
+ self.shed_tool_conf = resolve_path( kwargs.get( 'shed_tool_conf', 'shed_tool_conf.xml' ), self.mutable_config_dir )
+ for name in ( 'migrated_tools_config', 'shed_tool_conf' ):
+ setattr( self, name + '_set', kwargs.get( name, None ) is not None )
# Resolve paths of other config files
self.__parse_config_file_options( kwargs )
@@ -134,6 +138,8 @@ class Configuration( object ):
self.enable_unique_workflow_defaults = string_as_bool( kwargs.get( 'enable_unique_workflow_defaults', False ) )
self.tool_path = resolve_path( kwargs.get( "tool_path", "tools" ), self.root )
self.tool_data_path = resolve_path( kwargs.get( "tool_data_path", "tool-data" ), os.getcwd() )
+ if not self.running_from_source and kwargs.get( "tool_data_path", None ) is None:
+ self.tool_data_path = resolve_path( "tool-data", self.data_dir )
self.builds_file_path = resolve_path( kwargs.get( "builds_file_path", os.path.join( self.tool_data_path, 'shared', 'ucsc', 'builds.txt') ), self.root )
self.len_file_path = resolve_path( kwargs.get( "len_file_path", os.path.join( self.tool_data_path, 'shared', 'ucsc', 'chrom') ), self.root )
# The value of migrated_tools_config is the file reserved for containing only those tools that have been eliminated from the distribution
@@ -585,18 +591,18 @@ class Configuration( object ):
"""
defaults = dict(
auth_config_file=[ self.__in_config_dir( 'auth_conf.xml' ) ],
- data_manager_config_file=[ 'config/data_manager_conf.xml', 'data_manager_conf.xml', 'config/data_manager_conf.xml.sample' ],
+ data_manager_config_file=[ self.__in_config_dir( 'data_manager_conf.xml' ) ],
datatypes_config_file=[ self.__in_config_dir( 'datatypes_conf.xml' ), self.__in_sample_dir( 'datatypes_conf.xml.sample' ) ],
- external_service_type_config_file=[ 'config/external_service_types_conf.xml', 'external_service_types_conf.xml', 'config/external_service_types_conf.xml.sample' ],
+ external_service_type_config_file=[ self.__in_config_dir( 'external_service_types_conf.xml' ) ],
job_config_file=[ 'config/job_conf.xml', 'job_conf.xml' ],
job_metrics_config_file=[ 'config/job_metrics_conf.xml', 'job_metrics_conf.xml', 'config/job_metrics_conf.xml.sample' ],
dependency_resolvers_config_file=[ 'config/dependency_resolvers_conf.xml', 'dependency_resolvers_conf.xml' ],
job_resource_params_file=[ 'config/job_resource_params_conf.xml', 'job_resource_params_conf.xml' ],
object_store_config_file=[ 'config/object_store_conf.xml', 'object_store_conf.xml' ],
openid_config_file=[ 'config/openid_conf.xml', 'openid_conf.xml', 'config/openid_conf.xml.sample' ],
- shed_data_manager_config_file=[ 'shed_data_manager_conf.xml', 'config/shed_data_manager_conf.xml' ],
- shed_tool_data_table_config=[ 'shed_tool_data_table_conf.xml', 'config/shed_tool_data_table_conf.xml' ],
- workflow_schedulers_config_file=['config/workflow_schedulers_conf.xml', 'config/workflow_schedulers_conf.xml.sample'],
+ shed_data_manager_config_file=[ self.__in_mutable_config_dir( 'shed_data_manager_conf.xml' ) ],
+ shed_tool_data_table_config=[ self.__in_mutable_config_dir( 'shed_tool_data_table_conf.xml' ) ],
+ workflow_schedulers_config_file=[ self.__in_config_dir( 'config/workflow_schedulers_conf.xml' ) ],
)
listify_defaults = dict(
@@ -617,6 +623,14 @@ class Configuration( object ):
'config/tool_conf.xml.sample,config/shed_tool_conf.xml' ]
)
+ if not self.running_from_source:
+ listify_defaults['tool_data_table_config_path'] = [
+ self.__in_config_dir( 'tool_data_table_conf.xml' ),
+ self.__in_sample_dir( 'tool_data_table_conf.xml.sample' ) ]
+ listify_defaults['tool_config_file'] = [
+ self.__in_config_dir( 'tool_conf.xml' ),
+ self.__in_sample_dir( 'tool_conf.xml.sample' ) ]
+
for var, defaults in defaults.items():
if kwargs.get( var, None ) is not None:
path = kwargs.get( var )
@@ -647,6 +661,12 @@ class Configuration( object ):
paths = listify( defaults[-1] )
setattr( self, var, [ resolve_path( x, self.root ) for x in paths ] )
+ # If the user has configured a shed tool config in tool_config_file
+ # this would add a second, but since we're not parsing them yet we
+ # don't know if that's the case.
+ if not self.running_from_source and self.shed_tool_conf not in self.tool_config_file:
+ self.tool_config_file.append( self.shed_tool_conf )
+
# Backwards compatibility for names used in too many places to fix
self.datatypes_config = self.datatypes_config_file
self.tool_configs = self.tool_config_file
@@ -730,7 +750,7 @@ class Configuration( object ):
if self.migrated_tools_config not in tool_configs and os.path.exists( self.migrated_tools_config ):
tool_configs.append( self.migrated_tools_config )
for path in tool_configs:
- if not os.path.exists( path ):
+ if not os.path.exists( path ) and path != self.shed_tool_conf:
raise ConfigurationError("Tool config file not found: %s" % path )
for datatypes_config in listify( self.datatypes_config ):
if not os.path.isfile( datatypes_config ):
@@ -915,9 +935,14 @@ class ConfiguresGalaxyMixin:
self.tool_data_tables = ToolDataTableManager( tool_data_path=self.config.tool_data_path,
config_filename=self.config.tool_data_table_config_path )
# Load additional entries defined by self.config.shed_tool_data_table_config into tool data tables.
- self.tool_data_tables.load_from_config_file( config_filename=self.config.shed_tool_data_table_config,
- tool_data_path=self.tool_data_tables.tool_data_path,
- from_shed_config=from_shed_config )
+ try:
+ self.tool_data_tables.load_from_config_file( config_filename=self.config.shed_tool_data_table_config,
+ tool_data_path=self.tool_data_tables.tool_data_path,
+ from_shed_config=from_shed_config )
+ except (OSError, IOError) as exc:
+ # Missing shed_tool_data_table_config is okay if it's the default
+ if exc.errno != errno.ENOENT or self.config.shed_tool_data_table_config_set:
+ raise
def _configure_datatypes_registry( self, installed_repository_manager=None ):
from galaxy.datatypes import registry
diff --git a/config/external_service_types_conf.xml.sample b/lib/galaxy/config/sample/external_service_types_conf.xml.sample
similarity index 100%
rename from config/external_service_types_conf.xml.sample
rename to lib/galaxy/config/sample/external_service_types_conf.xml.sample
diff --git a/config/tool_data_table_conf.xml.sample b/lib/galaxy/config/sample/tool_data_table_conf.xml.sample
similarity index 75%
rename from config/tool_data_table_conf.xml.sample
rename to lib/galaxy/config/sample/tool_data_table_conf.xml.sample
index 62917970e41..a25afdf41c3 100644
--- a/config/tool_data_table_conf.xml.sample
+++ b/lib/galaxy/config/sample/tool_data_table_conf.xml.sample
@@ -1,69 +1,69 @@
-
+
value, dbkey, name, path
-
+
value, dbkey, formats, name, path
-
+
value, dbkey, name, path
-
+
value, dbkey, name, path
-
+
name, value, dbkey, species
-
+
value, dbkey, name, path
-
+
value, dbkey, name, path
-
+
value, dbkey, name, path
-
+
value, dbkey, name, path
-
+
@@ -73,21 +73,21 @@
diff --git a/config/workflow_schedulers_conf.xml.sample b/lib/galaxy/config/sample/workflow_schedulers_conf.xml.sample
similarity index 100%
rename from config/workflow_schedulers_conf.xml.sample
rename to lib/galaxy/config/sample/workflow_schedulers_conf.xml.sample
diff --git a/lib/galaxy/sample_tracking/external_service_types.py b/lib/galaxy/sample_tracking/external_service_types.py
index 1e52efc20d0..956b4b7418b 100644
--- a/lib/galaxy/sample_tracking/external_service_types.py
+++ b/lib/galaxy/sample_tracking/external_service_types.py
@@ -1,10 +1,14 @@
+import errno
import os
import logging
+
from galaxy.util.odict import odict
from galaxy import util, model
from galaxy.forms.forms import form_factory
from galaxy.external_services.service import ExternalServiceActionsGroup
from galaxy.sample_tracking.data_transfer import data_transfer_factories
+
+
log = logging.getLogger( __name__ )
@@ -25,8 +29,13 @@ class ExternalServiceTypesCollection( object ):
def load_all( self, config_filename ):
self.visible_external_service_types = []
- tree = util.parse_xml( config_filename )
- root = tree.getroot()
+ try:
+ tree = util.parse_xml( config_filename )
+ root = tree.getroot()
+ except (OSError, IOError) as exc:
+ if exc.errno != errno.ENOENT or self.app.config.external_service_type_config_file_set:
+ raise
+ root = []
for elem in root:
try:
if elem.tag == 'external_service_type':
diff --git a/lib/galaxy/tools/data/__init__.py b/lib/galaxy/tools/data/__init__.py
index cd287a84a32..661a0c72521 100644
--- a/lib/galaxy/tools/data/__init__.py
+++ b/lib/galaxy/tools/data/__init__.py
@@ -6,6 +6,7 @@ users to configure data tables for a local Galaxy instance without needing
to modify the tool configurations.
"""
+import errno
import logging
import os
import os.path
@@ -26,11 +27,16 @@ log = logging.getLogger( __name__ )
DEFAULT_TABLE_TYPE = 'tabular'
+TOOL_DATA_TABLE_CONF_XML = """
+
+
+"""
+
class ToolDataTableManager( object ):
"""Manages a collection of tool data tables"""
- def __init__( self, tool_data_path, config_filename=None ):
+ def __init__( self, tool_data_path, config_filename=None, tool_data_table_config_path_set=None ):
self.tool_data_path = tool_data_path
# This stores all defined data table entries from both the tool_data_table_conf.xml file and the shed_tool_data_table_conf.xml file
# at server startup. If tool shed repositories are installed that contain a valid file named tool_data_table_conf.xml.sample, entries
@@ -141,12 +147,20 @@ class ToolDataTableManager( object ):
full_path = os.path.abspath( shed_tool_data_table_config )
# FIXME: we should lock changing this file by other threads / head nodes
try:
- tree = util.parse_xml( full_path )
+ try:
+ tree = util.parse_xml( full_path )
+ except (OSError, IOError) as exc:
+ if exc.errno == errno.ENOENT:
+ with open( full_path, 'w' ) as fh:
+ fh.write( TOOL_DATA_TABLE_CONF_XML )
+ tree = util.parse_xml( full_path )
+ else:
+ raise
root = tree.getroot()
out_elems = [ elem for elem in root ]
- except Exception as e:
+ except Exception as exc:
out_elems = []
- log.debug( 'Could not parse existing tool data table config, assume no existing elements: %s', e )
+ log.debug( 'Could not parse existing tool data table config, assume no existing elements: %s', exc )
for elem in remove_elems:
# handle multiple occurrences of remove elem in existing elems
while elem in out_elems:
@@ -340,6 +354,12 @@ class TabularToolDataTable( ToolDataTable, Dictifiable ):
self._update_version()
else:
self.missing_index_file = filename
+ # TODO: some data tables need to exist (even if they are empty)
+ # for tools to load. In an installed Galaxy environment and the
+ # default tool_data_table_conf.xml, this will emit spurious
+ # warnings about missing location files that would otherwise be
+ # empty and we don't care about unless the admin chooses to
+ # populate them.
log.warning( "Cannot find index file '%s' for tool data table '%s'" % ( filename, self.name ) )
if filename not in self.filenames or not self.filenames[ filename ][ 'found' ]:
diff --git a/lib/galaxy/tools/data_manager/manager.py b/lib/galaxy/tools/data_manager/manager.py
index 086065beca0..354de5b5b99 100644
--- a/lib/galaxy/tools/data_manager/manager.py
+++ b/lib/galaxy/tools/data_manager/manager.py
@@ -37,6 +37,10 @@ class DataManagers( object ):
def load_from_xml( self, xml_filename, store_tool_path=True, replace_existing=False ):
try:
tree = util.parse_xml( xml_filename )
+ except (IOError, OSError) as e:
+ if e.errno != errno.ENOENT or self.app.config.data_manager_config_file_set:
+ raise
+ return # default config option and it doesn't exist, which is fine
except Exception as e:
log.error( 'There was an error parsing your Data Manager config file "%s": %s' % ( xml_filename, e ) )
return # we are not able to load any data managers
diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py
index 2a2b998b253..659b2f19e2a 100644
--- a/lib/galaxy/tools/toolbox/base.py
+++ b/lib/galaxy/tools/toolbox/base.py
@@ -1,3 +1,4 @@
+import errno
import logging
import os
import string
@@ -39,6 +40,11 @@ from galaxy.tools.deps import build_dependency_manager
log = logging.getLogger( __name__ )
+SHED_TOOL_CONF_XML = """
+
+
+"""
+
class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
"""
@@ -124,7 +130,21 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
"""
log.info( "Parsing the tool configuration %s" % config_filename )
- tool_conf_source = get_toolbox_parser(config_filename)
+ try:
+ tool_conf_source = get_toolbox_parser(config_filename)
+ except (OSError, IOError) as exc:
+ for opt in ('shed_tool_conf', 'migrated_tools_config'):
+ if (config_filename == getattr( self.app.config, opt )
+ and not getattr( self.app.config, opt + '_set' )
+ and exc.errno == errno.ENOENT):
+ log.debug( "Skipping loading missing default config file: %s", config_filename )
+ stcd = dict( config_filename=config_filename,
+ tool_path=self.app.config.shed_tools_dir,
+ config_elems=[],
+ create=SHED_TOOL_CONF_XML.format(shed_tools_dir=self.app.config.shed_tools_dir) )
+ self._dynamic_tool_confs.append( stcd )
+ return
+ raise
tool_path = tool_conf_source.parse_tool_path()
parsing_shed_tool_conf = tool_conf_source.is_shed_tool_conf()
if parsing_shed_tool_conf:
diff --git a/lib/galaxy/workflow/scheduling_manager.py b/lib/galaxy/workflow/scheduling_manager.py
index 222b878c22f..71ec00df326 100644
--- a/lib/galaxy/workflow/scheduling_manager.py
+++ b/lib/galaxy/workflow/scheduling_manager.py
@@ -93,8 +93,8 @@ class WorkflowSchedulingManager( object ):
def __init_schedulers( self ):
config_file = self.app.config.workflow_schedulers_config_file
use_default_scheduler = False
- if not config_file:
- log.info( "Not workflow schedulers plugin config file defined, using default scheduler." )
+ if not config_file or (not os.path.exists( config_file ) and not self.app.config.workflow_schedulers_config_file_set):
+ log.info( "No workflow schedulers plugin config file defined, using default scheduler." )
use_default_scheduler = True
elif not os.path.exists( config_file ):
log.info( "Cannot find workflow schedulers plugin config file '%s', using default scheduler." % config_file )
diff --git a/lib/tool_shed/galaxy_install/tools/data_manager.py b/lib/tool_shed/galaxy_install/tools/data_manager.py
index 7acf0bec5ca..65a685bba57 100644
--- a/lib/tool_shed/galaxy_install/tools/data_manager.py
+++ b/lib/tool_shed/galaxy_install/tools/data_manager.py
@@ -1,3 +1,4 @@
+import errno
import logging
import os
import threading
@@ -7,6 +8,11 @@ from tool_shed.util import xml_util
log = logging.getLogger( __name__ )
+SHED_DATA_MANAGER_CONF_XML = """
+
+
+"""
+
class DataManagerHandler( object ):
@@ -41,7 +47,15 @@ class DataManagerHandler( object ):
for tool_tup in repository_tools_tups:
repository_tools_by_guid[ tool_tup[ 1 ] ] = dict( tool_config_filename=tool_tup[ 0 ], tool=tool_tup[ 2 ] )
# Load existing data managers.
- tree, error_message = xml_util.parse_xml( shed_data_manager_conf_filename )
+ try:
+ tree, error_message = xml_util.parse_xml( shed_data_manager_conf_filename )
+ except (OSError, IOError) as exc:
+ if exc.errno == errno.ENOENT:
+ with open( shed_data_manager_conf_filename, 'w' ) as fh:
+ fh.write( SHED_DATA_MANAGER_CONF_XML )
+ tree, error_message = xml_util.parse_xml( shed_data_manager_conf_filename )
+ else:
+ raise
if tree is None:
return rval
config_elems = [ elem for elem in tree.getroot() ]
diff --git a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py
index 204ed0f2749..ac197e5ca2c 100644
--- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py
+++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py
@@ -1,3 +1,4 @@
+import errno
import logging
import threading
@@ -26,7 +27,15 @@ class ToolPanelManager( object ):
shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ]
tool_path = shed_tool_conf_dict[ 'tool_path' ]
config_elems = []
- tree, error_message = xml_util.parse_xml( shed_tool_conf )
+ try:
+ tree, error_message = xml_util.parse_xml( shed_tool_conf )
+ except (OSError, IOError) as exc:
+ if (exc.errno == errno.ENOENT and shed_tool_conf_dict.get( 'create', None ) is not None ):
+ with open( shed_tool_conf, 'w' ) as fh:
+ fh.write( shed_tool_conf_dict['create'] )
+ tree, error_message = xml_util.parse_xml( shed_tool_conf )
+ else:
+ raise
if tree:
root = tree.getroot()
for elem in root:
diff --git a/lib/tool_shed/util/common_util.py b/lib/tool_shed/util/common_util.py
index b213823c51f..7779e7c96b3 100644
--- a/lib/tool_shed/util/common_util.py
+++ b/lib/tool_shed/util/common_util.py
@@ -144,7 +144,12 @@ def get_non_shed_tool_panel_configs( app ):
for config_filename in app.config.tool_configs:
# Any config file that includes a tool_path attribute in the root tag set like the following is shed-related.
#
- tree, error_message = xml_util.parse_xml( config_filename )
+ try:
+ tree, error_message = xml_util.parse_xml( config_filename )
+ except (OSError, IOError) as exc:
+ if config_filename == app.config.shed_tool_conf and not app.config.shed_tool_conf_set:
+ continue
+ raise
if tree is None:
continue
root = tree.getroot()