mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-01 15:37:32 +08:00
Maintain entries for Galaxy's ToolDataTableManager acquired from installed tool shed repositrories that contain a valid file named tool_data_table_conf.xmnl.sample in a separate config file name shed_tool_data_table_conf.xml. This will ensure that manual edits to the original tool_data_table_conf.xml will not be munged when Galaxy's tool shed installation process automatically adds entries into the file. This enhancement alos includes a bug fix where ToolDataTableEntries that should have been persisted to the XML file were not being handled correctly.
This commit is contained in:
Greg Von Kuster
+7
-2
@@ -76,8 +76,13 @@ class UniverseApplication( object ):
|
||||
self.genomes = Genomes( self )
|
||||
# Data providers registry.
|
||||
self.data_provider_registry = DataProviderRegistry()
|
||||
# Tool data tables
|
||||
self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( self.config.tool_data_path, self.config.tool_data_table_config_path )
|
||||
# Initialize tool data tables using the config defined by self.config.tool_data_table_config_path.
|
||||
self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( tool_data_path=self.config.tool_data_path,
|
||||
config_filename=self.config.tool_data_table_config_path )
|
||||
# Load additional entries defined by self.config.shed_tool_data_table_config into tool data tables.
|
||||
self.tool_data_tables.load_from_config_file( config_filename=self.config.shed_tool_data_table_config,
|
||||
tool_data_path=self.tool_data_tables.tool_data_path,
|
||||
from_shed_config=True )
|
||||
# Initialize the tools, making sure the list of tool configs includes the reserved migrated_tools_conf.xml file.
|
||||
tool_configs = self.config.tool_configs
|
||||
if self.config.migrated_tools_config not in tool_configs:
|
||||
|
||||
@@ -66,6 +66,7 @@ class Configuration( object ):
|
||||
tcf = 'tool_conf.xml'
|
||||
self.tool_configs = [ resolve_path( p, self.root ) for p in listify( tcf ) ]
|
||||
self.tool_data_table_config_path = resolve_path( kwargs.get( 'tool_data_table_config_path', 'tool_data_table_conf.xml' ), self.root )
|
||||
self.shed_tool_data_table_config = resolve_path( kwargs.get( 'shed_tool_data_table_config', 'shed_tool_data_table_conf.xml' ), self.root )
|
||||
self.enable_tool_shed_check = string_as_bool( kwargs.get( 'enable_tool_shed_check', False ) )
|
||||
try:
|
||||
self.hours_between_check = int( kwargs.get( 'hours_between_check', 12 ) )
|
||||
|
||||
@@ -184,7 +184,8 @@ class InstallManager( object ):
|
||||
relative_install_dir=relative_install_dir,
|
||||
repository_files_dir=None,
|
||||
resetting_all_metadata_on_repository=False,
|
||||
updating_installed_repository=False )
|
||||
updating_installed_repository=False,
|
||||
persist=True )
|
||||
tool_shed_repository.metadata = metadata_dict
|
||||
self.app.sa_session.add( tool_shed_repository )
|
||||
self.app.sa_session.flush()
|
||||
|
||||
@@ -41,8 +41,13 @@ class MigrateToolsApplication( object ):
|
||||
self.datatypes_registry = galaxy.datatypes.registry.Registry()
|
||||
# Load the data types in the Galaxy distribution, which are defined in self.config.datatypes_config.
|
||||
self.datatypes_registry.load_datatypes( self.config.root, self.config.datatypes_config )
|
||||
# Tool data tables
|
||||
self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( self.config.tool_data_path, self.config.tool_data_table_config_path )
|
||||
# Initialize tool data tables using the config defined by self.config.tool_data_table_config_path.
|
||||
self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( tool_data_path=self.config.tool_data_path,
|
||||
config_filename=self.config.tool_data_table_config_path )
|
||||
# Load additional entries defined by self.config.shed_tool_data_table_config into tool data tables.
|
||||
self.tool_data_tables.load_from_config_file( config_filename=self.config.shed_tool_data_table_config,
|
||||
tool_data_path=self.tool_data_tables.tool_data_path,
|
||||
from_shed_config=True )
|
||||
# Initialize the tools, making sure the list of tool configs includes the reserved migrated_tools_conf.xml file.
|
||||
tool_configs = self.config.tool_configs
|
||||
if self.config.migrated_tools_config not in tool_configs:
|
||||
|
||||
@@ -15,17 +15,28 @@ class ToolDataTableManager( object ):
|
||||
"""Manages a collection of tool data tables"""
|
||||
def __init__( self, tool_data_path, config_filename=None ):
|
||||
self.tool_data_path = tool_data_path
|
||||
self.data_tables = {}
|
||||
# Store config elements for on-the-fly persistence.
|
||||
self.data_table_elems = []
|
||||
# This stores all defined data table entries from both the tool_data_table_conf.xml file and the shed_tool_data_table_conf.xml file
|
||||
# at server startup. If tool shed repositories are installed that contain a valid file named tool_data_table_conf.xml.sample, entries
|
||||
# from that file are inserted into this dict at the time of installation.
|
||||
self.data_tables = {}
|
||||
# Store config elements for on-the-fly persistence to the defined shed_tool_data_table_config file name.
|
||||
self.shed_data_table_elems = []
|
||||
self.data_table_elem_names = []
|
||||
if config_filename:
|
||||
self.load_from_config_file( config_filename, self.tool_data_path )
|
||||
self.load_from_config_file( config_filename, self.tool_data_path, from_shed_config=False )
|
||||
def __getitem__( self, key ):
|
||||
return self.data_tables.__getitem__( key )
|
||||
def __contains__( self, key ):
|
||||
return self.data_tables.__contains__( key )
|
||||
def load_from_config_file( self, config_filename, tool_data_path ):
|
||||
def load_from_config_file( self, config_filename, tool_data_path, from_shed_config=False ):
|
||||
"""
|
||||
This method is called under 3 conditions:
|
||||
1) When the ToolDataTableManager is initialized (see __init__ above).
|
||||
2) Just after the ToolDataTableManager is initialized and the additional entries defined by shed_tool_data_table_conf.xml
|
||||
are being loaded into the ToolDataTableManager.data_tables.
|
||||
3) When a tool shed repository that includes a tool_data_table_conf.xml.sample file is being installed into a local
|
||||
Galaxy instance. In this case, we have 2 entry types to handle, files whose root tag is <tables>, for example:
|
||||
"""
|
||||
tree = util.parse_xml( config_filename )
|
||||
root = tree.getroot()
|
||||
table_elems = []
|
||||
@@ -36,13 +47,14 @@ class ToolDataTableManager( object ):
|
||||
table_elem_name = table_elem.get( 'name', None )
|
||||
if table_elem_name and table_elem_name not in self.data_table_elem_names:
|
||||
self.data_table_elem_names.append( table_elem_name )
|
||||
self.data_table_elems.append( table_elem )
|
||||
if from_shed_config:
|
||||
self.shed_data_table_elems.append( table_elem )
|
||||
table = tool_data_table_types[ type ]( table_elem, tool_data_path )
|
||||
if table.name not in self.data_tables:
|
||||
self.data_tables[ table.name ] = table
|
||||
log.debug( "Loaded tool data table '%s'", table.name )
|
||||
return table_elems
|
||||
def add_new_entries_from_config_file( self, config_filename, tool_data_path, tool_data_table_config_path, persist=False ):
|
||||
def add_new_entries_from_config_file( self, config_filename, tool_data_path, shed_tool_data_table_config, persist=False ):
|
||||
"""
|
||||
This method is called when a tool shed repository that includes a tool_data_table_conf.xml.sample file is being
|
||||
installed into a local galaxy instance. We have 2 cases to handle, files whose root tag is <tables>, for example:
|
||||
@@ -65,7 +77,9 @@ class ToolDataTableManager( object ):
|
||||
# Make a copy of the current list of data_table_elem_names so we can persist later if changes to the config file are necessary.
|
||||
original_data_table_elem_names = [ name for name in self.data_table_elem_names ]
|
||||
if root.tag == 'tables':
|
||||
table_elems = self.load_from_config_file( config_filename, tool_data_path )
|
||||
table_elems = self.load_from_config_file( config_filename=config_filename,
|
||||
tool_data_path=tool_data_path,
|
||||
from_shed_config=True )
|
||||
else:
|
||||
table_elems = []
|
||||
type = root.get( 'type', 'tabular' )
|
||||
@@ -74,23 +88,22 @@ class ToolDataTableManager( object ):
|
||||
table_elem_name = root.get( 'name', None )
|
||||
if table_elem_name and table_elem_name not in self.data_table_elem_names:
|
||||
self.data_table_elem_names.append( table_elem_name )
|
||||
self.data_table_elems.append( root )
|
||||
self.shed_data_table_elems.append( root )
|
||||
table = tool_data_table_types[ type ]( root, tool_data_path )
|
||||
if table.name not in self.data_tables:
|
||||
self.data_tables[ table.name ] = table
|
||||
log.debug( "Added new tool data table '%s'", table.name )
|
||||
if persist and self.data_table_elem_names != original_data_table_elem_names:
|
||||
# Persist Galaxy's version of the changed tool_data_table_conf.xml file.
|
||||
self.to_xml_file( tool_data_table_config_path )
|
||||
self.to_xml_file( shed_tool_data_table_config )
|
||||
return table_elems
|
||||
def to_xml_file( self, tool_data_table_config_path ):
|
||||
"""Write the current in-memory version of the tool_data-table_conf.xml file to disk."""
|
||||
full_path = os.path.abspath( tool_data_table_config_path )
|
||||
def to_xml_file( self, shed_tool_data_table_config ):
|
||||
"""Write the current in-memory version of the shed_tool_data_table_conf.xml file to disk."""
|
||||
full_path = os.path.abspath( shed_tool_data_table_config )
|
||||
fd, filename = tempfile.mkstemp()
|
||||
os.write( fd, '<?xml version="1.0"?>\n' )
|
||||
os.write( fd, "<!-- Use the file tool_data_table_conf.xml.oldlocstyle if you don't want to update your loc files as changed in revision 4550:535d276c92bc-->\n" )
|
||||
os.write( fd, '<tables>\n' )
|
||||
for elem in self.data_table_elems:
|
||||
for elem in self.shed_data_table_elems:
|
||||
os.write( fd, '%s' % util.xml_to_string( elem ) )
|
||||
os.write( fd, '</tables>\n' )
|
||||
os.close( fd )
|
||||
|
||||
@@ -753,12 +753,15 @@ def generate_environment_dependency_metadata( elem, tool_dependencies_dict ):
|
||||
tool_dependencies_dict[ 'set_environment' ] = [ requirements_dict ]
|
||||
return tool_dependencies_dict
|
||||
def generate_metadata_for_changeset_revision( app, repository, repository_clone_url, shed_config_dict={}, relative_install_dir=None, repository_files_dir=None,
|
||||
resetting_all_metadata_on_repository=False, updating_installed_repository=False ):
|
||||
resetting_all_metadata_on_repository=False, updating_installed_repository=False, persist=False ):
|
||||
"""
|
||||
Generate metadata for a repository using it's files on disk. To generate metadata for changeset revisions older than the repository tip,
|
||||
the repository will have been cloned to a temporary location and updated to a specified changeset revision to access that changeset revision's
|
||||
disk files, so the value of repository_files_dir will not always be repository.repo_path (it could be an absolute path to a temporary directory
|
||||
containing a clone). If it is an absolute path, the value of relative_install_dir must contain repository.repo_path.
|
||||
|
||||
The value of persist will be True when the installed repository contains a valid tool_data_table_conf.xml.sample file, in which case the entries
|
||||
should ultimately be persisted to the file referred to by app.config.shed_tool_data_table_config.
|
||||
"""
|
||||
if updating_installed_repository:
|
||||
# Keep the original tool shed repository metadata if setting metadata on a repository installed into a local Galaxy instance for which
|
||||
@@ -810,9 +813,9 @@ def generate_metadata_for_changeset_revision( app, repository, repository_clone_
|
||||
relative_path, filename = os.path.split( sample_file )
|
||||
if filename == 'tool_data_table_conf.xml.sample':
|
||||
new_table_elems = app.tool_data_tables.add_new_entries_from_config_file( config_filename=sample_file,
|
||||
tool_data_path=app.config.tool_data_path,
|
||||
tool_data_table_config_path=app.config.tool_data_table_config_path,
|
||||
persist=False )
|
||||
tool_data_path=original_tool_data_path,
|
||||
shed_tool_data_table_config=app.config.shed_tool_data_table_config,
|
||||
persist=persist )
|
||||
for root, dirs, files in os.walk( files_dir ):
|
||||
if root.find( '.hg' ) < 0 and root.find( 'hgrc' ) < 0:
|
||||
if '.hg' in dirs:
|
||||
@@ -1768,15 +1771,15 @@ def handle_sample_files_and_load_tool_from_tmp_config( trans, repo, changeset_re
|
||||
return tool, message, sample_files
|
||||
def handle_sample_tool_data_table_conf_file( app, filename, persist=False ):
|
||||
"""
|
||||
Parse the incoming filename and add new entries to the in-memory app.tool_data_tables dictionary. If persist is True (should only occur)
|
||||
if call is from the Galaxy side (not the tool shed), the new entries will be appended to Galaxy's tool_data_table_conf.xml file on disk.
|
||||
Parse the incoming filename and add new entries to the in-memory app.tool_data_tables dictionary. If persist is True (should only occur
|
||||
if call is from the Galaxy side, not the tool shed), the new entries will be appended to Galaxy's shed_tool_data_table_conf.xml file on disk.
|
||||
"""
|
||||
error = False
|
||||
message = ''
|
||||
try:
|
||||
new_table_elems = app.tool_data_tables.add_new_entries_from_config_file( config_filename=filename,
|
||||
tool_data_path=app.config.tool_data_path,
|
||||
tool_data_table_config_path=app.config.tool_data_table_config_path,
|
||||
shed_tool_data_table_config=app.config.shed_tool_data_table_config,
|
||||
persist=persist )
|
||||
except Exception, e:
|
||||
message = str( e )
|
||||
@@ -2143,7 +2146,8 @@ def reset_all_metadata_on_installed_repository( trans, id ):
|
||||
relative_install_dir=relative_install_dir,
|
||||
repository_files_dir=None,
|
||||
resetting_all_metadata_on_repository=False,
|
||||
updating_installed_repository=False )
|
||||
updating_installed_repository=False,
|
||||
persist=False )
|
||||
repository.metadata = metadata_dict
|
||||
if metadata_dict != original_metadata_dict:
|
||||
update_in_shed_tool_config( trans.app, repository )
|
||||
@@ -2221,7 +2225,8 @@ def reset_all_metadata_on_repository_in_tool_shed( trans, id ):
|
||||
relative_install_dir=repo_dir,
|
||||
repository_files_dir=work_dir,
|
||||
resetting_all_metadata_on_repository=True,
|
||||
updating_installed_repository=False )
|
||||
updating_installed_repository=False,
|
||||
persist=False )
|
||||
if current_metadata_dict:
|
||||
if not metadata_changeset_revision and not metadata_dict:
|
||||
# We're at the first change set in the change log.
|
||||
|
||||
@@ -10,9 +10,8 @@ from galaxy.util.shed_util import check_tool_input_params, clone_repository, con
|
||||
from galaxy.util.shed_util import generate_clone_url_for_repository_in_tool_shed, generate_message_for_invalid_tools, generate_metadata_for_changeset_revision
|
||||
from galaxy.util.shed_util import get_changectx_for_changeset, get_config_from_disk, get_configured_ui, get_file_context_from_ctx, get_named_tmpfile_from_ctx
|
||||
from galaxy.util.shed_util import get_parent_id, get_repository_in_tool_shed, get_repository_metadata_by_changeset_revision
|
||||
from galaxy.util.shed_util import handle_sample_files_and_load_tool_from_disk, handle_sample_files_and_load_tool_from_tmp_config
|
||||
from galaxy.util.shed_util import handle_sample_tool_data_table_conf_file, INITIAL_CHANGELOG_HASH, is_downloadable, load_tool_from_config, remove_dir
|
||||
from galaxy.util.shed_util import reset_tool_data_tables, reversed_upper_bounded_changelog, strip_path
|
||||
from galaxy.util.shed_util import handle_sample_files_and_load_tool_from_disk, handle_sample_files_and_load_tool_from_tmp_config, INITIAL_CHANGELOG_HASH
|
||||
from galaxy.util.shed_util import is_downloadable, load_tool_from_config, remove_dir, reset_tool_data_tables, reversed_upper_bounded_changelog, strip_path
|
||||
from galaxy.web.base.controller import *
|
||||
from galaxy.web.base.controllers.admin import *
|
||||
from galaxy.webapps.community import model
|
||||
@@ -640,7 +639,8 @@ def set_repository_metadata( trans, repository, content_alert_str='', **kwd ):
|
||||
relative_install_dir=repo_dir,
|
||||
repository_files_dir=None,
|
||||
resetting_all_metadata_on_repository=False,
|
||||
updating_installed_repository=False )
|
||||
updating_installed_repository=False,
|
||||
persist=False )
|
||||
if metadata_dict:
|
||||
downloadable = is_downloadable( metadata_dict )
|
||||
repository_metadata = None
|
||||
|
||||
@@ -447,6 +447,12 @@ class AdminToolshed( AdminGalaxy ):
|
||||
@web.expose
|
||||
@web.require_admin
|
||||
def deactivate_or_uninstall_repository( self, trans, **kwd ):
|
||||
"""
|
||||
Handle all changes when a tool shed repository is being deactivated or uninstalled. Notice that if the repository contents include
|
||||
a file named tool_data_table_conf.xml.sample, it's entries are not removed from the defined config.shed_tool_data_table_config. This
|
||||
is because it becomes a bit complex to determine if other installed repositories include tools that require the same entry. For now
|
||||
we'll never delete entries from config.shed_tool_data_table_config, but we may choose to do so in the future if it becomes necessary.
|
||||
"""
|
||||
params = util.Params( kwd )
|
||||
message = util.restore_text( params.get( 'message', '' ) )
|
||||
status = params.get( 'status', 'done' )
|
||||
@@ -753,7 +759,8 @@ class AdminToolshed( AdminGalaxy ):
|
||||
relative_install_dir=relative_install_dir,
|
||||
repository_files_dir=None,
|
||||
resetting_all_metadata_on_repository=False,
|
||||
updating_installed_repository=False )
|
||||
updating_installed_repository=False,
|
||||
persist=True )
|
||||
tool_shed_repository.metadata = metadata_dict
|
||||
trans.sa_session.add( tool_shed_repository )
|
||||
trans.sa_session.flush()
|
||||
@@ -1491,7 +1498,8 @@ class AdminToolshed( AdminGalaxy ):
|
||||
relative_install_dir=relative_install_dir,
|
||||
repository_files_dir=None,
|
||||
resetting_all_metadata_on_repository=False,
|
||||
updating_installed_repository=False )
|
||||
updating_installed_repository=False,
|
||||
persist=False )
|
||||
repository.metadata = metadata_dict
|
||||
if metadata_dict != original_metadata_dict:
|
||||
update_in_shed_tool_config( trans.app, repository )
|
||||
@@ -1675,7 +1683,8 @@ class AdminToolshed( AdminGalaxy ):
|
||||
relative_install_dir=relative_install_dir,
|
||||
repository_files_dir=None,
|
||||
resetting_all_metadata_on_repository=False,
|
||||
updating_installed_repository=True )
|
||||
updating_installed_repository=True,
|
||||
persist=True )
|
||||
repository.metadata = metadata_dict
|
||||
# Update the repository changeset_revision in the database.
|
||||
repository.changeset_revision = latest_changeset_revision
|
||||
|
||||
@@ -13,6 +13,7 @@ SAMPLES="
|
||||
reports_wsgi.ini.sample
|
||||
shed_tool_conf.xml.sample
|
||||
tool_conf.xml.sample
|
||||
shed_tool_data_table_conf.xml.sample
|
||||
tool_data_table_conf.xml.sample
|
||||
tool_sheds_conf.xml.sample
|
||||
openid_conf.xml.sample
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
<?xml version="1.0"?>
|
||||
<tables>
|
||||
</tables>
|
||||
@@ -149,6 +149,16 @@ paste.app_factory = galaxy.web.buildapp:app_factory
|
||||
#enable_tool_shed_check = False
|
||||
#hours_between_check = 12
|
||||
|
||||
# XML config file that contains data table entries for the ToolDataTableManager. This file is manually
|
||||
# maintained by the Galaxy administrator.
|
||||
#tool_data_table_config_path = tool_data_table_conf.xml
|
||||
|
||||
# XML config file that contains additional data table entries for the ToolDataTableManager. This file
|
||||
# is automatically generated based on the current installed tool shed repositories that contain valid
|
||||
# tool_data_table_conf.xml.sample files. At the time of installation, these entries are automatically
|
||||
# added to the following file, which is parsed and applied to the ToolDataTableManager at server start up.
|
||||
#shed_tool_data_table_config = shed_tool_data_table_conf.xml
|
||||
|
||||
# Directory where data used by tools is located, see the samples in that
|
||||
# directory and the wiki for help:
|
||||
# http://wiki.g2.bx.psu.edu/Admin/Data%20Integration
|
||||
|
||||
Reference in New Issue
Block a user