mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
Raise an exception if attempting to install a repository into Galaxy that contains an invalid repository dependency definition (implying a bug in the Tool Shed framework).
This commit is contained in:
Greg Von Kuster
@@ -1440,16 +1440,17 @@ class AdminToolshed( AdminGalaxy ):
|
||||
tool_path, relative_install_dir = repository.get_tool_relative_path( trans.app )
|
||||
if relative_install_dir:
|
||||
original_metadata_dict = repository.metadata
|
||||
metadata_dict, invalid_file_tups = metadata_util.generate_metadata_for_changeset_revision( app=trans.app,
|
||||
repository=repository,
|
||||
changeset_revision=repository.changeset_revision,
|
||||
repository_clone_url=repository_clone_url,
|
||||
shed_config_dict = repository.get_shed_config_dict( trans.app ),
|
||||
relative_install_dir=relative_install_dir,
|
||||
repository_files_dir=None,
|
||||
resetting_all_metadata_on_repository=False,
|
||||
updating_installed_repository=False,
|
||||
persist=False )
|
||||
metadata_dict, invalid_file_tups = \
|
||||
metadata_util.generate_metadata_for_changeset_revision( app=trans.app,
|
||||
repository=repository,
|
||||
changeset_revision=repository.changeset_revision,
|
||||
repository_clone_url=repository_clone_url,
|
||||
shed_config_dict = repository.get_shed_config_dict( trans.app ),
|
||||
relative_install_dir=relative_install_dir,
|
||||
repository_files_dir=None,
|
||||
resetting_all_metadata_on_repository=False,
|
||||
updating_installed_repository=False,
|
||||
persist=False )
|
||||
repository.metadata = metadata_dict
|
||||
if metadata_dict != original_metadata_dict:
|
||||
suc.update_in_shed_tool_config( trans.app, repository )
|
||||
@@ -1607,7 +1608,11 @@ class AdminToolshed( AdminGalaxy ):
|
||||
changeset_revision = kwd.get( 'changeset_revision', None )
|
||||
latest_changeset_revision = kwd.get( 'latest_changeset_revision', None )
|
||||
latest_ctx_rev = kwd.get( 'latest_ctx_rev', None )
|
||||
repository = suc.get_tool_shed_repository_by_shed_name_owner_changeset_revision( trans.app, tool_shed_url, name, owner, changeset_revision )
|
||||
repository = suc.get_tool_shed_repository_by_shed_name_owner_changeset_revision( trans.app,
|
||||
tool_shed_url,
|
||||
name,
|
||||
owner,
|
||||
changeset_revision )
|
||||
if changeset_revision and latest_changeset_revision and latest_ctx_rev:
|
||||
if changeset_revision == latest_changeset_revision:
|
||||
message = "The installed repository named '%s' is current, there are no updates available. " % name
|
||||
@@ -1627,16 +1632,17 @@ class AdminToolshed( AdminGalaxy ):
|
||||
if repository.includes_data_managers:
|
||||
data_manager_util.remove_from_data_manager( trans.app, repository )
|
||||
# Update the repository metadata.
|
||||
metadata_dict, invalid_file_tups = metadata_util.generate_metadata_for_changeset_revision( app=trans.app,
|
||||
repository=repository,
|
||||
changeset_revision=latest_changeset_revision,
|
||||
repository_clone_url=repository_clone_url,
|
||||
shed_config_dict = repository.get_shed_config_dict( trans.app ),
|
||||
relative_install_dir=relative_install_dir,
|
||||
repository_files_dir=None,
|
||||
resetting_all_metadata_on_repository=False,
|
||||
updating_installed_repository=True,
|
||||
persist=True )
|
||||
metadata_dict, invalid_file_tups = \
|
||||
metadata_util.generate_metadata_for_changeset_revision( app=trans.app,
|
||||
repository=repository,
|
||||
changeset_revision=latest_changeset_revision,
|
||||
repository_clone_url=repository_clone_url,
|
||||
shed_config_dict=repository.get_shed_config_dict( trans.app ),
|
||||
relative_install_dir=relative_install_dir,
|
||||
repository_files_dir=None,
|
||||
resetting_all_metadata_on_repository=False,
|
||||
updating_installed_repository=True,
|
||||
persist=True )
|
||||
repository.metadata = metadata_dict
|
||||
# Update the repository.changeset_revision column in the database.
|
||||
repository.changeset_revision = latest_changeset_revision
|
||||
@@ -1674,10 +1680,15 @@ class AdminToolshed( AdminGalaxy ):
|
||||
repository_tools_tups )
|
||||
# Create tool_dependency records if necessary.
|
||||
if 'tool_dependencies' in metadata_dict:
|
||||
tool_dependencies = tool_dependency_util.create_tool_dependency_objects( trans.app, repository, relative_install_dir, set_status=False )
|
||||
message = "The installed repository named '%s' has been updated to change set revision '%s'. " % ( name, latest_changeset_revision )
|
||||
tool_dependencies = tool_dependency_util.create_tool_dependency_objects( trans.app,
|
||||
repository,
|
||||
relative_install_dir,
|
||||
set_status=False )
|
||||
message = "The installed repository named '%s' has been updated to change set revision '%s'. " % \
|
||||
( name, latest_changeset_revision )
|
||||
# See if any tool dependencies can be installed.
|
||||
shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository )
|
||||
shed_tool_conf, tool_path, relative_install_dir = \
|
||||
suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository )
|
||||
if repository.missing_tool_dependencies:
|
||||
message += "Click the name of one of the missing tool dependencies listed below to install tool dependencies."
|
||||
else:
|
||||
|
||||
@@ -330,12 +330,13 @@ class InstallManager( object ):
|
||||
tool_panel_elems.append( elem )
|
||||
return tool_panel_elems
|
||||
|
||||
def handle_repository_contents( self, tool_shed_repository, repository_clone_url, relative_install_dir, repository_elem, install_dependencies, is_repository_dependency=False ):
|
||||
def handle_repository_contents( self, tool_shed_repository, repository_clone_url, relative_install_dir, repository_elem,
|
||||
install_dependencies, is_repository_dependency=False ):
|
||||
"""
|
||||
Generate the metadata for the installed tool shed repository, among other things. If the installed tool_shed_repository contains tools
|
||||
that are loaded into the Galaxy tool panel, this method will automatically eliminate all entries for each of the tools defined in the
|
||||
received repository_elem from all non-shed-related tool panel configuration files since the entries are automatically added to the reserved
|
||||
migrated_tools_conf.xml file as part of the migration process.
|
||||
Generate the metadata for the installed tool shed repository, among other things. If the installed tool_shed_repository
|
||||
contains tools that are loaded into the Galaxy tool panel, this method will automatically eliminate all entries for each
|
||||
of the tools defined in the received repository_elem from all non-shed-related tool panel configuration files since the
|
||||
entries are automatically added to the reserved migrated_tools_conf.xml file as part of the migration process.
|
||||
"""
|
||||
tool_configs_to_filter = []
|
||||
tool_panel_dict_for_display = odict()
|
||||
@@ -351,14 +352,16 @@ class InstallManager( object ):
|
||||
# See if tool_config is defined inside of a section in self.proprietary_tool_panel_elems.
|
||||
is_displayed, tool_sections = self.get_containing_tool_sections( tool_config )
|
||||
if is_displayed:
|
||||
tool_panel_dict_for_tool_config = tool_util.generate_tool_panel_dict_for_tool_config( guid, tool_config, tool_sections=tool_sections )
|
||||
tool_panel_dict_for_tool_config = \
|
||||
tool_util.generate_tool_panel_dict_for_tool_config( guid, tool_config, tool_sections=tool_sections )
|
||||
# The tool-panel_dict has the following structure.
|
||||
# {<Tool guid> : [{ tool_config : <tool_config_file>, id: <ToolSection id>, version : <ToolSection version>, name : <TooSection name>}]}
|
||||
# {<Tool guid> : [{ tool_config : <tool_config_file>, id: <ToolSection id>, version : <ToolSection version>,
|
||||
# name : <TooSection name>}]}
|
||||
for k, v in tool_panel_dict_for_tool_config.items():
|
||||
tool_panel_dict_for_display[ k ] = v
|
||||
for tool_panel_dict in v:
|
||||
# Keep track of tool config file names associated with entries that have been made to the migrated_tools_conf.xml file so
|
||||
# they can be eliminated from all non-shed-related tool panel configs.
|
||||
# Keep track of tool config file names associated with entries that have been made to the
|
||||
# migrated_tools_conf.xml file so they can be eliminated from all non-shed-related tool panel configs.
|
||||
tool_config_file = tool_panel_dict.get( 'tool_config', None )
|
||||
if tool_config_file:
|
||||
if tool_config_file not in tool_configs_to_filter:
|
||||
@@ -375,23 +378,28 @@ class InstallManager( object ):
|
||||
log.exception( "Exception attempting to filter and persist non-shed-related tool panel configs:\n%s" % str( e ) )
|
||||
finally:
|
||||
lock.release()
|
||||
metadata_dict, invalid_file_tups = metadata_util.generate_metadata_for_changeset_revision( app=self.app,
|
||||
repository=tool_shed_repository,
|
||||
changeset_revision=tool_shed_repository.changeset_revision,
|
||||
repository_clone_url=repository_clone_url,
|
||||
shed_config_dict = self.shed_config_dict,
|
||||
relative_install_dir=relative_install_dir,
|
||||
repository_files_dir=None,
|
||||
resetting_all_metadata_on_repository=False,
|
||||
updating_installed_repository=False,
|
||||
persist=True )
|
||||
metadata_dict, invalid_file_tups = \
|
||||
metadata_util.generate_metadata_for_changeset_revision( app=self.app,
|
||||
repository=tool_shed_repository,
|
||||
changeset_revision=tool_shed_repository.changeset_revision,
|
||||
repository_clone_url=repository_clone_url,
|
||||
shed_config_dict = self.shed_config_dict,
|
||||
relative_install_dir=relative_install_dir,
|
||||
repository_files_dir=None,
|
||||
resetting_all_metadata_on_repository=False,
|
||||
updating_installed_repository=False,
|
||||
persist=True )
|
||||
tool_shed_repository.metadata = metadata_dict
|
||||
self.app.install_model.context.add( tool_shed_repository )
|
||||
self.app.install_model.context.flush()
|
||||
has_tool_dependencies = self.__has_tool_dependencies( metadata_dict )
|
||||
if has_tool_dependencies:
|
||||
# All tool_dependency objects must be created before the tools are processed even if no tool dependencies will be installed.
|
||||
tool_dependencies = tool_dependency_util.create_tool_dependency_objects( self.app, tool_shed_repository, relative_install_dir, set_status=True )
|
||||
# All tool_dependency objects must be created before the tools are processed even if no
|
||||
# tool dependencies will be installed.
|
||||
tool_dependencies = tool_dependency_util.create_tool_dependency_objects( self.app,
|
||||
tool_shed_repository,
|
||||
relative_install_dir,
|
||||
set_status=True )
|
||||
else:
|
||||
tool_dependencies = None
|
||||
if 'tools' in metadata_dict:
|
||||
|
||||
@@ -339,23 +339,24 @@ def get_update_to_changeset_revision_and_ctx_rev( trans, repository ):
|
||||
changeset_revision_dict[ 'ctx_rev' ] = None
|
||||
return changeset_revision_dict
|
||||
|
||||
def handle_repository_contents( trans, tool_shed_repository, tool_path, repository_clone_url, relative_install_dir, tool_shed=None, tool_section=None, shed_tool_conf=None,
|
||||
reinstalling=False ):
|
||||
def handle_repository_contents( trans, tool_shed_repository, tool_path, repository_clone_url, relative_install_dir,
|
||||
tool_shed=None, tool_section=None, shed_tool_conf=None, reinstalling=False ):
|
||||
"""
|
||||
Generate the metadata for the installed tool shed repository, among other things. This method is called from Galaxy (never the tool shed)
|
||||
when an administrator is installing a new repository or reinstalling an uninstalled repository.
|
||||
Generate the metadata for the installed tool shed repository, among other things. This method is called from Galaxy
|
||||
(never the tool shed) when an administrator is installing a new repository or reinstalling an uninstalled repository.
|
||||
"""
|
||||
shed_config_dict = trans.app.toolbox.get_shed_config_dict_by_filename( shed_tool_conf )
|
||||
metadata_dict, invalid_file_tups = metadata_util.generate_metadata_for_changeset_revision( app=trans.app,
|
||||
repository=tool_shed_repository,
|
||||
changeset_revision=tool_shed_repository.changeset_revision,
|
||||
repository_clone_url=repository_clone_url,
|
||||
shed_config_dict=shed_config_dict,
|
||||
relative_install_dir=relative_install_dir,
|
||||
repository_files_dir=None,
|
||||
resetting_all_metadata_on_repository=False,
|
||||
updating_installed_repository=False,
|
||||
persist=True )
|
||||
metadata_dict, invalid_file_tups = \
|
||||
metadata_util.generate_metadata_for_changeset_revision( app=trans.app,
|
||||
repository=tool_shed_repository,
|
||||
changeset_revision=tool_shed_repository.changeset_revision,
|
||||
repository_clone_url=repository_clone_url,
|
||||
shed_config_dict=shed_config_dict,
|
||||
relative_install_dir=relative_install_dir,
|
||||
repository_files_dir=None,
|
||||
resetting_all_metadata_on_repository=False,
|
||||
updating_installed_repository=False,
|
||||
persist=True )
|
||||
tool_shed_repository.metadata = metadata_dict
|
||||
# Update the tool_shed_repository.tool_shed_status column in the database.
|
||||
tool_shed_status_dict = suc.get_tool_shed_status_for_installed_repository( trans.app, tool_shed_repository )
|
||||
@@ -364,13 +365,20 @@ def handle_repository_contents( trans, tool_shed_repository, tool_path, reposito
|
||||
trans.install_model.context.add( tool_shed_repository )
|
||||
trans.install_model.context.flush()
|
||||
if 'tool_dependencies' in metadata_dict and not reinstalling:
|
||||
tool_dependencies = tool_dependency_util.create_tool_dependency_objects( trans.app, tool_shed_repository, relative_install_dir, set_status=True )
|
||||
tool_dependencies = tool_dependency_util.create_tool_dependency_objects( trans.app,
|
||||
tool_shed_repository,
|
||||
relative_install_dir,
|
||||
set_status=True )
|
||||
if 'sample_files' in metadata_dict:
|
||||
sample_files = metadata_dict.get( 'sample_files', [] )
|
||||
tool_index_sample_files = tool_util.get_tool_index_sample_files( sample_files )
|
||||
tool_data_table_conf_filename, tool_data_table_elems = tool_util.install_tool_data_tables( trans.app, tool_shed_repository, tool_index_sample_files )
|
||||
tool_data_table_conf_filename, tool_data_table_elems = \
|
||||
tool_util.install_tool_data_tables( trans.app, tool_shed_repository, tool_index_sample_files )
|
||||
if tool_data_table_elems:
|
||||
trans.app.tool_data_tables.add_new_entries_from_config_file( tool_data_table_conf_filename, None, trans.app.config.shed_tool_data_table_config, persist=True )
|
||||
trans.app.tool_data_tables.add_new_entries_from_config_file( tool_data_table_conf_filename,
|
||||
None,
|
||||
trans.app.config.shed_tool_data_table_config,
|
||||
persist=True )
|
||||
if 'tools' in metadata_dict:
|
||||
tool_panel_dict = tool_util.generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section )
|
||||
sample_files = metadata_dict.get( 'sample_files', [] )
|
||||
@@ -380,10 +388,19 @@ def handle_repository_contents( trans, tool_shed_repository, tool_path, reposito
|
||||
repository_tools_tups = suc.get_repository_tools_tups( trans.app, metadata_dict )
|
||||
if repository_tools_tups:
|
||||
# Handle missing data table entries for tool parameters that are dynamically generated select lists.
|
||||
repository_tools_tups = tool_util.handle_missing_data_table_entry( trans.app, relative_install_dir, tool_path, repository_tools_tups )
|
||||
repository_tools_tups = tool_util.handle_missing_data_table_entry( trans.app,
|
||||
relative_install_dir,
|
||||
tool_path,
|
||||
repository_tools_tups )
|
||||
# Handle missing index files for tool parameters that are dynamically generated select lists.
|
||||
repository_tools_tups, sample_files_copied = tool_util.handle_missing_index_file( trans.app, tool_path, sample_files, repository_tools_tups, sample_files_copied )
|
||||
# Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance.
|
||||
repository_tools_tups, sample_files_copied = \
|
||||
tool_util.handle_missing_index_file( trans.app,
|
||||
tool_path,
|
||||
sample_files,
|
||||
repository_tools_tups,
|
||||
sample_files_copied )
|
||||
# Copy remaining sample files included in the repository to the ~/tool-data directory of the
|
||||
# local Galaxy instance.
|
||||
tool_util.copy_sample_files( trans.app, sample_files, tool_path=tool_path, sample_files_copied=sample_files_copied )
|
||||
tool_util.add_to_tool_panel( app=trans.app,
|
||||
repository_name=tool_shed_repository.name,
|
||||
@@ -413,16 +430,18 @@ def handle_repository_contents( trans, tool_shed_repository, tool_path, reposito
|
||||
files_dir = os.path.join( shed_config_dict[ 'tool_path' ], files_dir )
|
||||
datatypes_config = suc.get_config_from_disk( suc.DATATYPES_CONFIG_FILENAME, files_dir )
|
||||
# Load data types required by tools.
|
||||
converter_path, display_path = datatype_util.alter_config_and_load_prorietary_datatypes( trans.app, datatypes_config, files_dir, override=False )
|
||||
converter_path, display_path = \
|
||||
datatype_util.alter_config_and_load_prorietary_datatypes( trans.app, datatypes_config, files_dir, override=False )
|
||||
if converter_path or display_path:
|
||||
# Create a dictionary of tool shed repository related information.
|
||||
repository_dict = datatype_util.create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed,
|
||||
name=tool_shed_repository.name,
|
||||
owner=tool_shed_repository.owner,
|
||||
installed_changeset_revision=tool_shed_repository.installed_changeset_revision,
|
||||
tool_dicts=metadata_dict.get( 'tools', [] ),
|
||||
converter_path=converter_path,
|
||||
display_path=display_path )
|
||||
repository_dict = \
|
||||
datatype_util.create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed,
|
||||
name=tool_shed_repository.name,
|
||||
owner=tool_shed_repository.owner,
|
||||
installed_changeset_revision=tool_shed_repository.installed_changeset_revision,
|
||||
tool_dicts=metadata_dict.get( 'tools', [] ),
|
||||
converter_path=converter_path,
|
||||
display_path=display_path )
|
||||
if converter_path:
|
||||
# Load proprietary datatype converters
|
||||
trans.app.datatypes_registry.load_datatype_converters( trans.app.toolbox, installed_repository_dict=repository_dict )
|
||||
|
||||
@@ -568,23 +568,25 @@ def generate_guid_for_object( repository_clone_url, guid_type, obj_id, version )
|
||||
tmp_url = suc.clean_repository_clone_url( repository_clone_url )
|
||||
return '%s/%s/%s/%s' % ( tmp_url, guid_type, obj_id, version )
|
||||
|
||||
def generate_metadata_for_changeset_revision( app, repository, changeset_revision, repository_clone_url, shed_config_dict=None, relative_install_dir=None,
|
||||
repository_files_dir=None, resetting_all_metadata_on_repository=False, updating_installed_repository=False,
|
||||
def generate_metadata_for_changeset_revision( app, repository, changeset_revision, repository_clone_url,
|
||||
shed_config_dict=None, relative_install_dir=None, repository_files_dir=None,
|
||||
resetting_all_metadata_on_repository=False, updating_installed_repository=False,
|
||||
persist=False ):
|
||||
"""
|
||||
Generate metadata for a repository using it's files on disk. To generate metadata for changeset revisions older than the repository tip,
|
||||
the repository will have been cloned to a temporary location and updated to a specified changeset revision to access that changeset revision's
|
||||
disk files, so the value of repository_files_dir will not always be repository.repo_path( app ) (it could be an absolute path to a temporary
|
||||
directory containing a clone). If it is an absolute path, the value of relative_install_dir must contain repository.repo_path( app ).
|
||||
Generate metadata for a repository using it's files on disk. To generate metadata for changeset revisions older than
|
||||
the repository tip, the repository will have been cloned to a temporary location and updated to a specified changeset
|
||||
revision to access that changeset revision's disk files, so the value of repository_files_dir will not always be
|
||||
repository.repo_path( app ) (it could be an absolute path to a temporary directory containing a clone). If it is an
|
||||
absolute path, the value of relative_install_dir must contain repository.repo_path( app ).
|
||||
|
||||
The value of persist will be True when the installed repository contains a valid tool_data_table_conf.xml.sample file, in which case the entries
|
||||
should ultimately be persisted to the file referred to by app.config.shed_tool_data_table_config.
|
||||
The value of persist will be True when the installed repository contains a valid tool_data_table_conf.xml.sample file,
|
||||
in which case the entries should ultimately be persisted to the file referred to by app.config.shed_tool_data_table_config.
|
||||
"""
|
||||
if shed_config_dict is None:
|
||||
shed_config_dict = {}
|
||||
if updating_installed_repository:
|
||||
# Keep the original tool shed repository metadata if setting metadata on a repository installed into a local Galaxy instance for which
|
||||
# we have pulled updates.
|
||||
# Keep the original tool shed repository metadata if setting metadata on a repository installed into a local Galaxy
|
||||
# instance for which we have pulled updates.
|
||||
original_repository_metadata = repository.metadata
|
||||
else:
|
||||
original_repository_metadata = None
|
||||
@@ -603,11 +605,12 @@ def generate_metadata_for_changeset_revision( app, repository, changeset_revisio
|
||||
if resetting_all_metadata_on_repository:
|
||||
if not relative_install_dir:
|
||||
raise Exception( "The value of repository.repo_path( app ) must be sent when resetting all metadata on a repository." )
|
||||
# Keep track of the location where the repository is temporarily cloned so that we can strip the path when setting metadata. The value of
|
||||
# repository_files_dir is the full path to the temporary directory to which the repository was cloned.
|
||||
# Keep track of the location where the repository is temporarily cloned so that we can strip the path when setting metadata.
|
||||
# The value of repository_files_dir is the full path to the temporary directory to which the repository was cloned.
|
||||
work_dir = repository_files_dir
|
||||
files_dir = repository_files_dir
|
||||
# Since we're working from a temporary directory, we can safely copy sample files included in the repository to the repository root.
|
||||
# Since we're working from a temporary directory, we can safely copy sample files included in the repository to the repository
|
||||
# root.
|
||||
app.config.tool_data_path = repository_files_dir
|
||||
app.config.tool_data_table_config_path = repository_files_dir
|
||||
else:
|
||||
@@ -624,10 +627,11 @@ def generate_metadata_for_changeset_revision( app, repository, changeset_revisio
|
||||
if datatypes_config:
|
||||
metadata_dict = generate_datatypes_metadata( app, repository, repository_clone_url, files_dir, datatypes_config, metadata_dict )
|
||||
# Get the relative path to all sample files included in the repository for storage in the repository's metadata.
|
||||
sample_file_metadata_paths, sample_file_copy_paths = get_sample_files_from_disk( repository_files_dir=files_dir,
|
||||
tool_path=shed_config_dict.get( 'tool_path' ),
|
||||
relative_install_dir=relative_install_dir,
|
||||
resetting_all_metadata_on_repository=resetting_all_metadata_on_repository )
|
||||
sample_file_metadata_paths, sample_file_copy_paths = \
|
||||
get_sample_files_from_disk( repository_files_dir=files_dir,
|
||||
tool_path=shed_config_dict.get( 'tool_path' ),
|
||||
relative_install_dir=relative_install_dir,
|
||||
resetting_all_metadata_on_repository=resetting_all_metadata_on_repository )
|
||||
if sample_file_metadata_paths:
|
||||
metadata_dict[ 'sample_files' ] = sample_file_metadata_paths
|
||||
# Copy all sample files included in the repository to a single directory location so we can load tools that depend on them.
|
||||
@@ -636,10 +640,11 @@ def generate_metadata_for_changeset_revision( app, repository, changeset_revisio
|
||||
# If the list of sample files includes a tool_data_table_conf.xml.sample file, laad it's table elements into memory.
|
||||
relative_path, filename = os.path.split( sample_file )
|
||||
if filename == 'tool_data_table_conf.xml.sample':
|
||||
new_table_elems, error_message = app.tool_data_tables.add_new_entries_from_config_file( config_filename=sample_file,
|
||||
tool_data_path=app.config.tool_data_path,
|
||||
shed_tool_data_table_config=app.config.shed_tool_data_table_config,
|
||||
persist=False )
|
||||
new_table_elems, error_message = \
|
||||
app.tool_data_tables.add_new_entries_from_config_file( config_filename=sample_file,
|
||||
tool_data_path=app.config.tool_data_path,
|
||||
shed_tool_data_table_config=app.config.shed_tool_data_table_config,
|
||||
persist=False )
|
||||
if error_message:
|
||||
invalid_file_tups.append( ( filename, error_message ) )
|
||||
for root, dirs, files in os.walk( files_dir ):
|
||||
@@ -650,7 +655,9 @@ def generate_metadata_for_changeset_revision( app, repository, changeset_revisio
|
||||
# See if we have a repository dependencies defined.
|
||||
if name == suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME:
|
||||
path_to_repository_dependencies_config = os.path.join( root, name )
|
||||
metadata_dict, error_message = generate_repository_dependency_metadata( app, path_to_repository_dependencies_config, metadata_dict )
|
||||
metadata_dict, error_message = generate_repository_dependency_metadata( app,
|
||||
path_to_repository_dependencies_config,
|
||||
metadata_dict )
|
||||
if error_message:
|
||||
invalid_file_tups.append( ( name, error_message ) )
|
||||
# See if we have one or more READ_ME files.
|
||||
@@ -666,8 +673,11 @@ def generate_metadata_for_changeset_revision( app, repository, changeset_revisio
|
||||
elif name not in NOT_TOOL_CONFIGS and name.endswith( '.xml' ):
|
||||
full_path = str( os.path.abspath( os.path.join( root, name ) ) )
|
||||
if os.path.getsize( full_path ) > 0:
|
||||
if not ( checkers.check_binary( full_path ) or checkers.check_image( full_path ) or checkers.check_gzip( full_path )[ 0 ]
|
||||
or checkers.check_bz2( full_path )[ 0 ] or checkers.check_zip( full_path ) ):
|
||||
if not ( checkers.check_binary( full_path ) or
|
||||
checkers.check_image( full_path ) or
|
||||
checkers.check_gzip( full_path )[ 0 ] or
|
||||
checkers.check_bz2( full_path )[ 0 ] or
|
||||
checkers.check_zip( full_path ) ):
|
||||
# Make sure we're looking at a tool config and not a display application config or something else.
|
||||
element_tree, error_message = xml_util.parse_xml( full_path )
|
||||
if element_tree is None:
|
||||
@@ -676,13 +686,15 @@ def generate_metadata_for_changeset_revision( app, repository, changeset_revisio
|
||||
element_tree_root = element_tree.getroot()
|
||||
is_tool = element_tree_root.tag == 'tool'
|
||||
if is_tool:
|
||||
tool, valid, error_message = tool_util.load_tool_from_config( app, app.security.encode_id( repository.id ), full_path )
|
||||
tool, valid, error_message = \
|
||||
tool_util.load_tool_from_config( app, app.security.encode_id( repository.id ), full_path )
|
||||
if tool is None:
|
||||
if not valid:
|
||||
invalid_tool_configs.append( name )
|
||||
invalid_file_tups.append( ( name, error_message ) )
|
||||
else:
|
||||
invalid_files_and_errors_tups = tool_util.check_tool_input_params( app, files_dir, name, tool, sample_file_copy_paths )
|
||||
invalid_files_and_errors_tups = \
|
||||
tool_util.check_tool_input_params( app, files_dir, name, tool, sample_file_copy_paths )
|
||||
can_set_metadata = True
|
||||
for tup in invalid_files_and_errors_tups:
|
||||
if name in tup:
|
||||
@@ -690,15 +702,17 @@ def generate_metadata_for_changeset_revision( app, repository, changeset_revisio
|
||||
invalid_tool_configs.append( name )
|
||||
break
|
||||
if can_set_metadata:
|
||||
relative_path_to_tool_config = get_relative_path_to_repository_file( root,
|
||||
name,
|
||||
relative_install_dir,
|
||||
work_dir,
|
||||
shed_config_dict,
|
||||
resetting_all_metadata_on_repository )
|
||||
|
||||
|
||||
metadata_dict = generate_tool_metadata( relative_path_to_tool_config, tool, repository_clone_url, metadata_dict )
|
||||
relative_path_to_tool_config = \
|
||||
get_relative_path_to_repository_file( root,
|
||||
name,
|
||||
relative_install_dir,
|
||||
work_dir,
|
||||
shed_config_dict,
|
||||
resetting_all_metadata_on_repository )
|
||||
metadata_dict = generate_tool_metadata( relative_path_to_tool_config,
|
||||
tool,
|
||||
repository_clone_url,
|
||||
metadata_dict )
|
||||
else:
|
||||
for tup in invalid_files_and_errors_tups:
|
||||
invalid_file_tups.append( tup )
|
||||
@@ -752,8 +766,9 @@ def generate_metadata_for_changeset_revision( app, repository, changeset_revisio
|
||||
|
||||
def generate_package_dependency_metadata( app, elem, valid_tool_dependencies_dict, invalid_tool_dependencies_dict ):
|
||||
"""
|
||||
Generate the metadata for a tool dependencies package defined for a repository. The value of package_name must match the value of the "package"
|
||||
type in the tool config's <requirements> tag set. This method is called from both Galaxy and the tool shed.
|
||||
Generate the metadata for a tool dependencies package defined for a repository. The value of package_name must
|
||||
match the value of the "package" type in the tool config's <requirements> tag set. This method is called from
|
||||
both Galaxy and the tool shed.
|
||||
"""
|
||||
repository_dependency_is_valid = True
|
||||
repository_dependency_tup = []
|
||||
@@ -833,22 +848,23 @@ def generate_repository_dependency_metadata( app, repository_dependencies_config
|
||||
valid_repository_dependencies_dict = dict( description=root.get( 'description' ) )
|
||||
valid_repository_dependency_tups = []
|
||||
for repository_elem in root.findall( 'repository' ):
|
||||
repository_dependency_tup, repository_dependency_is_valid, error_message = handle_repository_elem( app,
|
||||
repository_elem,
|
||||
only_if_compiling_contained_td=False )
|
||||
repository_dependency_tup, repository_dependency_is_valid, err_msg = \
|
||||
handle_repository_elem( app, repository_elem, only_if_compiling_contained_td=False )
|
||||
if repository_dependency_is_valid:
|
||||
valid_repository_dependency_tups.append( repository_dependency_tup )
|
||||
else:
|
||||
# Append the error_message to the repository dependencies tuple.
|
||||
toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = repository_dependency_tup
|
||||
toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \
|
||||
repository_dependency_tup
|
||||
repository_dependency_tup = ( toolshed,
|
||||
name,
|
||||
owner,
|
||||
changeset_revision,
|
||||
prior_installation_required,
|
||||
only_if_compiling_contained_td,
|
||||
error_message )
|
||||
err_msg )
|
||||
invalid_repository_dependency_tups.append( repository_dependency_tup )
|
||||
error_message += err_msg
|
||||
if invalid_repository_dependency_tups:
|
||||
invalid_repository_dependencies_dict[ 'repository_dependencies' ] = invalid_repository_dependency_tups
|
||||
metadata_dict[ 'invalid_repository_dependencies' ] = invalid_repository_dependencies_dict
|
||||
@@ -1142,50 +1158,72 @@ def handle_existing_tool_dependencies_that_changed_in_update( app, repository, o
|
||||
|
||||
def handle_repository_elem( app, repository_elem, only_if_compiling_contained_td=False ):
|
||||
"""
|
||||
Process the received repository_elem which is a <repository> tag either from a repository_dependencies.xml file or a tool_dependencies.xml file.
|
||||
If the former, we're generating repository dependencies metadata for a repository in the tool shed. If the latter, we're generating package
|
||||
dependency metadata within Galaxy or the tool shed.
|
||||
Process the received repository_elem which is a <repository> tag either from a repository_dependencies.xml
|
||||
file or a tool_dependencies.xml file. If the former, we're generating repository dependencies metadata for
|
||||
a repository in the tool shed. If the latter, we're generating package dependency metadata within Galaxy or
|
||||
the tool shed.
|
||||
"""
|
||||
sa_session = app.model.context.current
|
||||
is_valid = True
|
||||
error_message = ''
|
||||
toolshed = repository_elem.get( 'toolshed' )
|
||||
toolshed = repository_elem.get( 'toolshed', None )
|
||||
name = repository_elem.get( 'name', None )
|
||||
owner = repository_elem.get( 'owner', None )
|
||||
changeset_revision = repository_elem.get( 'changeset_revision', None )
|
||||
prior_installation_required = str( repository_elem.get( 'prior_installation_required', False ) )
|
||||
if app.name == 'galaxy':
|
||||
# We're installing a repository into Galaxy, so make sure its contained repository dependency definition
|
||||
# is valid.
|
||||
if toolshed is None or name is None or owner is None or changeset_revision is None:
|
||||
# Raise an exception here instead of returning an error_message to keep the installation from
|
||||
# proceeding. Reaching here implies a bug in the Tool Shed framework.
|
||||
error_message = 'Installation halted because the following repository dependency definition is invalid:\n'
|
||||
error_message += xml_util.xml_to_string( repository_elem, use_indent=True )
|
||||
raise Exception( error_message )
|
||||
if not toolshed:
|
||||
# Default to the current tool shed.
|
||||
toolshed = str( url_for( '/', qualified=True ) ).rstrip( '/' )
|
||||
repository_dependency_tup = [ toolshed,
|
||||
name,
|
||||
owner,
|
||||
changeset_revision,
|
||||
prior_installation_required,
|
||||
str( only_if_compiling_contained_td ) ]
|
||||
cleaned_toolshed = td_common_util.clean_tool_shed_url( toolshed )
|
||||
name = repository_elem.get( 'name' )
|
||||
owner = repository_elem.get( 'owner' )
|
||||
changeset_revision = repository_elem.get( 'changeset_revision' )
|
||||
prior_installation_required = str( repository_elem.get( 'prior_installation_required', False ) )
|
||||
repository_dependency_tup = [ toolshed, name, owner, changeset_revision, prior_installation_required, str( only_if_compiling_contained_td ) ]
|
||||
user = None
|
||||
repository = None
|
||||
if app.name == 'galaxy':
|
||||
# We're in Galaxy. We reach here when we're generating the metadata for a tool dependencies package defined for a repository or when we're
|
||||
# generating metadata for an installed repository. See if we can locate the installed repository via the changeset_revision defined in the
|
||||
# repository_elem (it may be outdated). If we're successful in locating an installed repository with the attributes defined in the
|
||||
# repository_elem, we know it is valid.
|
||||
# We're in Galaxy. We reach here when we're generating the metadata for a tool dependencies package defined
|
||||
# for a repository or when we're generating metadata for an installed repository. See if we can locate the
|
||||
# installed repository via the changeset_revision defined in the repository_elem (it may be outdated). If we're
|
||||
# successful in locating an installed repository with the attributes defined in the repository_elem, we know it
|
||||
# is valid.
|
||||
repository = suc.get_repository_for_dependency_relationship( app, cleaned_toolshed, name, owner, changeset_revision )
|
||||
if repository:
|
||||
return repository_dependency_tup, is_valid, error_message
|
||||
else:
|
||||
# Send a request to the tool shed to retrieve appropriate additional changeset revisions with which the repository may have been installed.
|
||||
# Send a request to the tool shed to retrieve appropriate additional changeset revisions with which the repository
|
||||
# may have been installed.
|
||||
text = install_util.get_updated_changeset_revisions_from_tool_shed( app, toolshed, name, owner, changeset_revision )
|
||||
if text:
|
||||
updated_changeset_revisions = util.listify( text )
|
||||
for updated_changeset_revision in updated_changeset_revisions:
|
||||
repository = suc.get_repository_for_dependency_relationship( app, cleaned_toolshed, name, owner, updated_changeset_revision )
|
||||
repository = suc.get_repository_for_dependency_relationship( app,
|
||||
cleaned_toolshed,
|
||||
name,
|
||||
owner,
|
||||
updated_changeset_revision )
|
||||
if repository:
|
||||
return repository_dependency_tup, is_valid, error_message
|
||||
# Don't generate an error message for missing repository dependencies that are required only if compiling the dependent repository's
|
||||
# tool dependency.
|
||||
# Don't generate an error message for missing repository dependencies that are required only if compiling the
|
||||
# dependent repository's tool dependency.
|
||||
if not only_if_compiling_contained_td:
|
||||
# We'll currently default to setting the repository dependency definition as invalid if an installed repository cannot be found.
|
||||
# This may not be ideal because the tool shed may have simply been inaccessible when metadata was being generated for the installed
|
||||
# tool shed repository.
|
||||
error_message = "Ignoring invalid repository dependency definition for tool shed %s, name %s, owner %s, changeset revision %s "% \
|
||||
( toolshed, name, owner, changeset_revision )
|
||||
# We'll currently default to setting the repository dependency definition as invalid if an installed repository
|
||||
# cannot be found. This may not be ideal because the tool shed may have simply been inaccessible when metadata
|
||||
# was being generated for the installed tool shed repository.
|
||||
error_message = "Ignoring invalid repository dependency definition for tool shed %s, name %s, owner %s, " % \
|
||||
( toolshed, name, owner )
|
||||
error_message += "changeset revision %s." % changeset_revision
|
||||
log.debug( error_message )
|
||||
is_valid = False
|
||||
return repository_dependency_tup, is_valid, error_message
|
||||
@@ -1197,9 +1235,9 @@ def handle_repository_elem( app, repository_elem, only_if_compiling_contained_td
|
||||
.filter( app.model.User.table.c.username == owner ) \
|
||||
.one()
|
||||
except Exception, e:
|
||||
error_message = "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, changeset revision %s "% \
|
||||
( toolshed, name, owner, changeset_revision )
|
||||
error_message += "because the owner is invalid. "
|
||||
error_message = "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, " % \
|
||||
( toolshed, name, owner )
|
||||
error_message += "changeset revision %s because the owner is invalid. " % changeset_revision
|
||||
log.debug( error_message )
|
||||
is_valid = False
|
||||
return repository_dependency_tup, is_valid, error_message
|
||||
@@ -1209,21 +1247,26 @@ def handle_repository_elem( app, repository_elem, only_if_compiling_contained_td
|
||||
app.model.Repository.table.c.user_id == user.id ) ) \
|
||||
.one()
|
||||
except:
|
||||
error_message = "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, changeset revision %s "% \
|
||||
( toolshed, name, owner, changeset_revision )
|
||||
error_message += "because the name is invalid. "
|
||||
error_message = "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, " % \
|
||||
( toolshed, name, owner )
|
||||
error_message += "changeset revision %s because the name is invalid. " % changeset_revision
|
||||
log.debug( error_message )
|
||||
is_valid = False
|
||||
return repository_dependency_tup, is_valid, error_message
|
||||
repo = hg.repository( suc.get_configured_ui(), repository.repo_path( app ) )
|
||||
# The received changeset_revision may be None since defining it in the dependency definition is optional. If this is the case,
|
||||
# the default will be to set it's value to the repository dependency tip revision. This probably occurs only when handling
|
||||
# circular dependency definitions.
|
||||
# The received changeset_revision may be None since defining it in the dependency definition is optional.
|
||||
# If this is the case, the default will be to set it's value to the repository dependency tip revision.
|
||||
# This probably occurs only when handling circular dependency definitions.
|
||||
tip_ctx = repo.changectx( repo.changelog.tip() )
|
||||
# Make sure the repo.changlog includes at least 1 revision.
|
||||
if changeset_revision is None and tip_ctx.rev() >= 0:
|
||||
changeset_revision = str( tip_ctx )
|
||||
repository_dependency_tup = [ toolshed, name, owner, changeset_revision, prior_installation_required, str( only_if_compiling_contained_td ) ]
|
||||
repository_dependency_tup = [ toolshed,
|
||||
name,
|
||||
owner,
|
||||
changeset_revision,
|
||||
prior_installation_required,
|
||||
str( only_if_compiling_contained_td ) ]
|
||||
return repository_dependency_tup, is_valid, error_message
|
||||
else:
|
||||
# Find the specified changeset revision in the repository's changelog to see if it's valid.
|
||||
@@ -1234,16 +1277,17 @@ def handle_repository_elem( app, repository_elem, only_if_compiling_contained_td
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
error_message = "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, changeset revision %s "% \
|
||||
( toolshed, name, owner, changeset_revision )
|
||||
error_message += "because the changeset revision is invalid. "
|
||||
error_message = "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, " % \
|
||||
( toolshed, name, owner )
|
||||
error_message += "changeset revision %s because the changeset revision is invalid. " % changeset_revision
|
||||
log.debug( error_message )
|
||||
is_valid = False
|
||||
return repository_dependency_tup, is_valid, error_message
|
||||
else:
|
||||
# Repository dependencies are currently supported within a single tool shed.
|
||||
error_message = "Repository dependencies are currently supported only within the same tool shed. Ignoring repository dependency definition "
|
||||
error_message += "for tool shed %s, name %s, owner %s, changeset revision %s. " % ( toolshed, name, owner, changeset_revision )
|
||||
error_message = "Repository dependencies are currently supported only within the same tool shed. Ignoring "
|
||||
error_message += "repository dependency definition for tool shed %s, name %s, owner %s, changeset revision %s. " % \
|
||||
( toolshed, name, owner, changeset_revision )
|
||||
log.debug( error_message )
|
||||
is_valid = False
|
||||
return repository_dependency_tup, is_valid, error_message
|
||||
@@ -1811,8 +1855,8 @@ def set_add_to_tool_panel_attribute_for_tool( tool, guid, datatypes ):
|
||||
|
||||
def set_repository_metadata( trans, repository, content_alert_str='', **kwd ):
|
||||
"""
|
||||
Set metadata using the repository's current disk files, returning specific error messages (if any) to alert the repository owner that the changeset
|
||||
has problems.
|
||||
Set metadata using the repository's current disk files, returning specific error messages (if any) to alert the
|
||||
repository owner that the changeset has problems.
|
||||
"""
|
||||
message = ''
|
||||
status = 'done'
|
||||
|
||||
@@ -945,7 +945,10 @@ def get_repository_dependency_types( repository_dependencies ):
|
||||
return has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td
|
||||
|
||||
def get_repository_for_dependency_relationship( app, tool_shed, name, owner, changeset_revision ):
|
||||
"""Return an installed tool_shed_repository database record that is defined by either the current changeset revision or the installed_changeset_revision."""
|
||||
"""
|
||||
Return an installed tool_shed_repository database record that is defined by either the current changeset
|
||||
revision or the installed_changeset_revision.
|
||||
"""
|
||||
# This method is used only in Galaxy, not the tool shed.
|
||||
if tool_shed.endswith( '/' ):
|
||||
tool_shed = tool_shed.rstrip( '/' )
|
||||
@@ -1330,8 +1333,8 @@ def get_tool_shed_repository_status_label( trans, tool_shed_repository=None, nam
|
||||
|
||||
def get_updated_changeset_revisions( trans, name, owner, changeset_revision ):
|
||||
"""
|
||||
Return a string of comma-separated changeset revision hashes for all available updates to the received changeset revision for the repository
|
||||
defined by the received name and owner.
|
||||
Return a string of comma-separated changeset revision hashes for all available updates to the received changeset
|
||||
revision for the repository defined by the received name and owner.
|
||||
"""
|
||||
repository = get_repository_by_name_and_owner( trans.app, name, owner )
|
||||
repo_dir = repository.repo_path( trans.app )
|
||||
@@ -1351,8 +1354,8 @@ def get_updated_changeset_revisions( trans, name, owner, changeset_revision ):
|
||||
|
||||
def get_url_from_tool_shed( app, tool_shed ):
|
||||
"""
|
||||
The value of tool_shed is something like: toolshed.g2.bx.psu.edu. We need the URL to this tool shed, which is something like:
|
||||
http://toolshed.g2.bx.psu.edu/
|
||||
The value of tool_shed is something like: toolshed.g2.bx.psu.edu. We need the URL to this tool shed, which is
|
||||
something like: http://toolshed.g2.bx.psu.edu/
|
||||
"""
|
||||
for shed_name, shed_url in app.tool_shed_registry.tool_sheds.items():
|
||||
if shed_url.find( tool_shed ) >= 0:
|
||||
|
||||
Reference in New Issue
Block a user