opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-10-25 10:24:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

Automated merge

Browse Source
This commit is contained in:
Jeremy Goecks
2014-07-21 13:42:14 -04:00
parent 6a3344ea24 b5137f31ec
commit 3ea21ff715
19 changed files with 743 additions and 661 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py
+2 -2
View File
@@ -413,12 +413,12 @@ class ToolShedRepositoriesController( BaseAPIController ):
# Make sure the current user's API key proves he is an admin user in this Galaxy instance.
if not trans.user_is_admin():
raise HTTPForbidden( detail='You are not authorized to reset metadata on repositories installed into this Galaxy instance.' )
query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=False, order=False )
irmm = InstalledRepositoryMetadataManager( trans.app )
query = irmm.get_query_for_setting_metadata_on_repositories( order=False )
# Now reset metadata on all remaining repositories.
for repository in query:
repository_id = trans.security.encode_id( repository.id )
try:
irmm = InstalledRepositoryMetadataManager( trans.app )
invalid_file_tups, metadata_dict = irmm.reset_all_metadata_on_installed_repository( repository_id )
if invalid_file_tups:
message = tool_util.generate_message_for_invalid_tools( trans.app,
lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py
+22 -20
View File
@@ -30,6 +30,7 @@ from tool_shed.galaxy_install.grids import admin_toolshed_grids
from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager
from tool_shed.galaxy_install.repair_repository_manager import RepairRepositoryManager
from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager
from tool_shed.galaxy_install.tools import tool_panel_manager
log = logging.getLogger( __name__ )
@@ -244,10 +245,10 @@ class AdminToolshed( AdminGalaxy ):
if kwd.get( 'deactivate_or_uninstall_repository_button', False ):
if tool_shed_repository.includes_tools_for_display_in_tool_panel:
# Handle tool panel alterations.
tool_util.remove_from_tool_panel( trans.app,
tool_shed_repository,
shed_tool_conf,
uninstall=remove_from_disk_checked )
tpm = tool_panel_manager.ToolPanelManager( trans.app )
tpm.remove_from_tool_panel( tool_shed_repository,
shed_tool_conf,
uninstall=remove_from_disk_checked )
if tool_shed_repository.includes_data_managers:
data_manager_util.remove_from_data_manager( trans.app, tool_shed_repository )
if tool_shed_repository.includes_datatypes:
@@ -1336,13 +1337,14 @@ class AdminToolshed( AdminGalaxy ):
# its repository dependencies.
includes_tool_dependencies = tool_shed_repository.includes_tool_dependencies
if tool_shed_repository.includes_tools_for_display_in_tool_panel:
tpm = tool_panel_manager.ToolPanelManager( trans.app )
# Handle the selected tool panel location for loading tools included in the tool shed repository.
tool_section, tool_panel_section_key = \
tool_util.handle_tool_panel_selection( toolbox=trans.app.toolbox,
metadata=metadata,
no_changes_checked=no_changes_checked,
tool_panel_section_id=tool_panel_section_id,
new_tool_panel_section_label=new_tool_panel_section_label )
tpm.handle_tool_panel_selection( toolbox=trans.app.toolbox,
metadata=metadata,
no_changes_checked=no_changes_checked,
tool_panel_section_id=tool_panel_section_id,
new_tool_panel_section_label=new_tool_panel_section_label )
if tool_section is not None:
# Just in case the tool_section.id differs from tool_panel_section_id, which it shouldn't...
tool_panel_section_id = str( tool_section.id )
@@ -1706,13 +1708,13 @@ class AdminToolshed( AdminGalaxy ):
@web.expose
@web.require_admin
def reset_metadata_on_selected_installed_repositories( self, trans, **kwd ):
irmm = InstalledRepositoryMetadataManager( trans.app )
if 'reset_metadata_on_selected_repositories_button' in kwd:
irmm = InstalledRepositoryMetadataManager( trans.app )
message, status = irmm.reset_metadata_on_selected_repositories( trans.user, **kwd )
else:
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
repositories_select_field = suc.build_repository_ids_select_field( trans )
repositories_select_field = irmm.build_repository_ids_select_field( trans )
return trans.fill_template( '/admin/tool_shed_repository/reset_metadata_on_selected_repositories.mako',
repositories_select_field=repositories_select_field,
message=message,
@@ -1942,19 +1944,19 @@ class AdminToolshed( AdminGalaxy ):
updating_installed_repository=True,
persist=True )
if 'tools' in metadata_dict:
tpm = tool_panel_manager.ToolPanelManager( trans.app )
tool_panel_dict = metadata_dict.get( 'tool_panel_section', None )
if tool_panel_dict is None:
tool_panel_dict = suc.generate_tool_panel_dict_from_shed_tool_conf_entries( trans.app, repository )
repository_tools_tups = suc.get_repository_tools_tups( trans.app, metadata_dict )
tool_util.add_to_tool_panel( app=trans.app,
repository_name=str( repository.name ),
repository_clone_url=repository_clone_url,
changeset_revision=str( repository.installed_changeset_revision ),
repository_tools_tups=repository_tools_tups,
owner=str( repository.owner ),
shed_tool_conf=shed_tool_conf,
tool_panel_dict=tool_panel_dict,
new_install=False )
tpm.add_to_tool_panel( repository_name=str( repository.name ),
repository_clone_url=repository_clone_url,
changeset_revision=str( repository.installed_changeset_revision ),
repository_tools_tups=repository_tools_tups,
owner=str( repository.owner ),
shed_tool_conf=shed_tool_conf,
tool_panel_dict=tool_panel_dict,
new_install=False )
# Add new Data Manager entries
if 'data_manager' in metadata_dict:
new_data_managers = data_manager_util.install_data_managers( trans.app,
lib/galaxy/webapps/tool_shed/api/repositories.py
+10 -11
View File
@@ -392,7 +392,8 @@ class RepositoriesController( BaseAPIController ):
my_writable = True
handled_repository_ids = []
repository_ids = []
query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False )
rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user )
query = rmm.get_query_for_setting_metadata_on_repositories( my_writable=my_writable, order=False )
# Make sure repositories of type tool_dependency_definition are first in the list.
for repository in query:
if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids:
@@ -426,13 +427,12 @@ class RepositoriesController( BaseAPIController ):
This param can be used as an alternative to the above encoded_ids_to_skip.
"""
def handle_repository( trans, repository, results ):
def handle_repository( trans, rmm, repository, results ):
log.debug( "Resetting metadata on repository %s" % str( repository.name ) )
repository_id = trans.security.encode_id( repository.id )
try:
rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app )
invalid_file_tups, metadata_dict = \
rmm.reset_all_metadata_on_repository_in_tool_shed( trans.user, repository_id )
rmm.reset_all_metadata_on_repository_in_tool_shed( repository_id )
if invalid_file_tups:
message = tool_util.generate_message_for_invalid_tools( trans.app,
invalid_file_tups,
@@ -451,7 +451,7 @@ class RepositoriesController( BaseAPIController ):
status = '%s : %s' % ( str( repository.name ), message )
results[ 'repository_status' ].append( status )
return results
rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user )
start_time = strftime( "%Y-%m-%d %H:%M:%S" )
results = dict( start_time=start_time,
repository_status=[],
@@ -473,7 +473,7 @@ class RepositoriesController( BaseAPIController ):
my_writable = util.asbool( payload.get( 'my_writable', False ) )
else:
my_writable = True
query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False )
query = rmm.get_query_for_setting_metadata_on_repositories( my_writable=my_writable, order=False )
# First reset metadata on all repositories of type repository_dependency_definition.
for repository in query:
encoded_id = trans.security.encode_id( repository.id )
@@ -481,7 +481,7 @@ class RepositoriesController( BaseAPIController ):
log.debug( "Skipping repository with id %s because it is in encoded_ids_to_skip %s" % \
( str( repository.id ), str( encoded_ids_to_skip ) ) )
elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids:
results = handle_repository( trans, repository, results )
results = handle_repository( trans, rmm, repository, results )
# Now reset metadata on all remaining repositories.
for repository in query:
encoded_id = trans.security.encode_id( repository.id )
@@ -489,7 +489,7 @@ class RepositoriesController( BaseAPIController ):
log.debug( "Skipping repository with id %s because it is in encoded_ids_to_skip %s" % \
( str( repository.id ), str( encoded_ids_to_skip ) ) )
elif repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids:
results = handle_repository( trans, repository, results )
results = handle_repository( trans, rmm, repository, results )
stop_time = strftime( "%Y-%m-%d %H:%M:%S" )
results[ 'stop_time' ] = stop_time
return json.dumps( results, sort_keys=True, indent=4 )
@@ -511,10 +511,9 @@ class RepositoriesController( BaseAPIController ):
results = dict( start_time=start_time,
repository_status=[] )
try:
rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app )
rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user )
invalid_file_tups, metadata_dict = \
rmm.reset_all_metadata_on_repository_in_tool_shed( trans.user,
trans.security.encode_id( repository.id ) )
rmm.reset_all_metadata_on_repository_in_tool_shed( trans.security.encode_id( repository.id ) )
if invalid_file_tups:
message = tool_util.generate_message_for_invalid_tools( trans.app,
invalid_file_tups,
lib/galaxy/webapps/tool_shed/controllers/admin.py
+6 -3
View File
@@ -348,13 +348,16 @@ class AdminController( BaseUIController, Admin ):
@web.expose
@web.require_admin
def reset_metadata_on_selected_repositories_in_tool_shed( self, trans, **kwd ):
rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user )
if 'reset_metadata_on_selected_repositories_button' in kwd:
rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app )
message, status = rmm.reset_metadata_on_selected_repositories( trans.user, **kwd )
message, status = rmm.reset_metadata_on_selected_repositories( **kwd )
else:
message = util.restore_text( kwd.get( 'message', '' ) )
status = kwd.get( 'status', 'done' )
repositories_select_field = suc.build_repository_ids_select_field( trans )
repositories_select_field = rmm.build_repository_ids_select_field( name='repository_ids',
multiple=True,
display='checkboxes',
my_writable=False )
return trans.fill_template( '/webapps/tool_shed/common/reset_metadata_on_selected_repositories.mako',
repositories_select_field=repositories_select_field,
message=message,
lib/galaxy/webapps/tool_shed/controllers/hg.py
+2 -2
View File
@@ -48,8 +48,8 @@ class HgController( BaseUIController ):
repo = hg.repository( ui.ui(), repository.repo_path( trans.app ) )
update_repository( repo, ctx_rev=None )
# Set metadata using the repository files on disk.
rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app )
error_message, status = rmm.set_repository_metadata( trans.request.host, trans.user, repository )
rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user )
error_message, status = rmm.set_repository_metadata( trans.request.host, repository )
if status == 'ok' and error_message:
log.debug( "Successfully reset metadata on repository %s owned by %s, but encountered problem: %s" % \
( str( repository.name ), str( repository.user.username ), error_message ) )
lib/galaxy/webapps/tool_shed/controllers/repository.py
+9 -7
View File
@@ -2753,9 +2753,9 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
def reset_all_metadata( self, trans, id, **kwd ):
"""Reset all metadata on the complete changelog for a single repository in the tool shed."""
# This method is called only from the ~/templates/webapps/tool_shed/repository/manage_repository.mako template.
rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app )
rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user )
invalid_file_tups, metadata_dict = \
rmm.reset_all_metadata_on_repository_in_tool_shed( trans.user, id, **kwd )
rmm.reset_all_metadata_on_repository_in_tool_shed( id, **kwd )
if invalid_file_tups:
repository = suc.get_repository_in_tool_shed( trans.app, id )
message = tool_util.generate_message_for_invalid_tools( trans.app,
@@ -2774,13 +2774,16 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
@web.expose
def reset_metadata_on_my_writable_repositories_in_tool_shed( self, trans, **kwd ):
rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user )
if 'reset_metadata_on_selected_repositories_button' in kwd:
rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app )
message, status = rmm.reset_metadata_on_selected_repositories( trans.user, **kwd )
message, status = rmm.reset_metadata_on_selected_repositories( **kwd )
else:
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
repositories_select_field = suc.build_repository_ids_select_field( trans, my_writable=True )
repositories_select_field = rmm.build_repository_ids_select_field( name='repository_ids',
multiple=True,
display='checkboxes',
my_writable=True )
return trans.fill_template( '/webapps/tool_shed/common/reset_metadata_on_selected_repositories.mako',
repositories_select_field=repositories_select_field,
message=message,
@@ -2840,9 +2843,8 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
if tip == repository.tip( trans.app ):
message += 'No changes to repository. '
else:
rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app )
rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user )
status, error_message = rmm.set_repository_metadata_due_to_new_tip( trans.request.host,
trans.user,
repository,
**kwd )
if error_message:
lib/galaxy/webapps/tool_shed/controllers/upload.py
+1 -2
View File
@@ -262,10 +262,9 @@ class UploadController( BaseUIController ):
( len( files_to_remove ), upload_point )
else:
message += " %d files were removed from the repository root. " % len( files_to_remove )
rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app )
rmm = repository_metadata_manager.RepositoryMetadataManager( trans.app, trans.user )
status, error_message = \
rmm.set_repository_metadata_due_to_new_tip( trans.request.host,
trans.user,
repository,
content_alert_str=content_alert_str,
**kwd )
lib/tool_shed/capsule/capsule_manager.py
+1 -2
View File
@@ -727,9 +727,8 @@ class ImportRepositoryManager( object ):
results_dict[ 'ok' ] = False
results_dict[ 'error_message' ] += error_message
try:
rmm = repository_metadata_manager.RepositoryMetadataManager( self.app )
rmm = repository_metadata_manager.RepositoryMetadataManager( self.app, self.user )
status, error_message = rmm.set_repository_metadata_due_to_new_tip( self.host,
self.user,
repository,
content_alert_str=content_alert_str )
if error_message:
lib/tool_shed/galaxy_install/install_manager.py
+19 -14
View File
@@ -33,6 +33,7 @@ from tool_shed.galaxy_install.tool_dependencies.recipe.env_file_builder import E
from tool_shed.galaxy_install.tool_dependencies.recipe.install_environment import InstallEnvironment
from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import StepManager
from tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import TagManager
from tool_shed.galaxy_install.tools import tool_panel_manager
log = logging.getLogger( __name__ )
@@ -535,7 +536,8 @@ class InstallRepositoryManager( object ):
self.app.config.shed_tool_data_table_config,
persist=True )
if 'tools' in metadata_dict:
tool_panel_dict = tool_util.generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section )
tpm = tool_panel_manager.ToolPanelManager( self.app )
tool_panel_dict = tpm.generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section )
sample_files = metadata_dict.get( 'sample_files', [] )
tool_index_sample_files = tool_util.get_tool_index_sample_files( sample_files )
tool_util.copy_sample_files( self.app, tool_index_sample_files, tool_path=tool_path )
@@ -555,16 +557,18 @@ class InstallRepositoryManager( object ):
sample_files_copied )
# Copy remaining sample files included in the repository to the ~/tool-data directory of the
# local Galaxy instance.
tool_util.copy_sample_files( self.app, sample_files, tool_path=tool_path, sample_files_copied=sample_files_copied )
tool_util.add_to_tool_panel( app=self.app,
repository_name=tool_shed_repository.name,
repository_clone_url=repository_clone_url,
changeset_revision=tool_shed_repository.installed_changeset_revision,
repository_tools_tups=repository_tools_tups,
owner=tool_shed_repository.owner,
shed_tool_conf=shed_tool_conf,
tool_panel_dict=tool_panel_dict,
new_install=True )
tool_util.copy_sample_files( self.app,
sample_files,
tool_path=tool_path,
sample_files_copied=sample_files_copied )
tpm.add_to_tool_panel( repository_name=tool_shed_repository.name,
repository_clone_url=repository_clone_url,
changeset_revision=tool_shed_repository.installed_changeset_revision,
repository_tools_tups=repository_tools_tups,
owner=tool_shed_repository.owner,
shed_tool_conf=shed_tool_conf,
tool_panel_dict=tool_panel_dict,
new_install=True )
if 'data_manager' in metadata_dict:
new_data_managers = data_manager_util.install_data_managers( self.app,
self.app.config.shed_data_manager_config_file,
@@ -644,10 +648,11 @@ class InstallRepositoryManager( object ):
tool_shed_url = installation_dict[ 'tool_shed_url' ]
# Handle contained tools.
if includes_tools_for_display_in_tool_panel and ( new_tool_panel_section_label or tool_panel_section_id ):
tpm = tool_panel_manager.ToolPanelManager( self.app )
tool_panel_section_key, tool_section = \
tool_util.handle_tool_panel_section( self.app.toolbox,
tool_panel_section_id=tool_panel_section_id,
new_tool_panel_section_label=new_tool_panel_section_label )
tpm.handle_tool_panel_section( self.app.toolbox,
tool_panel_section_id=tool_panel_section_id,
new_tool_panel_section_label=new_tool_panel_section_label )
else:
tool_panel_section_key = None
tool_section = None
lib/tool_shed/galaxy_install/installed_repository_manager.py
+10 -10
View File
@@ -11,11 +11,11 @@ from tool_shed.util import data_manager_util
from tool_shed.util import datatype_util
from tool_shed.util import shed_util_common as suc
from tool_shed.util import tool_dependency_util
from tool_shed.util import tool_util
from tool_shed.util import xml_util
from galaxy.model.orm import and_
from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager
from tool_shed.galaxy_install.tools import tool_panel_manager
log = logging.getLogger( __name__ )
@@ -80,19 +80,19 @@ class InstalledRepositoryManager( object ):
repository.deleted = False
repository.status = self.install_model.ToolShedRepository.installation_status.INSTALLED
if repository.includes_tools_for_display_in_tool_panel:
tpm = tool_panel_manager.ToolPanelManager( self.app )
metadata = repository.metadata
repository_tools_tups = suc.get_repository_tools_tups( self.app, metadata )
# Reload tools into the appropriate tool panel section.
tool_panel_dict = repository.metadata[ 'tool_panel_section' ]
tool_util.add_to_tool_panel( self.app,
repository.name,
repository_clone_url,
repository.installed_changeset_revision,
repository_tools_tups,
repository.owner,
shed_tool_conf,
tool_panel_dict,
new_install=False )
tpm.add_to_tool_panel( repository.name,
repository_clone_url,
repository.installed_changeset_revision,
repository_tools_tups,
repository.owner,
shed_tool_conf,
tool_panel_dict,
new_install=False )
if repository.includes_data_managers:
tp, data_manager_relative_install_dir = repository.get_tool_relative_path( self.app )
# Hack to add repository.name here, which is actually the root of the installed repository
lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py
+27
View File
@@ -3,6 +3,7 @@ import os
from galaxy import util
from galaxy.util import inflector
from galaxy.web.form_builder import SelectField
from tool_shed.metadata import metadata_generator
@@ -19,6 +20,32 @@ class InstalledRepositoryMetadataManager( metadata_generator.MetadataGenerator )
super( InstalledRepositoryMetadataManager, self ).__init__( app )
self.app = app
def build_repository_ids_select_field( self, name='repository_ids', multiple=True, display='checkboxes' ):
"""Generate the current list of repositories for resetting metadata."""
repositories_select_field = SelectField( name=name, multiple=multiple, display=display )
query = self.get_query_for_setting_metadata_on_repositories( order=True )
for repository in query:
owner = str( repository.owner )
option_label = '%s (%s)' % ( str( repository.name ), owner )
option_value = '%s' % self.app.security.encode_id( repository.id )
repositories_select_field.add_option( option_label, option_value )
return repositories_select_field
def get_query_for_setting_metadata_on_repositories( self, order=True ):
"""
Return a query containing repositories for resetting metadata. The order parameter
is used for displaying the list of repositories ordered alphabetically for display on
a page. When called from the Galaxy API, order is False.
"""
if order:
return self.app.install_model.context.query( self.app.install_model.ToolShedRepository ) \
.filter( self.app.install_model.ToolShedRepository.table.c.uninstalled == False ) \
.order_by( self.app.install_model.ToolShedRepository.table.c.name,
self.app.install_model.ToolShedRepository.table.c.owner )
else:
return self.app.install_model.context.query( self.app.install_model.ToolShedRepository ) \
.filter( self.app.install_model.ToolShedRepository.table.c.uninstalled == False )
def reset_all_metadata_on_installed_repository( self, id ):
"""Reset all metadata on a single tool shed repository installed into a Galaxy instance."""
invalid_file_tups = []
lib/tool_shed/galaxy_install/repair_repository_manager.py
+14 -10
View File
@@ -5,13 +5,14 @@ log = logging.getLogger( __name__ )
from tool_shed.galaxy_install import install_manager
from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager
from tool_shed.galaxy_install.tools import tool_panel_manager
from tool_shed.util import common_util
from tool_shed.util import container_util
from tool_shed.util import shed_util_common as suc
from tool_shed.util import repository_maintenance_util
from tool_shed.util import tool_dependency_util
from tool_shed.util import tool_util
class RepairRepositoryManager():
@@ -119,10 +120,11 @@ class RepairRepositoryManager():
tool_panel_section_id = section_dict[ 'id' ]
tool_panel_section_name = section_dict[ 'name' ]
if tool_panel_section_id:
tpm = tool_panel_manager.ToolPanelManager( self.app )
tool_panel_section_key, tool_panel_section = \
tool_util.get_or_create_tool_section( self.app.toolbox,
tool_panel_section_id=tool_panel_section_id,
new_tool_panel_section_label=tool_panel_section_name )
tpm.get_or_create_tool_section( self.app.toolbox,
tool_panel_section_id=tool_panel_section_id,
new_tool_panel_section_label=tool_panel_section_name )
else:
tool_dependencies = None
repo_info_dict = repository_maintenance_util.create_repo_info_dict( app=self.app,
@@ -156,15 +158,17 @@ class RepairRepositoryManager():
log.debug( error_message )
repair_dict [ repository.name ] = error_message
elif repository.status not in [ self.app.install_model.ToolShedRepository.installation_status.INSTALLED ]:
shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( self.app, repository )
shed_tool_conf, tool_path, relative_install_dir = \
suc.get_tool_panel_config_tool_path_install_dir( self.app, repository )
# Reset the repository attributes to the New state for installation.
if metadata:
tpm = tool_panel_manager.ToolPanelManager( self.app )
tool_section, tool_panel_section_key = \
tool_util.handle_tool_panel_selection( self.app.toolbox,
metadata,
no_changes_checked=True,
tool_panel_section_id=None,
new_tool_panel_section_label=None )
tpm.handle_tool_panel_selection( self.app.toolbox,
metadata,
no_changes_checked=True,
tool_panel_section_id=None,
new_tool_panel_section_label=None )
else:
# The tools will be loaded outside of any sections in the tool panel.
tool_panel_section_key = None
lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py
+11 -9
View File
@@ -10,11 +10,12 @@ import urllib2
from galaxy.util import asbool
from tool_shed.galaxy_install.tools import tool_panel_manager
from tool_shed.util import common_util
from tool_shed.util import container_util
from tool_shed.util import encoding_util
from tool_shed.util import shed_util_common as suc
from tool_shed.util import tool_util
log = logging.getLogger( __name__ )
@@ -216,19 +217,20 @@ class RepositoryDependencyInstallManager( object ):
if can_update_db_record:
# The database record for the tool shed repository currently being processed can be updated.
# Get the repository metadata to see where it was previously located in the tool panel.
tpm = tool_panel_manager.ToolPanelManager( self.app )
if repository_db_record and repository_db_record.metadata:
tool_section, tool_panel_section_key = \
tool_util.handle_tool_panel_selection( toolbox=self.app.toolbox,
metadata=repository_db_record.metadata,
no_changes_checked=no_changes_checked,
tool_panel_section_id=tool_panel_section_id,
new_tool_panel_section_label=new_tool_panel_section_label )
tpm.handle_tool_panel_selection( toolbox=self.app.toolbox,
metadata=repository_db_record.metadata,
no_changes_checked=no_changes_checked,
tool_panel_section_id=tool_panel_section_id,
new_tool_panel_section_label=new_tool_panel_section_label )
else:
# We're installing a new tool shed repository that does not yet have a database record.
tool_panel_section_key, tool_section = \
tool_util.handle_tool_panel_section( self.app.toolbox,
tool_panel_section_id=tool_panel_section_id,
new_tool_panel_section_label=new_tool_panel_section_label )
tpm.handle_tool_panel_section( self.app.toolbox,
tool_panel_section_id=tool_panel_section_id,
new_tool_panel_section_label=new_tool_panel_section_label )
tool_shed_repository = \
suc.create_or_update_tool_shed_repository( app=self.app,
name=name,
lib/tool_shed/galaxy_install/tool_migration_manager.py
+72 -45
View File
@@ -14,6 +14,7 @@ from galaxy.tools import ToolSection
from tool_shed.galaxy_install import install_manager
from tool_shed.galaxy_install.metadata.installed_repository_metadata_manager import InstalledRepositoryMetadataManager
from tool_shed.galaxy_install.tools import tool_panel_manager
from tool_shed.util import basic_util
from tool_shed.util import common_util
@@ -30,23 +31,31 @@ log = logging.getLogger( __name__ )
class ToolMigrationManager( object ):
def __init__( self, app, latest_migration_script_number, tool_shed_install_config, migrated_tools_config, install_dependencies ):
def __init__( self, app, latest_migration_script_number, tool_shed_install_config, migrated_tools_config,
install_dependencies ):
"""
Check tool settings in tool_shed_install_config and install all repositories that are not already installed. The tool
panel configuration file is the received migrated_tools_config, which is the reserved file named migrated_tools_conf.xml.
Check tool settings in tool_shed_install_config and install all repositories
that are not already installed. The tool panel configuration file is the received
migrated_tools_config, which is the reserved file named migrated_tools_conf.xml.
"""
self.app = app
self.toolbox = self.app.toolbox
self.migrated_tools_config = migrated_tools_config
# If install_dependencies is True but tool_dependency_dir is not set, do not attempt to install but print informative error message.
self.tpm = tool_panel_manager.ToolPanelManager( self.app )
# If install_dependencies is True but tool_dependency_dir is not set, do not attempt
# to install but print informative error message.
if install_dependencies and app.config.tool_dependency_dir is None:
message = 'You are attempting to install tool dependencies but do not have a value for "tool_dependency_dir" set in your universe_wsgi.ini '
message += 'file. Set this location value to the path where you want tool dependencies installed and rerun the migration script.'
message = 'You are attempting to install tool dependencies but do not have a value '
message += 'for "tool_dependency_dir" set in your universe_wsgi.ini file. Set this '
message += 'location value to the path where you want tool dependencies installed and '
message += 'rerun the migration script.'
raise Exception( message )
# Get the local non-shed related tool panel configs (there can be more than one, and the default name is tool_conf.xml).
# Get the local non-shed related tool panel configs (there can be more than one, and the
# default name is tool_conf.xml).
self.proprietary_tool_confs = self.non_shed_tool_panel_configs
self.proprietary_tool_panel_elems = self.get_proprietary_tool_panel_elems( latest_migration_script_number )
# Set the location where the repositories will be installed by retrieving the tool_path setting from migrated_tools_config.
# Set the location where the repositories will be installed by retrieving the tool_path
# setting from migrated_tools_config.
tree, error_message = xml_util.parse_xml( migrated_tools_config )
if tree is None:
print error_message
@@ -66,21 +75,24 @@ class ToolMigrationManager( object ):
self.tool_shed = common_util.remove_protocol_and_port_from_tool_shed_url( self.tool_shed_url )
self.repository_owner = common_util.REPOSITORY_OWNER
index, self.shed_config_dict = suc.get_shed_tool_conf_dict( app, self.migrated_tools_config )
# Since tool migration scripts can be executed any number of times, we need to make sure the appropriate tools are defined in
# tool_conf.xml. If no tools associated with the migration stage are defined, no repositories will be installed on disk.
# The default behavior is that the tool shed is down.
# Since tool migration scripts can be executed any number of times, we need to
# make sure the appropriate tools are defined in tool_conf.xml. If no tools
# associated with the migration stage are defined, no repositories will be installed
# on disk. The default behavior is that the tool shed is down.
tool_shed_accessible = False
tool_panel_configs = common_util.get_non_shed_tool_panel_configs( app )
if tool_panel_configs:
# The missing_tool_configs_dict contents are something like:
# {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]}
tool_shed_accessible, missing_tool_configs_dict = common_util.check_for_missing_tools( app,
tool_panel_configs,
latest_migration_script_number )
tool_shed_accessible, missing_tool_configs_dict = \
common_util.check_for_missing_tools( app,
tool_panel_configs,
latest_migration_script_number )
else:
# It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but
# we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in
# the database.
# It doesn't matter if the tool shed is accessible since there are no migrated
# tools defined in the local Galaxy instance, but we have to set the value of
# tool_shed_accessible to True so that the value of migrate_tools.version can
# be correctly set in the database.
tool_shed_accessible = True
missing_tool_configs_dict = odict()
if tool_shed_accessible:
@@ -93,15 +105,18 @@ class ToolMigrationManager( object ):
if missing_tool_configs_dict:
for proprietary_tool_conf in self.proprietary_tool_confs:
# Create a backup of the tool configuration in the un-migrated state.
shutil.copy( proprietary_tool_conf, '%s-pre-stage-%04d' % ( proprietary_tool_conf, latest_migration_script_number ) )
shutil.copy( proprietary_tool_conf, '%s-pre-stage-%04d' % ( proprietary_tool_conf,
latest_migration_script_number ) )
for repository_elem in root:
# Make sure we have a valid repository tag.
if self.__is_valid_repository_tag( repository_elem ):
# Get all repository dependencies for the repository defined by the current repository_elem. Repository dependency
# definitions contained in tool shed repositories with migrated tools must never define a relationship to a repository
# dependency that contains a tool. The repository dependency can only contain items that are not loaded into the Galaxy
# tool panel (e.g., tool dependency definitions, custom datatypes, etc). This restriction must be followed down the
# entire dependency hierarchy.
# Get all repository dependencies for the repository defined by the
# current repository_elem. Repository dependency definitions contained
# in tool shed repositories with migrated tools must never define a
# relationship to a repository dependency that contains a tool. The
# repository dependency can only contain items that are not loaded into
# the Galaxy tool panel (e.g., tool dependency definitions, custom datatypes,
# etc). This restriction must be followed down the entire dependency hierarchy.
name = repository_elem.get( 'name' )
changeset_revision = repository_elem.get( 'changeset_revision' )
tool_shed_accessible, repository_dependencies_dict = \
@@ -110,30 +125,38 @@ class ToolMigrationManager( object ):
name,
self.repository_owner,
changeset_revision )
# Make sure all repository dependency records exist (as tool_shed_repository table rows) in the Galaxy database.
created_tool_shed_repositories = self.create_or_update_tool_shed_repository_records( name,
changeset_revision,
repository_dependencies_dict )
# Make sure all repository dependency records exist (as tool_shed_repository
# table rows) in the Galaxy database.
created_tool_shed_repositories = \
self.create_or_update_tool_shed_repository_records( name,
changeset_revision,
repository_dependencies_dict )
# Order the repositories for proper installation. This process is similar to the
# process used when installing tool shed repositories, but does not handle managing
# tool panel sections and other components since repository dependency definitions
# contained in tool shed repositories with migrated tools must never define a relationship
# to a repository dependency that contains a tool.
ordered_tool_shed_repositories = self.order_repositories_for_installation( created_tool_shed_repositories,
repository_dependencies_dict )
ordered_tool_shed_repositories = \
self.order_repositories_for_installation( created_tool_shed_repositories,
repository_dependencies_dict )
for tool_shed_repository in ordered_tool_shed_repositories:
is_repository_dependency = self.__is_repository_dependency( name, changeset_revision, tool_shed_repository )
is_repository_dependency = self.__is_repository_dependency( name,
changeset_revision,
tool_shed_repository )
self.install_repository( repository_elem,
tool_shed_repository,
install_dependencies,
is_repository_dependency=is_repository_dependency )
else:
message = "\nNo tools associated with migration stage %s are defined in your " % str( latest_migration_script_number )
message += "file%s named %s,\nso no repositories will be installed on disk.\n" % ( plural, file_names )
message = "\nNo tools associated with migration stage %s are defined in your " % \
str( latest_migration_script_number )
message += "file%s named %s,\nso no repositories will be installed on disk.\n" % \
( plural, file_names )
print message
else:
message = "\nThe main Galaxy tool shed is not currently available, so skipped migration stage %s.\n" % str( latest_migration_script_number )
message = "\nThe main Galaxy tool shed is not currently available, so skipped migration stage %s.\n" % \
str( latest_migration_script_number )
message += "Try again later.\n"
print message
@@ -354,16 +377,21 @@ class ToolMigrationManager( object ):
repo_install_dir = relative_install_dir
if not is_repository_dependency:
for tool_elem in repository_elem:
# The tool_elem looks something like this: <tool id="EMBOSS: antigenic1" version="5.0.0" file="emboss_antigenic.xml" />
# The tool_elem looks something like this:
# <tool id="EMBOSS: antigenic1" version="5.0.0" file="emboss_antigenic.xml" />
tool_config = tool_elem.get( 'file' )
guid = self.get_guid( repository_clone_url, relative_install_dir, tool_config )
# See if tool_config is defined inside of a section in self.proprietary_tool_panel_elems.
is_displayed, tool_sections = self.get_containing_tool_sections( tool_config )
if is_displayed:
tool_panel_dict_for_tool_config = \
tool_util.generate_tool_panel_dict_for_tool_config( guid, tool_config, tool_sections=tool_sections )
self.tpm.generate_tool_panel_dict_for_tool_config( guid,
tool_config,
tool_sections=tool_sections )
# The tool-panel_dict has the following structure.
# {<Tool guid> : [{ tool_config : <tool_config_file>, id: <ToolSection id>, version : <ToolSection version>,
# {<Tool guid> : [{ tool_config : <tool_config_file>,
# id: <ToolSection id>,
# version : <ToolSection version>,
# name : <TooSection name>}]}
for k, v in tool_panel_dict_for_tool_config.items():
tool_panel_dict_for_display[ k ] = v
@@ -433,15 +461,14 @@ class ToolMigrationManager( object ):
tool_path=self.tool_path,
sample_files_copied=sample_files_copied )
if not is_repository_dependency:
tool_util.add_to_tool_panel( self.app,
tool_shed_repository.name,
repository_clone_url,
tool_shed_repository.installed_changeset_revision,
repository_tools_tups,
self.repository_owner,
self.migrated_tools_config,
tool_panel_dict=tool_panel_dict_for_display,
new_install=True )
self.tpm.add_to_tool_panel( tool_shed_repository.name,
repository_clone_url,
tool_shed_repository.installed_changeset_revision,
repository_tools_tups,
self.repository_owner,
self.migrated_tools_config,
tool_panel_dict=tool_panel_dict_for_display,
new_install=True )
if install_dependencies and tool_dependencies and has_tool_dependencies:
# Install tool dependencies.
suc.update_tool_shed_repository_status( self.app,
lib/tool_shed/galaxy_install/tools/__init__.py
View File
lib/tool_shed/galaxy_install/tools/tool_panel_manager.py
+465
View File
@@ -0,0 +1,465 @@
import logging
import os
import galaxy.tools
from galaxy.tools.search import ToolBoxSearch
from xml.etree import ElementTree as XmlET
from tool_shed.util import basic_util
from tool_shed.util import common_util
from tool_shed.util import shed_util_common as suc
from tool_shed.util import xml_util
log = logging.getLogger( __name__ )
class ToolPanelManager( object ):
def __init__( self, app ):
self.app = app
def add_to_shed_tool_config( self, shed_tool_conf_dict, elem_list ):
# A tool shed repository is being installed so change the shed_tool_conf file. Parse the
# config file to generate the entire list of config_elems instead of using the in-memory list
# since it will be a subset of the entire list if one or more repositories have been deactivated.
shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ]
tool_path = shed_tool_conf_dict[ 'tool_path' ]
config_elems = []
tree, error_message = xml_util.parse_xml( shed_tool_conf )
if tree:
root = tree.getroot()
for elem in root:
config_elems.append( elem )
# Add the elements to the in-memory list of config_elems.
for elem_entry in elem_list:
config_elems.append( elem_entry )
# Persist the altered shed_tool_config file.
suc.config_elems_to_xml_file( self.app, config_elems, shed_tool_conf, tool_path )
def add_to_tool_panel( self, repository_name, repository_clone_url, changeset_revision, repository_tools_tups, owner,
shed_tool_conf, tool_panel_dict, new_install=True ):
"""A tool shed repository is being installed or updated so handle tool panel alterations accordingly."""
# We need to change the in-memory version and the file system version of the shed_tool_conf file.
index, shed_tool_conf_dict = suc.get_shed_tool_conf_dict( self.app, shed_tool_conf )
tool_path = shed_tool_conf_dict[ 'tool_path' ]
# Generate the list of ElementTree Element objects for each section or tool.
elem_list = self.generate_tool_panel_elem_list( repository_name,
repository_clone_url,
changeset_revision,
tool_panel_dict,
repository_tools_tups,
owner=owner )
if new_install:
# Add the new elements to the shed_tool_conf file on disk.
self.add_to_shed_tool_config( shed_tool_conf_dict, elem_list )
# Use the new elements to add entries to the
config_elems = shed_tool_conf_dict[ 'config_elems' ]
for config_elem in elem_list:
# Add the new elements to the in-memory list of config_elems.
config_elems.append( config_elem )
# Load the tools into the in-memory tool panel.
if config_elem.tag == 'section':
self.app.toolbox.load_section_tag_set( config_elem, tool_path, load_panel_dict=True )
elif config_elem.tag == 'workflow':
self.app.toolbox.load_workflow_tag_set( config_elem,
self.app.toolbox.tool_panel,
self.app.toolbox.integrated_tool_panel,
load_panel_dict=True )
elif config_elem.tag == 'tool':
guid = config_elem.get( 'guid' )
self.app.toolbox.load_tool_tag_set( config_elem,
self.app.toolbox.tool_panel,
self.app.toolbox.integrated_tool_panel,
tool_path,
load_panel_dict=True,
guid=guid )
# Replace the old list of in-memory config_elems with the new list for this shed_tool_conf_dict.
shed_tool_conf_dict[ 'config_elems' ] = config_elems
self.app.toolbox.shed_tool_confs[ index ] = shed_tool_conf_dict
if self.app.config.update_integrated_tool_panel:
# Write the current in-memory version of the integrated_tool_panel.xml file to disk.
self.app.toolbox.write_integrated_tool_panel_config_file()
self.app.toolbox_search = ToolBoxSearch( self.app.toolbox )
def generate_tool_panel_dict_for_new_install( self, tool_dicts, tool_section=None ):
"""
When installing a repository that contains tools, all tools must currently be defined
within the same tool section in the tool panel or outside of any sections.
"""
tool_panel_dict = {}
if tool_section:
section_id = tool_section.id
section_name = tool_section.name
section_version = tool_section.version or ''
else:
section_id = ''
section_name = ''
section_version = ''
for tool_dict in tool_dicts:
if tool_dict.get( 'add_to_tool_panel', True ):
guid = tool_dict[ 'guid' ]
tool_config = tool_dict[ 'tool_config' ]
tool_section_dict = dict( tool_config=tool_config, id=section_id, name=section_name, version=section_version )
if guid in tool_panel_dict:
tool_panel_dict[ guid ].append( tool_section_dict )
else:
tool_panel_dict[ guid ] = [ tool_section_dict ]
return tool_panel_dict
def generate_tool_panel_dict_for_tool_config( self, guid, tool_config, tool_sections=None ):
"""
Create a dictionary of the following type for a single tool config file name.
The intent is to call this method for every tool config in a repository and
append each of these as entries to a tool panel dictionary for the repository.
This enables each tool to be loaded into a different section in the tool panel.
{<Tool guid> :
[{ tool_config : <tool_config_file>,
id: <ToolSection id>,
version : <ToolSection version>,
name : <TooSection name>}]}
"""
tool_panel_dict = {}
file_name = basic_util.strip_path( tool_config )
tool_section_dicts = self. generate_tool_section_dicts( tool_config=file_name,
tool_sections=tool_sections )
tool_panel_dict[ guid ] = tool_section_dicts
return tool_panel_dict
def generate_tool_panel_elem_list( self, repository_name, repository_clone_url, changeset_revision,
tool_panel_dict, repository_tools_tups, owner='' ):
"""Generate a list of ElementTree Element objects for each section or tool."""
elem_list = []
tool_elem = None
cleaned_repository_clone_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url )
if not owner:
owner = suc.get_repository_owner( cleaned_repository_clone_url )
tool_shed = cleaned_repository_clone_url.split( '/repos/' )[ 0 ].rstrip( '/' )
for guid, tool_section_dicts in tool_panel_dict.items():
for tool_section_dict in tool_section_dicts:
tool_section = None
inside_section = False
section_in_elem_list = False
if tool_section_dict[ 'id' ]:
inside_section = True
# Create a new section element only if we haven't already created it.
for index, elem in enumerate( elem_list ):
if elem.tag == 'section':
section_id = elem.get( 'id', None )
if section_id == tool_section_dict[ 'id' ]:
section_in_elem_list = True
tool_section = elem
break
if tool_section is None:
tool_section = self.generate_tool_section_element_from_dict( tool_section_dict )
# Find the tuple containing the current guid from the list of repository_tools_tups.
for repository_tool_tup in repository_tools_tups:
tool_file_path, tup_guid, tool = repository_tool_tup
if tup_guid == guid:
break
if inside_section:
tool_elem = suc.generate_tool_elem( tool_shed,
repository_name,
changeset_revision,
owner,
tool_file_path,
tool,
tool_section )
else:
tool_elem = suc.generate_tool_elem( tool_shed,
repository_name,
changeset_revision,
owner,
tool_file_path,
tool,
None )
if inside_section:
if section_in_elem_list:
elem_list[ index ] = tool_section
else:
elem_list.append( tool_section )
else:
elem_list.append( tool_elem )
return elem_list
def generate_tool_section_dicts( self, tool_config=None, tool_sections=None ):
tool_section_dicts = []
if tool_config is None:
tool_config = ''
if tool_sections:
for tool_section in tool_sections:
# The value of tool_section will be None if the tool is displayed outside
# of any sections in the tool panel.
if tool_section:
section_id = tool_section.id or ''
section_version = tool_section.version or ''
section_name = tool_section.name or ''
else:
section_id = ''
section_version = ''
section_name = ''
tool_section_dicts.append( dict( tool_config=tool_config,
id=section_id,
version=section_version,
name=section_name ) )
else:
tool_section_dicts.append( dict( tool_config=tool_config, id='', version='', name='' ) )
return tool_section_dicts
def generate_tool_section_element_from_dict( self, tool_section_dict ):
# The value of tool_section_dict looks like the following.
# { id: <ToolSection id>, version : <ToolSection version>, name : <TooSection name>}
if tool_section_dict[ 'id' ]:
# Create a new tool section.
tool_section = XmlET.Element( 'section' )
tool_section.attrib[ 'id' ] = tool_section_dict[ 'id' ]
tool_section.attrib[ 'name' ] = tool_section_dict[ 'name' ]
tool_section.attrib[ 'version' ] = tool_section_dict[ 'version' ]
else:
tool_section = None
return tool_section
def get_or_create_tool_section( self, toolbox, tool_panel_section_id, new_tool_panel_section_label=None ):
tool_panel_section_key = str( tool_panel_section_id )
if tool_panel_section_key in toolbox.tool_panel:
# Appending a tool to an existing section in toolbox.tool_panel
tool_section = toolbox.tool_panel[ tool_panel_section_key ]
log.debug( "Appending to tool panel section: %s" % str( tool_section.name ) )
else:
# Appending a new section to toolbox.tool_panel
if new_tool_panel_section_label is None:
# This might add an ugly section label to the tool panel, but, oh well...
new_tool_panel_section_label = tool_panel_section_id
elem = XmlET.Element( 'section' )
elem.attrib[ 'name' ] = new_tool_panel_section_label
elem.attrib[ 'id' ] = tool_panel_section_id
elem.attrib[ 'version' ] = ''
tool_section = galaxy.tools.ToolSection( elem )
toolbox.tool_panel[ tool_panel_section_key ] = tool_section
log.debug( "Loading new tool panel section: %s" % str( tool_section.name ) )
return tool_panel_section_key, tool_section
def handle_tool_panel_section( self, toolbox, tool_panel_section_id=None, new_tool_panel_section_label=None ):
"""Return a ToolSection object retrieved from the current in-memory tool_panel."""
# If tool_panel_section_id is received, the section exists in the tool panel. In this
# case, the value of the received tool_panel_section_id must be the id retrieved from a
# tool panel config (e.g., tool_conf.xml, which may have getext). If new_tool_panel_section_label
# is received, a new section will be added to the tool panel.
if new_tool_panel_section_label:
section_id = str( new_tool_panel_section_label.lower().replace( ' ', '_' ) )
tool_panel_section_key, tool_section = \
self.get_or_create_tool_section( toolbox,
tool_panel_section_id=section_id,
new_tool_panel_section_label=new_tool_panel_section_label )
elif tool_panel_section_id:
tool_panel_section_key = str( tool_panel_section_id )
tool_section = toolbox.tool_panel[ tool_panel_section_key ]
else:
return None, None
return tool_panel_section_key, tool_section
def handle_tool_panel_selection( self, toolbox, metadata, no_changes_checked, tool_panel_section_id,
new_tool_panel_section_label ):
"""
Handle the selected tool panel location for loading tools included in tool shed
repositories when installing or reinstalling them.
"""
# Get the location in the tool panel in which each tool was originally loaded.
tool_section = None
tool_panel_section_key = None
if 'tools' in metadata:
# This forces everything to be loaded into the same section (or no section)
# in the tool panel.
if no_changes_checked:
# Make sure the no_changes check box overrides the new_tool_panel_section_label
# if the user checked the check box and entered something into the field.
new_tool_panel_section_label = None
if 'tool_panel_section' in metadata:
tool_panel_dict = metadata[ 'tool_panel_section' ]
if not tool_panel_dict:
tool_panel_dict = self.generate_tool_panel_dict_for_new_install( metadata[ 'tools' ] )
else:
tool_panel_dict = self.generate_tool_panel_dict_for_new_install( metadata[ 'tools' ] )
if tool_panel_dict:
# The tool_panel_dict is empty when tools exist but are not installed into a tool panel section.
tool_section_dicts = tool_panel_dict[ tool_panel_dict.keys()[ 0 ] ]
tool_section_dict = tool_section_dicts[ 0 ]
original_section_id = tool_section_dict[ 'id' ]
original_section_name = tool_section_dict[ 'name' ]
if original_section_id:
tool_panel_section_key, tool_section = \
self.get_or_create_tool_section( toolbox,
tool_panel_section_id=original_section_id,
new_tool_panel_section_label=new_tool_panel_section_label )
else:
# The user elected to change the tool panel section to contain the tools.
tool_panel_section_key, tool_section = \
self.handle_tool_panel_section( toolbox,
tool_panel_section_id=tool_panel_section_id,
new_tool_panel_section_label=new_tool_panel_section_label )
return tool_section, tool_panel_section_key
def remove_from_shed_tool_config( self, shed_tool_conf_dict, guids_to_remove ):
# A tool shed repository is being uninstalled so change the shed_tool_conf file.
# Parse the config file to generate the entire list of config_elems instead of
# using the in-memory list since it will be a subset of the entire list if one
# or more repositories have been deactivated.
shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ]
tool_path = shed_tool_conf_dict[ 'tool_path' ]
config_elems = []
tree, error_message = xml_util.parse_xml( shed_tool_conf )
if tree:
root = tree.getroot()
for elem in root:
config_elems.append( elem )
config_elems_to_remove = []
for config_elem in config_elems:
if config_elem.tag == 'section':
tool_elems_to_remove = []
for tool_elem in config_elem:
if tool_elem.get( 'guid' ) in guids_to_remove:
tool_elems_to_remove.append( tool_elem )
for tool_elem in tool_elems_to_remove:
# Remove all of the appropriate tool sub-elements from the section element.
config_elem.remove( tool_elem )
if len( config_elem ) < 1:
# Keep a list of all empty section elements so they can be removed.
config_elems_to_remove.append( config_elem )
elif config_elem.tag == 'tool':
if config_elem.get( 'guid' ) in guids_to_remove:
config_elems_to_remove.append( config_elem )
for config_elem in config_elems_to_remove:
config_elems.remove( config_elem )
# Persist the altered in-memory version of the tool config.
suc.config_elems_to_xml_file( self.app, config_elems, shed_tool_conf, tool_path )
def remove_from_tool_panel( self, repository, shed_tool_conf, uninstall ):
"""
A tool shed repository is being deactivated or uninstalled, so handle tool panel
alterations accordingly.
"""
# Determine where the tools are currently defined in the tool panel and store this
# information so the tools can be displayed in the same way when the repository is
# activated or reinstalled.
tool_panel_dict = suc.generate_tool_panel_dict_from_shed_tool_conf_entries( self.app, repository )
repository.metadata[ 'tool_panel_section' ] = tool_panel_dict
self.app.install_model.context.add( repository )
self.app.install_model.context.flush()
# Create a list of guids for all tools that will be removed from the in-memory tool panel
# and config file on disk.
guids_to_remove = [ k for k in tool_panel_dict.keys() ]
# Remove the tools from the toolbox's tools_by_id dictionary.
for guid_to_remove in guids_to_remove:
if guid_to_remove in self.app.toolbox.tools_by_id:
del self.app.toolbox.tools_by_id[ guid_to_remove ]
index, shed_tool_conf_dict = suc.get_shed_tool_conf_dict( self.app, shed_tool_conf )
if uninstall:
# Remove from the shed_tool_conf file on disk.
self.remove_from_shed_tool_config( shed_tool_conf_dict, guids_to_remove )
config_elems = shed_tool_conf_dict[ 'config_elems' ]
config_elems_to_remove = []
for config_elem in config_elems:
if config_elem.tag == 'section':
# Get the section key for the in-memory tool panel.
section_key = str( config_elem.get( "id" ) )
# Generate the list of tool elements to remove.
tool_elems_to_remove = []
for tool_elem in config_elem:
if tool_elem.get( 'guid' ) in guids_to_remove:
tool_elems_to_remove.append( tool_elem )
for tool_elem in tool_elems_to_remove:
if tool_elem in config_elem:
# Remove the tool sub-element from the section element.
config_elem.remove( tool_elem )
# Remove the tool from the section in the in-memory tool panel.
if section_key in self.app.toolbox.tool_panel:
tool_section = self.app.toolbox.tool_panel[ section_key ]
guid = tool_elem.get( 'guid' )
tool_key = 'tool_%s' % str( guid )
# Get the list of versions of this tool that are currently available in the toolbox.
available_tool_versions = self.app.toolbox.get_loaded_tools_by_lineage( guid )
if tool_key in tool_section.elems:
if available_tool_versions:
available_tool_versions.reverse()
replacement_tool_key = None
replacement_tool_version = None
# Since we are going to remove the tool from the section, replace it with the
# newest loaded version of the tool.
for available_tool_version in available_tool_versions:
available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section()
if available_tool_version.id in tool_section.elems.keys() or section_key == available_tool_section_id:
replacement_tool_key = 'tool_%s' % str( available_tool_version.id )
replacement_tool_version = available_tool_version
break
if replacement_tool_key and replacement_tool_version:
# Get the index of the tool_key in the tool_section.
for tool_section_elems_index, key in enumerate( tool_section.elems.keys() ):
if key == tool_key:
break
# Remove the tool from the tool section.
del tool_section.elems[ tool_key ]
# Add the replacement tool at the same location in the tool section.
tool_section.elems.insert( tool_section_elems_index,
replacement_tool_key,
replacement_tool_version )
else:
del tool_section.elems[ tool_key ]
else:
del tool_section.elems[ tool_key ]
if uninstall:
# Remove the tool from the section in the in-memory integrated tool panel.
if section_key in self.app.toolbox.integrated_tool_panel:
tool_section = self.app.toolbox.integrated_tool_panel[ section_key ]
tool_key = 'tool_%s' % str( tool_elem.get( 'guid' ) )
if tool_key in tool_section.elems:
del tool_section.elems[ tool_key ]
if len( config_elem ) < 1:
# Keep a list of all empty section elements so they can be removed.
config_elems_to_remove.append( config_elem )
elif config_elem.tag == 'tool':
guid = config_elem.get( 'guid' )
if guid in guids_to_remove:
tool_key = 'tool_%s' % str( config_elem.get( 'guid' ) )
# Get the list of versions of this tool that are currently available in the toolbox.
available_tool_versions = self.app.toolbox.get_loaded_tools_by_lineage( guid )
if tool_key in self.app.toolbox.tool_panel:
if available_tool_versions:
available_tool_versions.reverse()
replacement_tool_key = None
replacement_tool_version = None
# Since we are going to remove the tool from the section, replace it with
# the newest loaded version of the tool.
for available_tool_version in available_tool_versions:
available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section()
if available_tool_version.id in self.app.toolbox.tool_panel.keys() or not available_tool_section_id:
replacement_tool_key = 'tool_%s' % str( available_tool_version.id )
replacement_tool_version = available_tool_version
break
if replacement_tool_key and replacement_tool_version:
# Get the index of the tool_key in the tool_section.
for tool_panel_index, key in enumerate( self.app.toolbox.tool_panel.keys() ):
if key == tool_key:
break
# Remove the tool from the tool panel.
del self.app.toolbox.tool_panel[ tool_key ]
# Add the replacement tool at the same location in the tool panel.
self.app.toolbox.tool_panel.insert( tool_panel_index,
replacement_tool_key,
replacement_tool_version )
else:
del self.app.toolbox.tool_panel[ tool_key ]
else:
del self.app.toolbox.tool_panel[ tool_key ]
if uninstall:
if tool_key in self.app.toolbox.integrated_tool_panel:
del self.app.toolbox.integrated_tool_panel[ tool_key ]
config_elems_to_remove.append( config_elem )
for config_elem in config_elems_to_remove:
# Remove the element from the in-memory list of elements.
config_elems.remove( config_elem )
# Update the config_elems of the in-memory shed_tool_conf_dict.
shed_tool_conf_dict[ 'config_elems' ] = config_elems
self.app.toolbox.shed_tool_confs[ index ] = shed_tool_conf_dict
self.app.toolbox_search = ToolBoxSearch( self.app.toolbox )
if uninstall and self.app.config.update_integrated_tool_panel:
# Write the current in-memory version of the integrated_tool_panel.xml file to disk.
self.app.toolbox.write_integrated_tool_panel_config_file()
lib/tool_shed/metadata/repository_metadata_manager.py
+72 -10
View File
@@ -2,11 +2,13 @@ import logging
import os
import tempfile
from galaxy.util import inflector
from galaxy import util
from galaxy.util import inflector
from galaxy.web.form_builder import SelectField
from tool_shed.metadata import metadata_generator
from tool_shed.repository_types.metadata import TipOnly
from tool_shed.repository_types import util as rt_util
from tool_shed.util import basic_util
from tool_shed.util import common_util
@@ -20,9 +22,10 @@ log = logging.getLogger( __name__ )
class RepositoryMetadataManager( metadata_generator.MetadataGenerator ):
def __init__( self, app ):
def __init__( self, app, user ):
super( RepositoryMetadataManager, self ).__init__( app )
self.app = app
self.user = user
# Repository metadata comparisons for changeset revisions.
self.EQUAL = 'equal'
self.NO_METADATA = 'no metadata'
@@ -47,6 +50,18 @@ class RepositoryMetadataManager( metadata_generator.MetadataGenerator ):
self.sa_session.add( repository_metadata )
self.sa_session.flush()
def build_repository_ids_select_field( self, name='repository_ids', multiple=True, display='checkboxes',
my_writable=False ):
"""Generate the current list of repositories for resetting metadata."""
repositories_select_field = SelectField( name=name, multiple=multiple, display=display )
query = self.get_query_for_setting_metadata_on_repositories( my_writable=my_writable, order=True )
for repository in query:
owner = str( repository.user.username )
option_label = '%s (%s)' % ( str( repository.name ), owner )
option_value = '%s' % self.app.security.encode_id( repository.id )
repositories_select_field.add_option( option_label, option_value )
return repositories_select_field
def clean_repository_metadata( self, id, changeset_revisions ):
# Delete all repository_metadata records associated with the repository that have
# a changeset_revision that is not in changeset_revisions. We sometimes see multiple
@@ -458,6 +473,54 @@ class RepositoryMetadataManager( metadata_generator.MetadataGenerator ):
# The tool did not change through all of the changeset revisions.
return old_id
def get_query_for_setting_metadata_on_repositories( self, my_writable=False, order=True ):
"""
Return a query containing repositories for resetting metadata. The order parameter
is used for displaying the list of repositories ordered alphabetically for display on
a page. When called from the Tool Shed API, order is False.
"""
# When called from the Tool Shed API, the metadata is reset on all repositories of types
# repository_suite_definition and tool_dependency_definition in addition to other selected
# repositories.
if my_writable:
username = self.user.username
clause_list = []
for repository in self.sa_session.query( self.app.model.Repository ) \
.filter( self.app.model.Repository.table.c.deleted == False ):
# Always reset metadata on all repositories of types repository_suite_definition and
# tool_dependency_definition.
if repository.type in [ rt_util.REPOSITORY_SUITE_DEFINITION, rt_util.TOOL_DEPENDENCY_DEFINITION ]:
clause_list.append( self.app.model.Repository.table.c.id == repository.id )
else:
allow_push = repository.allow_push( self.app )
if allow_push:
# Include all repositories that are writable by the current user.
allow_push_usernames = allow_push.split( ',' )
if username in allow_push_usernames:
clause_list.append( self.app.model.Repository.table.c.id == repository.id )
if clause_list:
if order:
return self.sa_session.query( self.app.model.Repository ) \
.filter( or_( *clause_list ) ) \
.order_by( self.app.model.Repository.table.c.name,
self.app.model.Repository.table.c.user_id )
else:
return self.sa_session.query( self.app.model.Repository ) \
.filter( or_( *clause_list ) )
else:
# Return an empty query.
return self.sa_session.query( self.app.model.Repository ) \
.filter( self.app.model.Repository.table.c.id == -1 )
else:
if order:
return self.sa_session.query( self.app.model.Repository ) \
.filter( self.app.model.Repository.table.c.deleted == False ) \
.order_by( self.app.model.Repository.table.c.name,
self.app.model.Repository.table.c.user_id )
else:
return self.sa_session.query( self.app.model.Repository ) \
.filter( self.app.model.Repository.table.c.deleted == False )
def new_datatypes_metadata_required( self, repository_metadata, metadata_dict ):
"""
Compare the last saved metadata for each datatype in the repository with the new metadata
@@ -732,13 +795,13 @@ class RepositoryMetadataManager( metadata_generator.MetadataGenerator ):
# repository_metadata table record is not needed.
return False
def reset_all_metadata_on_repository_in_tool_shed( self, user, id ):
def reset_all_metadata_on_repository_in_tool_shed( self, id ):
"""Reset all metadata on a single repository in a tool shed."""
repository = suc.get_repository_in_tool_shed( self.app, id )
log.debug( "Resetting all metadata on repository: %s" % repository.name )
repo_dir = repository.repo_path( self.app )
repo = hg_util.get_repo_for_repository( self.app, repository=None, repo_path=repo_dir, create=False )
repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( user, repository )
repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( self.user, repository )
# The list of changeset_revisions refers to repository_metadata records that have been created
# or updated. When the following loop completes, we'll delete all repository_metadata records
# for this repository that do not have a changeset_revision value in this list.
@@ -875,7 +938,7 @@ class RepositoryMetadataManager( metadata_generator.MetadataGenerator ):
self.sa_session.add( repository_metadata )
self.sa_session.flush()
def reset_metadata_on_selected_repositories( self, user, **kwd ):
def reset_metadata_on_selected_repositories( self, **kwd ):
"""
Inspect the repository changelog to reset metadata for all appropriate changeset revisions.
This method is called from both Galaxy and the Tool Shed.
@@ -892,7 +955,7 @@ class RepositoryMetadataManager( metadata_generator.MetadataGenerator ):
repository = suc.get_repository_in_tool_shed( self.app, repository_id )
owner = str( repository.user.username )
invalid_file_tups, metadata_dict = \
self.reset_all_metadata_on_repository_in_tool_shed( user, repository_id )
self.reset_all_metadata_on_repository_in_tool_shed( self.user, repository_id )
if invalid_file_tups:
message = tool_util.generate_message_for_invalid_tools( self.app,
invalid_file_tups,
@@ -918,7 +981,7 @@ class RepositoryMetadataManager( metadata_generator.MetadataGenerator ):
status = 'error'
return message, status
def set_repository_metadata( self, host, user, repository, content_alert_str='', **kwd ):
def set_repository_metadata( self, host, repository, content_alert_str='', **kwd ):
"""
Set metadata using the repository's current disk files, returning specific error
messages (if any) to alert the repository owner that the changeset has problems.
@@ -926,7 +989,7 @@ class RepositoryMetadataManager( metadata_generator.MetadataGenerator ):
message = ''
status = 'done'
encoded_id = self.app.security.encode_id( repository.id )
repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( user, repository )
repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( self.user, repository )
repo_dir = repository.repo_path( self.app )
repo = hg_util.get_repo_for_repository( self.app, repository=None, repo_path=repo_dir, create=False )
metadata_dict, invalid_file_tups = \
@@ -1027,10 +1090,9 @@ class RepositoryMetadataManager( metadata_generator.MetadataGenerator ):
tool_util.reset_tool_data_tables( self.app )
return message, status
def set_repository_metadata_due_to_new_tip( self, host, user, repository, content_alert_str=None, **kwd ):
def set_repository_metadata_due_to_new_tip( self, host, repository, content_alert_str=None, **kwd ):
"""Set metadata on the repository tip in the tool shed."""
error_message, status = self.set_repository_metadata( host,
user,
repository,
content_alert_str=content_alert_str,
**kwd )
lib/tool_shed/util/shed_util_common.py
-74
View File
@@ -84,20 +84,6 @@ This message was sent from the Galaxy Tool Shed instance hosted on the server
'${host}'
"""
def build_repository_ids_select_field( trans, name='repository_ids', multiple=True, display='checkboxes', my_writable=False ):
"""Method called from both Galaxy and the Tool Shed to generate the current list of repositories for resetting metadata."""
repositories_select_field = SelectField( name=name, multiple=multiple, display=display )
query = get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=True )
for repository in query:
if trans.webapp.name == 'tool_shed':
owner = str( repository.user.username )
else:
owner = str( repository.owner )
option_label = '%s (%s)' % ( str( repository.name ), owner )
option_value = '%s' % trans.security.encode_id( repository.id )
repositories_select_field.add_option( option_label, option_value )
return repositories_select_field
def build_tool_dependencies_select_field( app, tool_shed_repository, name, multiple=True, display='checkboxes', uninstalled_only=False ):
"""Method called from Galaxy to generate the current list of tool dependency ids for an installed tool shed repository."""
tool_dependencies_select_field = SelectField( name=name, multiple=multiple, display=display )
@@ -645,66 +631,6 @@ def get_prior_import_or_install_required_dict( app, tsr_ids, repo_info_dicts ):
prior_import_or_install_required_dict[ encoded_repository_id ] = prior_import_or_install_ids
return prior_import_or_install_required_dict
def get_query_for_setting_metadata_on_repositories( trans, my_writable=False, order=True ):
"""
Return a query containing repositories for resetting metadata. This method is called from both the
Tool Shed and Galaxy. The my_writable parameter is ignored unless called from the Tool Shed, and the
order parameter is used for displaying the list of repositories ordered alphabetically for display on
a page. When called from either the Tool Shed or Galaxy API, order is False.
"""
if trans.webapp.name == 'tool_shed':
# When called from the Tool Shed API, the metadata is reset on all repositories of types
# repository_suite_definition and tool_dependency_definition in addition to other selected
# repositories.
if my_writable:
username = trans.user.username
clause_list = []
for repository in trans.sa_session.query( trans.model.Repository ) \
.filter( trans.model.Repository.table.c.deleted == False ):
# Always reset metadata on all repositories of types repository_suite_definition and
# tool_dependency_definition.
if repository.type in [ rt_util.REPOSITORY_SUITE_DEFINITION, rt_util.TOOL_DEPENDENCY_DEFINITION ]:
clause_list.append( trans.model.Repository.table.c.id == repository.id )
else:
allow_push = repository.allow_push( trans.app )
if allow_push:
# Include all repositories that are writable by the current user.
allow_push_usernames = allow_push.split( ',' )
if username in allow_push_usernames:
clause_list.append( trans.model.Repository.table.c.id == repository.id )
if clause_list:
if order:
return trans.sa_session.query( trans.model.Repository ) \
.filter( or_( *clause_list ) ) \
.order_by( trans.model.Repository.table.c.name,
trans.model.Repository.table.c.user_id )
else:
return trans.sa_session.query( trans.model.Repository ) \
.filter( or_( *clause_list ) )
else:
# Return an empty query.
return trans.sa_session.query( trans.model.Repository ) \
.filter( trans.model.Repository.table.c.id == -1 )
else:
if order:
return trans.sa_session.query( trans.model.Repository ) \
.filter( trans.model.Repository.table.c.deleted == False ) \
.order_by( trans.model.Repository.table.c.name,
trans.model.Repository.table.c.user_id )
else:
return trans.sa_session.query( trans.model.Repository ) \
.filter( trans.model.Repository.table.c.deleted == False )
else:
# We're in Galaxy.
if order:
return trans.install_model.context.query( trans.install_model.ToolShedRepository ) \
.filter( trans.install_model.ToolShedRepository.table.c.uninstalled == False ) \
.order_by( trans.install_model.ToolShedRepository.table.c.name,
trans.install_model.ToolShedRepository.table.c.owner )
else:
return trans.install_model.context.query( trans.install_model.ToolShedRepository ) \
.filter( trans.install_model.ToolShedRepository.table.c.uninstalled == False )
def get_repo_info_tuple_contents( repo_info_tuple ):
"""Take care in handling the repo_info_tuple as it evolves over time as new tool shed features are introduced."""
if len( repo_info_tuple ) == 6:
lib/tool_shed/util/tool_util.py
-440
View File
@@ -19,69 +19,6 @@ from xml.etree import ElementTree as XmlET
log = logging.getLogger( __name__ )
def add_to_shed_tool_config( app, shed_tool_conf_dict, elem_list ):
# A tool shed repository is being installed so change the shed_tool_conf file. Parse the
# config file to generate the entire list of config_elems instead of using the in-memory list
# since it will be a subset of the entire list if one or more repositories have been deactivated.
shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ]
tool_path = shed_tool_conf_dict[ 'tool_path' ]
config_elems = []
tree, error_message = xml_util.parse_xml( shed_tool_conf )
if tree:
root = tree.getroot()
for elem in root:
config_elems.append( elem )
# Add the elements to the in-memory list of config_elems.
for elem_entry in elem_list:
config_elems.append( elem_entry )
# Persist the altered shed_tool_config file.
suc.config_elems_to_xml_file( app, config_elems, shed_tool_conf, tool_path )
def add_to_tool_panel( app, repository_name, repository_clone_url, changeset_revision, repository_tools_tups, owner,
shed_tool_conf, tool_panel_dict, new_install=True ):
"""A tool shed repository is being installed or updated so handle tool panel alterations accordingly."""
# We need to change the in-memory version and the file system version of the shed_tool_conf file.
index, shed_tool_conf_dict = suc.get_shed_tool_conf_dict( app, shed_tool_conf )
tool_path = shed_tool_conf_dict[ 'tool_path' ]
# Generate the list of ElementTree Element objects for each section or tool.
elem_list = generate_tool_panel_elem_list( repository_name,
repository_clone_url,
changeset_revision,
tool_panel_dict,
repository_tools_tups,
owner=owner )
if new_install:
# Add the new elements to the shed_tool_conf file on disk.
add_to_shed_tool_config( app, shed_tool_conf_dict, elem_list )
# Use the new elements to add entries to the
config_elems = shed_tool_conf_dict[ 'config_elems' ]
for config_elem in elem_list:
# Add the new elements to the in-memory list of config_elems.
config_elems.append( config_elem )
# Load the tools into the in-memory tool panel.
if config_elem.tag == 'section':
app.toolbox.load_section_tag_set( config_elem, tool_path, load_panel_dict=True )
elif config_elem.tag == 'workflow':
app.toolbox.load_workflow_tag_set( config_elem,
app.toolbox.tool_panel,
app.toolbox.integrated_tool_panel,
load_panel_dict=True )
elif config_elem.tag == 'tool':
guid = config_elem.get( 'guid' )
app.toolbox.load_tool_tag_set( config_elem,
app.toolbox.tool_panel,
app.toolbox.integrated_tool_panel,
tool_path,
load_panel_dict=True,
guid=guid )
# Replace the old list of in-memory config_elems with the new list for this shed_tool_conf_dict.
shed_tool_conf_dict[ 'config_elems' ] = config_elems
app.toolbox.shed_tool_confs[ index ] = shed_tool_conf_dict
if app.config.update_integrated_tool_panel:
# Write the current in-memory version of the integrated_tool_panel.xml file to disk.
app.toolbox.write_integrated_tool_panel_config_file()
app.toolbox_search = ToolBoxSearch( app.toolbox )
def build_shed_tool_conf_select_field( app ):
"""Build a SelectField whose options are the keys in app.toolbox.shed_tool_confs."""
options = []
@@ -192,142 +129,6 @@ def generate_message_for_invalid_tools( app, invalid_file_tups, repository, meta
message += "%s%s%s - %s%s" % ( bold_start, tool_file, bold_end, correction_msg, new_line )
return message
def generate_tool_panel_dict_for_new_install( tool_dicts, tool_section=None ):
"""
When installing a repository that contains tools, all tools must currently be defined
within the same tool section in the tool panel or outside of any sections.
"""
tool_panel_dict = {}
if tool_section:
section_id = tool_section.id
section_name = tool_section.name
section_version = tool_section.version or ''
else:
section_id = ''
section_name = ''
section_version = ''
for tool_dict in tool_dicts:
if tool_dict.get( 'add_to_tool_panel', True ):
guid = tool_dict[ 'guid' ]
tool_config = tool_dict[ 'tool_config' ]
tool_section_dict = dict( tool_config=tool_config, id=section_id, name=section_name, version=section_version )
if guid in tool_panel_dict:
tool_panel_dict[ guid ].append( tool_section_dict )
else:
tool_panel_dict[ guid ] = [ tool_section_dict ]
return tool_panel_dict
def generate_tool_panel_dict_for_tool_config( guid, tool_config, tool_sections=None ):
"""
Create a dictionary of the following type for a single tool config file name.
The intent is to call this method for every tool config in a repository and
append each of these as entries to a tool panel dictionary for the repository.
This allows for each tool to be loaded into a different section in the tool panel.
{<Tool guid> :
[{ tool_config : <tool_config_file>,
id: <ToolSection id>,
version : <ToolSection version>,
name : <TooSection name>}]}
"""
tool_panel_dict = {}
file_name = basic_util.strip_path( tool_config )
tool_section_dicts = generate_tool_section_dicts( tool_config=file_name, tool_sections=tool_sections )
tool_panel_dict[ guid ] = tool_section_dicts
return tool_panel_dict
def generate_tool_panel_elem_list( repository_name, repository_clone_url, changeset_revision, tool_panel_dict,
repository_tools_tups, owner='' ):
"""Generate a list of ElementTree Element objects for each section or tool."""
elem_list = []
tool_elem = None
cleaned_repository_clone_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url )
if not owner:
owner = suc.get_repository_owner( cleaned_repository_clone_url )
tool_shed = cleaned_repository_clone_url.split( '/repos/' )[ 0 ].rstrip( '/' )
for guid, tool_section_dicts in tool_panel_dict.items():
for tool_section_dict in tool_section_dicts:
tool_section = None
inside_section = False
section_in_elem_list = False
if tool_section_dict[ 'id' ]:
inside_section = True
# Create a new section element only if we haven't already created it.
for index, elem in enumerate( elem_list ):
if elem.tag == 'section':
section_id = elem.get( 'id', None )
if section_id == tool_section_dict[ 'id' ]:
section_in_elem_list = True
tool_section = elem
break
if tool_section is None:
tool_section = generate_tool_section_element_from_dict( tool_section_dict )
# Find the tuple containing the current guid from the list of repository_tools_tups.
for repository_tool_tup in repository_tools_tups:
tool_file_path, tup_guid, tool = repository_tool_tup
if tup_guid == guid:
break
if inside_section:
tool_elem = suc.generate_tool_elem( tool_shed,
repository_name,
changeset_revision,
owner,
tool_file_path,
tool,
tool_section )
else:
tool_elem = suc.generate_tool_elem( tool_shed,
repository_name,
changeset_revision,
owner,
tool_file_path,
tool,
None )
if inside_section:
if section_in_elem_list:
elem_list[ index ] = tool_section
else:
elem_list.append( tool_section )
else:
elem_list.append( tool_elem )
return elem_list
def generate_tool_section_dicts( tool_config=None, tool_sections=None ):
tool_section_dicts = []
if tool_config is None:
tool_config = ''
if tool_sections:
for tool_section in tool_sections:
# The value of tool_section will be None if the tool is displayed outside
# of any sections in the tool panel.
if tool_section:
section_id = tool_section.id or ''
section_version = tool_section.version or ''
section_name = tool_section.name or ''
else:
section_id = ''
section_version = ''
section_name = ''
tool_section_dicts.append( dict( tool_config=tool_config,
id=section_id,
version=section_version,
name=section_name ) )
else:
tool_section_dicts.append( dict( tool_config=tool_config, id='', version='', name='' ) )
return tool_section_dicts
def generate_tool_section_element_from_dict( tool_section_dict ):
# The value of tool_section_dict looks like the following.
# { id: <ToolSection id>, version : <ToolSection version>, name : <TooSection name>}
if tool_section_dict[ 'id' ]:
# Create a new tool section.
tool_section = XmlET.Element( 'section' )
tool_section.attrib[ 'id' ] = tool_section_dict[ 'id' ]
tool_section.attrib[ 'name' ] = tool_section_dict[ 'name' ]
tool_section.attrib[ 'version' ] = tool_section_dict[ 'version' ]
else:
tool_section = None
return tool_section
def get_headers( fname, sep, count=60, is_multi_byte=False ):
"""Returns a list with the first 'count' lines split by 'sep'."""
headers = []
@@ -341,26 +142,6 @@ def get_headers( fname, sep, count=60, is_multi_byte=False ):
break
return headers
def get_or_create_tool_section( toolbox, tool_panel_section_id, new_tool_panel_section_label=None ):
tool_panel_section_key = str( tool_panel_section_id )
if tool_panel_section_key in toolbox.tool_panel:
# Appending a tool to an existing section in toolbox.tool_panel
tool_section = toolbox.tool_panel[ tool_panel_section_key ]
log.debug( "Appending to tool panel section: %s" % str( tool_section.name ) )
else:
# Appending a new section to toolbox.tool_panel
if new_tool_panel_section_label is None:
# This might add an ugly section label to the tool panel, but, oh well...
new_tool_panel_section_label = tool_panel_section_id
elem = XmlET.Element( 'section' )
elem.attrib[ 'name' ] = new_tool_panel_section_label
elem.attrib[ 'id' ] = tool_panel_section_id
elem.attrib[ 'version' ] = ''
tool_section = galaxy.tools.ToolSection( elem )
toolbox.tool_panel[ tool_panel_section_key ] = tool_section
log.debug( "Loading new tool panel section: %s" % str( tool_section.name ) )
return tool_panel_section_key, tool_section
def get_tool_path_install_dir( partial_install_dir, shed_tool_conf_dict, tool_dict, config_elems ):
for elem in config_elems:
if elem.tag == 'tool':
@@ -514,63 +295,6 @@ def handle_sample_tool_data_table_conf_file( app, filename, persist=False ):
error = True
return error, message
def handle_tool_panel_selection( toolbox, metadata, no_changes_checked, tool_panel_section_id, new_tool_panel_section_label ):
"""
Handle the selected tool panel location for loading tools included in tool shed repositories when installing
or reinstalling them.
"""
# Get the location in the tool panel in which each tool was originally loaded.
tool_section = None
tool_panel_section_key = None
if 'tools' in metadata:
# This forces everything to be loaded into the same section (or no section) in the tool panel.
if no_changes_checked:
# Make sure the no_changes check box overrides the new_tool_panel_section_label if the user checked the check
# box and entered something into the field.
new_tool_panel_section_label = None
if 'tool_panel_section' in metadata:
tool_panel_dict = metadata[ 'tool_panel_section' ]
if not tool_panel_dict:
tool_panel_dict = generate_tool_panel_dict_for_new_install( metadata[ 'tools' ] )
else:
tool_panel_dict = generate_tool_panel_dict_for_new_install( metadata[ 'tools' ] )
if tool_panel_dict:
# The tool_panel_dict is empty when tools exist but are not installed into a tool panel section.
tool_section_dicts = tool_panel_dict[ tool_panel_dict.keys()[ 0 ] ]
tool_section_dict = tool_section_dicts[ 0 ]
original_section_id = tool_section_dict[ 'id' ]
original_section_name = tool_section_dict[ 'name' ]
if original_section_id:
tool_panel_section_key, tool_section = \
get_or_create_tool_section( toolbox,
tool_panel_section_id=original_section_id,
new_tool_panel_section_label=new_tool_panel_section_label )
else:
# The user elected to change the tool panel section to contain the tools.
tool_panel_section_key, tool_section = handle_tool_panel_section( toolbox,
tool_panel_section_id=tool_panel_section_id,
new_tool_panel_section_label=new_tool_panel_section_label )
return tool_section, tool_panel_section_key
def handle_tool_panel_section( toolbox, tool_panel_section_id=None, new_tool_panel_section_label=None ):
"""Return a ToolSection object retrieved from the current in-memory tool_panel."""
# If tool_panel_section_id is received, the section exists in the tool panel. In this
# case, the value of the received tool_panel_section_id must be the id retrieved from a
# tool panel config (e.g., tool_conf.xml, which may have getext). If new_tool_panel_section_label
# is received, a new section will be added to the tool panel.
if new_tool_panel_section_label:
section_id = str( new_tool_panel_section_label.lower().replace( ' ', '_' ) )
tool_panel_section_key, tool_section = \
get_or_create_tool_section( toolbox,
tool_panel_section_id=section_id,
new_tool_panel_section_label=new_tool_panel_section_label )
elif tool_panel_section_id:
tool_panel_section_key = str( tool_panel_section_id )
tool_section = toolbox.tool_panel[ tool_panel_section_key ]
else:
return None, None
return tool_panel_section_key, tool_section
def handle_tool_versions( app, tool_version_dicts, tool_shed_repository ):
"""
Using the list of tool_version_dicts retrieved from the tool shed (one per changeset
@@ -741,170 +465,6 @@ def reload_upload_tools( app ):
if isinstance( tool.tool_action, UploadToolAction ):
app.toolbox.reload_tool_by_id( tool_id )
def remove_from_shed_tool_config( app, shed_tool_conf_dict, guids_to_remove ):
# A tool shed repository is being uninstalled so change the shed_tool_conf file.
# Parse the config file to generate the entire list of config_elems instead of
# using the in-memory list since it will be a subset of the entire list if one
# or more repositories have been deactivated.
shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ]
tool_path = shed_tool_conf_dict[ 'tool_path' ]
config_elems = []
tree, error_message = xml_util.parse_xml( shed_tool_conf )
if tree:
root = tree.getroot()
for elem in root:
config_elems.append( elem )
config_elems_to_remove = []
for config_elem in config_elems:
if config_elem.tag == 'section':
tool_elems_to_remove = []
for tool_elem in config_elem:
if tool_elem.get( 'guid' ) in guids_to_remove:
tool_elems_to_remove.append( tool_elem )
for tool_elem in tool_elems_to_remove:
# Remove all of the appropriate tool sub-elements from the section element.
config_elem.remove( tool_elem )
if len( config_elem ) < 1:
# Keep a list of all empty section elements so they can be removed.
config_elems_to_remove.append( config_elem )
elif config_elem.tag == 'tool':
if config_elem.get( 'guid' ) in guids_to_remove:
config_elems_to_remove.append( config_elem )
for config_elem in config_elems_to_remove:
config_elems.remove( config_elem )
# Persist the altered in-memory version of the tool config.
suc.config_elems_to_xml_file( app, config_elems, shed_tool_conf, tool_path )
def remove_from_tool_panel( app, repository, shed_tool_conf, uninstall ):
"""
A tool shed repository is being deactivated or uninstalled, so handle tool panel
alterations accordingly.
"""
# Determine where the tools are currently defined in the tool panel and store this
# information so the tools can be displayed in the same way when the repository is
# activated or reinstalled.
tool_panel_dict = suc.generate_tool_panel_dict_from_shed_tool_conf_entries( app, repository )
repository.metadata[ 'tool_panel_section' ] = tool_panel_dict
app.install_model.context.add( repository )
app.install_model.context.flush()
# Create a list of guids for all tools that will be removed from the in-memory tool panel
# and config file on disk.
guids_to_remove = [ k for k in tool_panel_dict.keys() ]
# Remove the tools from the toolbox's tools_by_id dictionary.
for guid_to_remove in guids_to_remove:
if guid_to_remove in app.toolbox.tools_by_id:
del app.toolbox.tools_by_id[ guid_to_remove ]
index, shed_tool_conf_dict = suc.get_shed_tool_conf_dict( app, shed_tool_conf )
if uninstall:
# Remove from the shed_tool_conf file on disk.
remove_from_shed_tool_config( app, shed_tool_conf_dict, guids_to_remove )
config_elems = shed_tool_conf_dict[ 'config_elems' ]
config_elems_to_remove = []
for config_elem in config_elems:
if config_elem.tag == 'section':
# Get the section key for the in-memory tool panel.
section_key = str( config_elem.get( "id" ) )
# Generate the list of tool elements to remove.
tool_elems_to_remove = []
for tool_elem in config_elem:
if tool_elem.get( 'guid' ) in guids_to_remove:
tool_elems_to_remove.append( tool_elem )
for tool_elem in tool_elems_to_remove:
if tool_elem in config_elem:
# Remove the tool sub-element from the section element.
config_elem.remove( tool_elem )
# Remove the tool from the section in the in-memory tool panel.
if section_key in app.toolbox.tool_panel:
tool_section = app.toolbox.tool_panel[ section_key ]
guid = tool_elem.get( 'guid' )
tool_key = 'tool_%s' % str( guid )
# Get the list of versions of this tool that are currently available in the toolbox.
available_tool_versions = app.toolbox.get_loaded_tools_by_lineage( guid )
if tool_key in tool_section.elems:
if available_tool_versions:
available_tool_versions.reverse()
replacement_tool_key = None
replacement_tool_version = None
# Since we are going to remove the tool from the section, replace it with the
# newest loaded version of the tool.
for available_tool_version in available_tool_versions:
available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section()
if available_tool_version.id in tool_section.elems.keys() or section_key == available_tool_section_id:
replacement_tool_key = 'tool_%s' % str( available_tool_version.id )
replacement_tool_version = available_tool_version
break
if replacement_tool_key and replacement_tool_version:
# Get the index of the tool_key in the tool_section.
for tool_section_elems_index, key in enumerate( tool_section.elems.keys() ):
if key == tool_key:
break
# Remove the tool from the tool section.
del tool_section.elems[ tool_key ]
# Add the replacement tool at the same location in the tool section.
tool_section.elems.insert( tool_section_elems_index,
replacement_tool_key,
replacement_tool_version )
else:
del tool_section.elems[ tool_key ]
else:
del tool_section.elems[ tool_key ]
if uninstall:
# Remove the tool from the section in the in-memory integrated tool panel.
if section_key in app.toolbox.integrated_tool_panel:
tool_section = app.toolbox.integrated_tool_panel[ section_key ]
tool_key = 'tool_%s' % str( tool_elem.get( 'guid' ) )
if tool_key in tool_section.elems:
del tool_section.elems[ tool_key ]
if len( config_elem ) < 1:
# Keep a list of all empty section elements so they can be removed.
config_elems_to_remove.append( config_elem )
elif config_elem.tag == 'tool':
guid = config_elem.get( 'guid' )
if guid in guids_to_remove:
tool_key = 'tool_%s' % str( config_elem.get( 'guid' ) )
# Get the list of versions of this tool that are currently available in the toolbox.
available_tool_versions = app.toolbox.get_loaded_tools_by_lineage( guid )
if tool_key in app.toolbox.tool_panel:
if available_tool_versions:
available_tool_versions.reverse()
replacement_tool_key = None
replacement_tool_version = None
# Since we are going to remove the tool from the section, replace it with
# the newest loaded version of the tool.
for available_tool_version in available_tool_versions:
available_tool_section_id, available_tool_section_name = available_tool_version.get_panel_section()
if available_tool_version.id in app.toolbox.tool_panel.keys() or not available_tool_section_id:
replacement_tool_key = 'tool_%s' % str( available_tool_version.id )
replacement_tool_version = available_tool_version
break
if replacement_tool_key and replacement_tool_version:
# Get the index of the tool_key in the tool_section.
for tool_panel_index, key in enumerate( app.toolbox.tool_panel.keys() ):
if key == tool_key:
break
# Remove the tool from the tool panel.
del app.toolbox.tool_panel[ tool_key ]
# Add the replacement tool at the same location in the tool panel.
app.toolbox.tool_panel.insert( tool_panel_index, replacement_tool_key, replacement_tool_version )
else:
del app.toolbox.tool_panel[ tool_key ]
else:
del app.toolbox.tool_panel[ tool_key ]
if uninstall:
if tool_key in app.toolbox.integrated_tool_panel:
del app.toolbox.integrated_tool_panel[ tool_key ]
config_elems_to_remove.append( config_elem )
for config_elem in config_elems_to_remove:
# Remove the element from the in-memory list of elements.
config_elems.remove( config_elem )
# Update the config_elems of the in-memory shed_tool_conf_dict.
shed_tool_conf_dict[ 'config_elems' ] = config_elems
app.toolbox.shed_tool_confs[ index ] = shed_tool_conf_dict
app.toolbox_search = ToolBoxSearch( app.toolbox )
if uninstall and app.config.update_integrated_tool_panel:
# Write the current in-memory version of the integrated_tool_panel.xml file to disk.
app.toolbox.write_integrated_tool_panel_config_file()
def reset_tool_data_tables( app ):
# Reset the tool_data_tables to an empty dictionary.
app.tool_data_tables.data_tables = {}