diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 967b81e7389..bd7ccafd59e 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -55,6 +55,7 @@ from galaxy.web.form_builder import SelectField from galaxy.util.dictifiable import Dictifiable from galaxy.work.context import WorkRequestContext from tool_shed.util import common_util +import tool_shed.util.repository_util as repository_util from tool_shed.util import shed_util_common as suc from .loader import template_macro_params, raw_tool_xml_tree, imported_macro_paths @@ -164,7 +165,7 @@ class ToolBox( BaseGalaxyToolBox ): # Abstract toolbox doesn't have a dependency on the the database, so # override _get_tool_shed_repository here to provide this information. - return suc.get_installed_repository( + return repository_util.get_installed_repository( self.app, tool_shed=tool_shed, name=name, @@ -323,11 +324,11 @@ class Tool( object, Dictifiable ): def tool_shed_repository( self ): # If this tool is included in an installed tool shed repository, return it. if self.tool_shed: - return suc.get_installed_repository( self.app, - tool_shed=self.tool_shed, - name=self.repository_name, - owner=self.repository_owner, - installed_changeset_revision=self.installed_changeset_revision ) + return repository_util.get_installed_repository( self.app, + tool_shed=self.tool_shed, + name=self.repository_name, + owner=self.repository_owner, + installed_changeset_revision=self.installed_changeset_revision ) return None @property diff --git a/lib/galaxy/tools/data_manager/manager.py b/lib/galaxy/tools/data_manager/manager.py index b9035cd8947..086065beca0 100644 --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -8,7 +8,7 @@ from galaxy.util.odict import odict from galaxy.util.template import fill_template from galaxy.tools.data import TabularToolDataTable from tool_shed.util import common_util -import tool_shed.util.shed_util_common as suc +from tool_shed.util import repository_util import galaxy.queue_worker # set up logger @@ -150,11 +150,11 @@ class DataManager( object ): owner=repository_owner, installed_changeset_revision=installed_changeset_revision ) tool_shed_repository = \ - suc.get_installed_repository( self.data_managers.app, - tool_shed=tool_shed, - name=repository_name, - owner=repository_owner, - installed_changeset_revision=installed_changeset_revision ) + repository_util.get_installed_repository( self.data_managers.app, + tool_shed=tool_shed, + name=repository_name, + owner=repository_owner, + installed_changeset_revision=installed_changeset_revision ) if tool_shed_repository is None: log.warning( 'Could not determine tool shed repository from database. This should only ever happen when running tests.' ) # we'll set tool_path manually here from shed_conf_file diff --git a/lib/galaxy/web/base/controllers/admin.py b/lib/galaxy/web/base/controllers/admin.py index 52a692df4b5..e2fdf0ecfe9 100644 --- a/lib/galaxy/web/base/controllers/admin.py +++ b/lib/galaxy/web/base/controllers/admin.py @@ -10,7 +10,7 @@ import galaxy.queue_worker from galaxy import util, web from galaxy.util import inflector from galaxy.web.form_builder import CheckboxField -from tool_shed.util import shed_util_common as suc +from tool_shed.util import repository_util from tool_shed.util.web_util import escape log = logging.getLogger( __name__ ) @@ -35,7 +35,7 @@ class Admin( object ): status = kwd.get( 'status', 'done' ) if trans.webapp.name == 'galaxy': is_repo_installed = trans.install_model.context.query( trans.install_model.ToolShedRepository ).first() is not None - installing_repository_ids = suc.get_ids_of_tool_shed_repositories_being_installed( trans.app, as_string=True ) + installing_repository_ids = repository_util.get_ids_of_tool_shed_repositories_being_installed( trans.app, as_string=True ) return trans.fill_template( '/webapps/galaxy/admin/index.mako', is_repo_installed=is_repo_installed, installing_repository_ids=installing_repository_ids, @@ -53,7 +53,7 @@ class Admin( object ): status = kwd.get( 'status', 'done' ) if trans.webapp.name == 'galaxy': is_repo_installed = trans.install_model.context.query( trans.install_model.ToolShedRepository ).first() is not None - installing_repository_ids = suc.get_ids_of_tool_shed_repositories_being_installed( trans.app, as_string=True ) + installing_repository_ids = repository_util.get_ids_of_tool_shed_repositories_being_installed( trans.app, as_string=True ) return trans.fill_template( '/webapps/galaxy/admin/center.mako', is_repo_installed=is_repo_installed, installing_repository_ids=installing_repository_ids, diff --git a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py index fdf890deb50..8222ba308bf 100644 --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -75,7 +75,7 @@ class ToolShedRepositoriesController( BaseAPIController ): # Since exported workflows are dictionaries with very few attributes that differentiate them from each # other, we'll build the list based on the following dictionary of those few attributes. exported_workflows = [] - repository = suc.get_tool_shed_repository_by_id( self.app, id ) + repository = repository_util.get_tool_shed_repository_by_id( self.app, id ) metadata = repository.metadata if metadata: exported_workflow_tups = metadata.get( 'workflows', [] ) @@ -172,7 +172,7 @@ class ToolShedRepositoriesController( BaseAPIController ): index = payload.get( 'index', None ) if index is None: raise HTTPBadRequest( detail="Missing required parameter 'index'." ) - repository = suc.get_tool_shed_repository_by_id( self.app, tool_shed_repository_id ) + repository = repository_util.get_tool_shed_repository_by_id( self.app, tool_shed_repository_id ) exported_workflows = json.loads( self.exported_workflows( trans, tool_shed_repository_id ) ) # Since we don't have an in-memory object with an id, we'll identify the exported workflow via its location (i.e., index) in the list. exported_workflow = exported_workflows[ int( index ) ] @@ -199,7 +199,7 @@ class ToolShedRepositoriesController( BaseAPIController ): tool_shed_repository_id = kwd.get( 'id', '' ) if not tool_shed_repository_id: raise HTTPBadRequest( detail="Missing required parameter 'id'." ) - repository = suc.get_tool_shed_repository_by_id( self.app, tool_shed_repository_id ) + repository = repository_util.get_tool_shed_repository_by_id( self.app, tool_shed_repository_id ) exported_workflows = json.loads( self.exported_workflows( trans, tool_shed_repository_id ) ) imported_workflow_dicts = [] for exported_workflow_dict in exported_workflows: @@ -434,11 +434,11 @@ class ToolShedRepositoriesController( BaseAPIController ): # Get the information about the repository to be installed from the payload. tool_shed_url, name, owner, changeset_revision = self.__parse_repository_from_payload( payload, include_changeset=True ) tool_shed_repositories = [] - tool_shed_repository = suc.get_installed_repository( self.app, - tool_shed=tool_shed_url, - name=name, - owner=owner, - changeset_revision=changeset_revision ) + tool_shed_repository = repository_util.get_installed_repository( self.app, + tool_shed=tool_shed_url, + name=name, + owner=owner, + changeset_revision=changeset_revision ) rrm = RepairRepositoryManager( self.app ) repair_dict = rrm.get_repair_dict( tool_shed_repository ) ordered_tsr_ids = repair_dict.get( 'ordered_tsr_ids', [] ) @@ -535,7 +535,7 @@ class ToolShedRepositoriesController( BaseAPIController ): :param id: the encoded id of the ToolShedRepository object """ # Example URL: http://localhost:8763/api/tool_shed_repositories/df7a1f0c02a5b08e - tool_shed_repository = suc.get_tool_shed_repository_by_id( self.app, id ) + tool_shed_repository = repository_util.get_tool_shed_repository_by_id( self.app, id ) if tool_shed_repository is None: log.debug( "Unable to locate tool_shed_repository record for id %s." % ( str( id ) ) ) return {} @@ -555,7 +555,7 @@ class ToolShedRepositoriesController( BaseAPIController ): :param id: the repository's encoded id """ - tool_shed_repository = suc.get_tool_shed_repository_by_id( self.app, id ) + tool_shed_repository = repository_util.get_tool_shed_repository_by_id( self.app, id ) if tool_shed_repository is None: log.debug( "Unable to locate tool_shed_repository record for id %s." % ( str( id ) ) ) return {} diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 7b3b4ceb0ab..144c78b071e 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -394,11 +394,11 @@ class AdminToolshed( AdminGalaxy ): repository_id = kwd.get( 'repository_id', None ) relative_path_to_image_file = kwd.get( 'image_file', None ) if repository_id and relative_path_to_image_file: - repository = suc.get_tool_shed_repository_by_id( trans.app, repository_id ) + repository = repository_util.get_tool_shed_repository_by_id( trans.app, repository_id ) if repository: repo_files_dir = repository.repo_files_directory( trans.app ) # The following line sometimes returns None. TODO: Figure out why. - path_to_file = suc.get_absolute_path_to_file_in_repository( repo_files_dir, relative_path_to_image_file ) + path_to_file = repository_util.get_absolute_path_to_file_in_repository( repo_files_dir, relative_path_to_image_file ) if path_to_file and os.path.exists( path_to_file ): file_name = os.path.basename( relative_path_to_image_file ) try: @@ -500,7 +500,7 @@ class AdminToolshed( AdminGalaxy ): status = kwd.get( 'status', 'done' ) if workflow_name: workflow_name = encoding_util.tool_shed_decode( workflow_name ) - repository = suc.get_tool_shed_repository_by_id( trans.app, repository_id ) + repository = repository_util.get_tool_shed_repository_by_id( trans.app, repository_id ) if repository: workflow, status, message = workflow_util.import_workflow( trans, repository, workflow_name ) if workflow: @@ -593,7 +593,7 @@ class AdminToolshed( AdminGalaxy ): tmp_clone_url = util.build_url( tool_shed_url, pathspec=[ 'repos', owner, name ] ) tmp_repo_info_tuple = ( None, tmp_clone_url, latest_downloadable_revision, None, owner, None, None ) installed_repository, installed_changeset_revision = \ - suc.repository_was_previously_installed( trans.app, tool_shed_url, name, tmp_repo_info_tuple, from_tip=False ) + repository_util.repository_was_previously_installed( trans.app, tool_shed_url, name, tmp_repo_info_tuple, from_tip=False ) if installed_repository: current_changeset_revision = str( installed_repository.changeset_revision ) message = 'Revision %s of repository %s owned by %s has already been installed.' % \ @@ -829,7 +829,7 @@ class AdminToolshed( AdminGalaxy ): # The user must be on the manage_repository_tool_dependencies page and clicked the button to either install or uninstall a # tool dependency, but they didn't check any of the available tool dependencies on which to perform the action. repository_id = kwd.get( 'repository_id', None ) - tool_shed_repository = suc.get_tool_shed_repository_by_id( trans.app, repository_id ) + tool_shed_repository = repository_util.get_tool_shed_repository_by_id( trans.app, repository_id ) if 'operation' in kwd: operation = kwd[ 'operation' ].lower() if not tool_dependency_ids: @@ -911,7 +911,7 @@ class AdminToolshed( AdminGalaxy ): else: # The user must be on the manage_repository_tool_dependencies page and clicked the button to either install or uninstall a # tool dependency, but they didn't check any of the available tool dependencies on which to perform the action. - tool_shed_repository = suc.get_tool_shed_repository_by_id( trans.app, repository_id ) + tool_shed_repository = repository_util.get_tool_shed_repository_by_id( trans.app, repository_id ) self.tool_dependency_grid.title = "Tool shed repository '%s' tool dependencies" % escape( tool_shed_repository.name ) if 'operation' in kwd: operation = kwd[ 'operation' ].lower() @@ -963,7 +963,7 @@ class AdminToolshed( AdminGalaxy ): def monitor_repository_installation( self, trans, **kwd ): tsridslist = common_util.get_tool_shed_repository_ids( **kwd ) if not tsridslist: - tsridslist = suc.get_ids_of_tool_shed_repositories_being_installed( trans.app, as_string=False ) + tsridslist = repository_util.get_ids_of_tool_shed_repositories_being_installed( trans.app, as_string=False ) kwd[ 'tool_shed_repository_ids' ] = tsridslist return self.repository_installation_grid( trans, **kwd ) @@ -1029,7 +1029,7 @@ class AdminToolshed( AdminGalaxy ): # but the received repository id is from the Galaxy side (the caller is this controller's # update_to_changeset_revision() method. We need to get the id of the same repository from the # Tool Shed side. - repository = suc.get_tool_shed_repository_by_id( trans.app, updating_repository_id ) + repository = repository_util.get_tool_shed_repository_by_id( trans.app, updating_repository_id ) # For backward compatibility to the 12/20/12 Galaxy release. try: params = dict( name=str( repository.name ), owner=str( repository.owner ) ) @@ -1074,7 +1074,7 @@ class AdminToolshed( AdminGalaxy ): if ( ( not includes_tools_for_display_in_tool_panel and kwd.get( 'select_shed_tool_panel_config_button', False ) ) or ( includes_tools_for_display_in_tool_panel and kwd.get( 'select_tool_panel_section_button', False ) ) ): if updating: - repository = suc.get_tool_shed_repository_by_id( trans.app, updating_repository_id ) + repository = repository_util.get_tool_shed_repository_by_id( trans.app, updating_repository_id ) decoded_updated_metadata = encoding_util.tool_shed_decode( encoded_updated_metadata ) # Now that the user has decided whether they will handle dependencies, we can update # the repository to the latest revision. @@ -1379,17 +1379,17 @@ class AdminToolshed( AdminGalaxy ): suc.clean_dependency_relationships(trans, new_meta, tool_shed_repository, tool_shed_url) # The repository's status must be updated from 'Uninstalled' to 'New' when initiating reinstall # so the repository_installation_updater will function. - tool_shed_repository = suc.create_or_update_tool_shed_repository( trans.app, - tool_shed_repository.name, - tool_shed_repository.description, - tool_shed_repository.installed_changeset_revision, - tool_shed_repository.ctx_rev, - repository_clone_url, - metadata, - trans.install_model.ToolShedRepository.installation_status.NEW, - tool_shed_repository.changeset_revision, - tool_shed_repository.owner, - tool_shed_repository.dist_to_shed ) + tool_shed_repository = repository_util.create_or_update_tool_shed_repository( trans.app, + tool_shed_repository.name, + tool_shed_repository.description, + tool_shed_repository.installed_changeset_revision, + tool_shed_repository.ctx_rev, + repository_clone_url, + metadata, + trans.install_model.ToolShedRepository.installation_status.NEW, + tool_shed_repository.changeset_revision, + tool_shed_repository.owner, + tool_shed_repository.dist_to_shed ) ctx_rev = suc.get_ctx_rev( trans.app, tool_shed_url, tool_shed_repository.name, @@ -1775,13 +1775,13 @@ class AdminToolshed( AdminGalaxy ): """An error occurred while cloning the repository, so reset everything necessary to enable another attempt.""" repository = repository_util.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] ) if kwd.get( 'reset_repository', False ): - suc.set_repository_attributes( trans.app, - repository, - status=trans.install_model.ToolShedRepository.installation_status.NEW, - error_message=None, - deleted=False, - uninstalled=False, - remove_from_disk=True ) + repository_util.set_repository_attributes( trans.app, + repository, + status=trans.install_model.ToolShedRepository.installation_status.NEW, + error_message=None, + deleted=False, + uninstalled=False, + remove_from_disk=True ) new_kwd = {} new_kwd[ 'message' ] = "You can now attempt to install the repository named %s again." % escape( str( repository.name ) ) new_kwd[ 'status' ] = "done" @@ -1910,11 +1910,11 @@ class AdminToolshed( AdminGalaxy ): changeset_revision = kwd.get( 'changeset_revision', None ) latest_changeset_revision = kwd.get( 'latest_changeset_revision', None ) latest_ctx_rev = kwd.get( 'latest_ctx_rev', None ) - repository = suc.get_installed_repository( trans.app, - tool_shed=tool_shed_url, - name=name, - owner=owner, - changeset_revision=changeset_revision ) + repository = repository_util.get_installed_repository( trans.app, + tool_shed=tool_shed_url, + name=name, + owner=owner, + changeset_revision=changeset_revision ) original_metadata_dict = repository.metadata original_repository_dependencies_dict = original_metadata_dict.get( 'repository_dependencies', {} ) original_repository_dependencies = original_repository_dependencies_dict.get( 'repository_dependencies', [] ) @@ -2001,11 +2001,11 @@ class AdminToolshed( AdminGalaxy ): # we set from_tip to True to make sure we get the entire set of changeset revisions # from the Tool Shed. new_repository_db_record, installed_changeset_revision = \ - suc.repository_was_previously_installed( trans.app, - tool_shed_url, - new_name, - repo_info_tuple, - from_tip=True ) + repository_util.repository_was_previously_installed( trans.app, + tool_shed_url, + new_name, + repo_info_tuple, + from_tip=True ) if new_repository_db_record: if new_repository_db_record.status in [ trans.install_model.ToolShedRepository.installation_status.ERROR, trans.install_model.ToolShedRepository.installation_status.NEW, @@ -2120,7 +2120,7 @@ class AdminToolshed( AdminGalaxy ): status = kwd.get( 'status', 'done' ) if workflow_name: workflow_name = encoding_util.tool_shed_decode( workflow_name ) - repository = suc.get_tool_shed_repository_by_id( trans.app, repository_id ) + repository = repository_util.get_tool_shed_repository_by_id( trans.app, repository_id ) changeset_revision = repository.changeset_revision metadata = repository.metadata return trans.fill_template( "/admin/tool_shed_repository/view_workflow.mako", diff --git a/lib/galaxy/webapps/tool_shed/api/categories.py b/lib/galaxy/webapps/tool_shed/api/categories.py index 1fc1b0f2fe1..4b5ab860fc7 100644 --- a/lib/galaxy/webapps/tool_shed/api/categories.py +++ b/lib/galaxy/webapps/tool_shed/api/categories.py @@ -7,6 +7,7 @@ from galaxy.web import require_admin as require_admin from galaxy.web import _future_expose_api as expose_api from galaxy.web import _future_expose_api_anonymous_and_sessionless as expose_api_anonymous_and_sessionless from galaxy.web.base.controller import BaseAPIController +from tool_shed.util import repository_util import tool_shed.util.shed_util_common as suc log = logging.getLogger( __name__ ) @@ -81,7 +82,7 @@ class CategoriesController( BaseAPIController ): category_dict[ 'url' ] = web.url_for( controller='categories', action='show', id=trans.security.encode_id( category.id ) ) - repositories = suc.get_repositories_by_category( self.app, category.id ) + repositories = repository_util.get_repositories_by_category( self.app, category.id ) category_dict[ 'repositories' ] = repositories return category_dict diff --git a/lib/galaxy/webapps/tool_shed/api/repositories.py b/lib/galaxy/webapps/tool_shed/api/repositories.py index cd41e335ed8..d671181fb0e 100644 --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -36,7 +36,6 @@ from tool_shed.util import hg_util from tool_shed.util import metadata_util from tool_shed.util import repository_content_util from tool_shed.util import repository_util -from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_util log = logging.getLogger( __name__ ) @@ -73,7 +72,7 @@ class RepositoriesController( BaseAPIController ): owner = payload.get( 'owner', '' ) if not owner: raise HTTPBadRequest( detail="Missing required parameter 'owner'." ) - repository = suc.get_repository_by_name_and_owner( self.app, name, owner ) + repository = repository_util.get_repository_by_name_and_owner( self.app, name, owner ) if repository is None: error_message = 'Cannot locate repository with name %s and owner %s,' % ( str( name ), str( owner ) ) log.debug( error_message ) @@ -106,12 +105,12 @@ class RepositoriesController( BaseAPIController ): tsr_id = kwd.get( 'tsr_id', None ) if None not in [ name, owner ]: # Get the repository information. - repository = suc.get_repository_by_name_and_owner( self.app, name, owner ) + repository = repository_util.get_repository_by_name_and_owner( self.app, name, owner ) if repository is None: trans.response.status = 404 return { 'status': 'error', 'message': 'No repository named %s found with owner %s' % ( name, owner ) } elif tsr_id is not None: - repository = suc.get_repository_in_tool_shed( self.app, tsr_id ) + repository = repository_util.get_repository_in_tool_shed( self.app, tsr_id ) else: error_message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: " error_message += "invalid parameters received." @@ -190,7 +189,7 @@ class RepositoriesController( BaseAPIController ): # http:///api/repositories/get_repository_revision_install_info?name=&owner=&changeset_revision= if name and owner and changeset_revision: # Get the repository information. - repository = suc.get_repository_by_name_and_owner( self.app, name, owner ) + repository = repository_util.get_repository_by_name_and_owner( self.app, name, owner ) if repository is None: log.debug( 'Cannot locate repository %s owned by %s' % ( str( name ), str( owner ) ) ) return {}, {}, {} @@ -201,17 +200,17 @@ class RepositoriesController( BaseAPIController ): action='show', id=encoded_repository_id ) # Get the repository_metadata information. - repository_metadata = suc.get_repository_metadata_by_changeset_revision( self.app, - encoded_repository_id, - changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( self.app, + encoded_repository_id, + changeset_revision ) if repository_metadata is None: # The changeset_revision column in the repository_metadata table has been updated with a new # value value, so find the changeset_revision to which we need to update. repo = hg_util.get_repo_for_repository( self.app, repository=repository, repo_path=None, create=False ) - new_changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( self.app, - encoded_repository_id, - new_changeset_revision ) + new_changeset_revision = metadata_util.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( self.app, + encoded_repository_id, + new_changeset_revision ) changeset_revision = new_changeset_revision if repository_metadata is not None: encoded_repository_metadata_id = trans.security.encode_id( repository_metadata.id ) @@ -257,7 +256,7 @@ class RepositoriesController( BaseAPIController ): # Example URL: http://localhost:9009/api/repositories/get_installable_revisions?tsr_id=9d37e53072ff9fa4 tsr_id = kwd.get( 'tsr_id', None ) if tsr_id is not None: - repository = suc.get_repository_in_tool_shed( self.app, tsr_id ) + repository = repository_util.get_repository_in_tool_shed( self.app, tsr_id ) else: error_message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: " error_message += "missing or invalid parameter received." @@ -494,7 +493,7 @@ class RepositoriesController( BaseAPIController ): owner = payload.get( 'owner', '' ) if not owner: raise HTTPBadRequest( detail="Missing required parameter 'owner'." ) - repository = suc.get_repository_by_name_and_owner( self.app, name, owner ) + repository = repository_util.get_repository_by_name_and_owner( self.app, name, owner ) if repository is None: error_message = 'Cannot locate repository with name %s and owner %s,' % ( str( name ), str( owner ) ) log.debug( error_message ) @@ -679,7 +678,7 @@ class RepositoriesController( BaseAPIController ): repository_id = payload.get( 'repository_id', None ) if repository_id is not None: - repository = suc.get_repository_in_tool_shed( self.app, repository_id ) + repository = repository_util.get_repository_in_tool_shed( self.app, repository_id ) start_time = strftime( "%Y-%m-%d %H:%M:%S" ) log.debug( "%s...resetting metadata on repository %s" % ( start_time, str( repository.name ) ) ) results = handle_repository( trans, start_time, repository ) @@ -708,7 +707,7 @@ class RepositoriesController( BaseAPIController ): except Exception: raise MalformedId( 'The given id is invalid.' ) - repository = suc.get_repository_in_tool_shed( self.app, id ) + repository = repository_util.get_repository_in_tool_shed( self.app, id ) if repository is None: raise ObjectNotFound( 'Unable to locate repository for the given id.' ) repository_dict = repository.to_dict( view='element', @@ -720,9 +719,9 @@ class RepositoriesController( BaseAPIController ): @expose_api_anonymous_and_sessionless def show_tools( self, trans, id, changeset, **kwd ): - repository_metadata = suc.get_repository_metadata_by_changeset_revision( self.app, - id, - changeset ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( self.app, + id, + changeset ) if repository_metadata is not None: encoded_repository_metadata_id = trans.security.encode_id( repository_metadata.id ) repository_metadata_dict = repository_metadata.to_dict( view='collection', @@ -770,9 +769,9 @@ class RepositoriesController( BaseAPIController ): raise MalformedId( 'The given id is invalid.' ) recursive = util.asbool( kwd.get( 'recursive', 'True' ) ) all_metadata = {} - repository = suc.get_repository_in_tool_shed( self.app, id ) + repository = repository_util.get_repository_in_tool_shed( self.app, id ) for changeset, changehash in repository.installable_revisions( self.app ): - metadata = suc.get_current_repository_metadata_for_changeset_revision( self.app, repository, changehash ) + metadata = metadata_util.get_current_repository_metadata_for_changeset_revision( self.app, repository, changehash ) if metadata is None: continue metadata_dict = metadata.to_dict( value_mapper={ 'id': self.app.security.encode_id, 'repository_id': self.app.security.encode_id } ) @@ -931,7 +930,7 @@ class RepositoriesController( BaseAPIController ): rdah = attribute_handlers.RepositoryDependencyAttributeHandler( self.app, unpopulate=False ) tdah = attribute_handlers.ToolDependencyAttributeHandler( self.app, unpopulate=False ) - repository = suc.get_repository_in_tool_shed( self.app, id ) + repository = repository_util.get_repository_in_tool_shed( self.app, id ) if not ( trans.user_is_admin() or self.app.security_agent.user_can_administer_repository( trans.user, repository ) or diff --git a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py index 21d9f2a7390..fb0428ff66d 100644 --- a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py +++ b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py @@ -2,13 +2,13 @@ import logging from sqlalchemy import and_ -import tool_shed.util.shed_util_common as suc from galaxy import util from galaxy import web from galaxy.web.base.controller import BaseAPIController, HTTPBadRequest from tool_shed.capsule import capsule_manager from tool_shed.util import hg_util from tool_shed.util import metadata_util +from tool_shed.util import repository_util log = logging.getLogger( __name__ ) @@ -47,7 +47,7 @@ class RepositoryRevisionsController( BaseAPIController ): # We'll currently support only gzip-compressed tar archives. export_repository_dependencies = util.asbool( export_repository_dependencies ) # Get the repository information. - repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + repository = repository_util.get_repository_by_name_and_owner( trans.app, name, owner ) if repository is None: error_message = 'Cannot locate repository with name %s and owner %s,' % ( str( name ), str( owner ) ) log.debug( error_message ) @@ -131,13 +131,13 @@ class RepositoryRevisionsController( BaseAPIController ): rd_tups = metadata[ 'repository_dependencies' ][ 'repository_dependencies' ] for rd_tup in rd_tups: tool_shed, name, owner, changeset_revision = rd_tup[ 0:4 ] - repository_dependency = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + repository_dependency = repository_util.get_repository_by_name_and_owner( trans.app, name, owner ) if repository_dependency is None: log.dbug( 'Cannot locate repository dependency %s owned by %s.' % ( name, owner ) ) continue repository_dependency_id = trans.security.encode_id( repository_dependency.id ) repository_dependency_repository_metadata = \ - suc.get_repository_metadata_by_changeset_revision( trans.app, repository_dependency_id, changeset_revision ) + metadata_util.get_repository_metadata_by_changeset_revision( trans.app, repository_dependency_id, changeset_revision ) if repository_dependency_repository_metadata is None: # The changeset_revision column in the repository_metadata table has been updated with a new # value value, so find the changeset_revision to which we need to update. @@ -145,14 +145,14 @@ class RepositoryRevisionsController( BaseAPIController ): repository=repository_dependency, repo_path=None, create=False ) - new_changeset_revision = suc.get_next_downloadable_changeset_revision( repository_dependency, - repo, - changeset_revision ) + new_changeset_revision = metadata_util.get_next_downloadable_changeset_revision( repository_dependency, + repo, + changeset_revision ) if new_changeset_revision != changeset_revision: repository_dependency_repository_metadata = \ - suc.get_repository_metadata_by_changeset_revision( trans.app, - repository_dependency_id, - new_changeset_revision ) + metadata_util.get_repository_metadata_by_changeset_revision( trans.app, + repository_dependency_id, + new_changeset_revision ) changeset_revision = new_changeset_revision else: decoded_repository_dependency_id = trans.security.decode_id( repository_dependency_id ) diff --git a/lib/galaxy/webapps/tool_shed/controllers/admin.py b/lib/galaxy/webapps/tool_shed/controllers/admin.py index dc7dcdbde91..06d1c755932 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/admin.py +++ b/lib/galaxy/webapps/tool_shed/controllers/admin.py @@ -56,7 +56,7 @@ class AdminController( BaseUIController, Admin ): # The received id is the repository id, so we need to get the id of the user # that uploaded the repository. repository_id = kwd.get( 'id', None ) - repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id ) kwd[ 'f-email' ] = repository.user.email elif operation == "repositories_by_category": # Eliminate the current filters if any exist. @@ -88,7 +88,7 @@ class AdminController( BaseUIController, Admin ): changeset_revision_str = 'changeset_revision_' if k.startswith( changeset_revision_str ): repository_id = trans.security.encode_id( int( k.lstrip( changeset_revision_str ) ) ) - repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id ) if repository.tip( trans.app ) != v: return trans.response.send_redirect( web.url_for( controller='repository', action='browse_repositories', @@ -164,7 +164,7 @@ class AdminController( BaseUIController, Admin ): count = 0 deleted_repositories = "" for repository_id in ids: - repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id ) if repository: if not repository.deleted: # Mark all installable repository_metadata records as not installable. @@ -376,7 +376,7 @@ class AdminController( BaseUIController, Admin ): count = 0 undeleted_repositories = "" for repository_id in ids: - repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id ) if repository: if repository.deleted: # Inspect all repository_metadata records to determine those that are installable, and mark diff --git a/lib/galaxy/webapps/tool_shed/controllers/hg.py b/lib/galaxy/webapps/tool_shed/controllers/hg.py index 4609e0a5ed1..7ac550d66ff 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/hg.py +++ b/lib/galaxy/webapps/tool_shed/controllers/hg.py @@ -3,7 +3,7 @@ from galaxy import web from galaxy.web.base.controller import BaseUIController from tool_shed.util.common_util import generate_clone_url_for_repository_in_tool_shed -from tool_shed.util.shed_util_common import get_repository_by_name_and_owner +from tool_shed.util.repository_util import get_repository_by_name_and_owner from tool_shed.util.hg_util import update_repository from tool_shed.metadata import repository_metadata_manager diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository.py b/lib/galaxy/webapps/tool_shed/controllers/repository.py index 45e581e74e8..a0d86ea3198 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -101,7 +101,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): # We'll try to get the desired encoded repository id to pass on. try: repository_name = kwd[ 'id' ] - repository = suc.get_repository_by_name( trans.app, repository_name ) + repository = repository_util.get_repository_by_name( trans.app, repository_name ) kwd[ 'id' ] = trans.security.encode_id( repository.id ) except: pass @@ -280,7 +280,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): # The received id is the repository id, so we need to get the id of the user that owns the repository. repository_id = kwd.get( 'id', None ) if repository_id: - repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id ) user_id = trans.security.encode_id( repository.user.id ) kwd[ 'user_id' ] = user_id else: @@ -348,7 +348,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): # The received id is the repository id, so we need to get the id of the user that owns the repository. repository_id = kwd.get( 'id', None ) if repository_id: - repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id ) user_id = trans.security.encode_id( repository.user.id ) user = suc.get_user( trans.app, user_id ) self.repositories_by_user_grid.title = "Repositories owned by %s" % user.username @@ -413,7 +413,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): message = escape( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) commit_message = escape( kwd.get( 'commit_message', 'Deleted selected files' ) ) - repository = suc.get_repository_in_tool_shed( trans.app, id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, id ) repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) # Update repository files for browsing. hg_util.update_repository( repo ) @@ -505,7 +505,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): # We'll try to get the desired encoded repository id to pass on. try: name = kwd[ 'id' ] - repository = suc.get_repository_by_name( trans.app, name ) + repository = repository_util.get_repository_by_name( trans.app, name ) kwd[ 'id' ] = trans.security.encode_id( repository.id ) except: pass @@ -543,7 +543,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): if 'operation' in kwd: operation = kwd[ 'operation' ].lower() if operation == "preview_tools_in_changeset": - repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id ) repository_metadata = metadata_util.get_latest_repository_metadata( trans.app, repository.id, downloadable=True ) latest_installable_changeset_revision = repository_metadata.changeset_revision return trans.response.send_redirect( web.url_for( controller='repository', @@ -586,7 +586,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): name = kwd.get( 'name', None ) owner = kwd.get( 'owner', None ) changeset_revision = kwd.get( 'changeset_revision', None ) - repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + repository = repository_util.get_repository_by_name_and_owner( trans.app, name, owner ) repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) # Default to the current changeset revision. update_to_ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) @@ -612,9 +612,9 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): return no_update # Return the same value for changeset_revision and latest_changeset_revision. else: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, - trans.security.encode_id( repository.id ), - changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, + trans.security.encode_id( repository.id ), + changeset_revision ) if repository_metadata: # If changeset_revision is in the repository_metadata table for this repository, there are no # additional updates. @@ -635,9 +635,9 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): latest_changeset_revision = changeset_hash break else: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, - trans.security.encode_id( repository.id ), - changeset_hash ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, + trans.security.encode_id( repository.id ), + changeset_hash ) if repository_metadata: # We found a RepositoryMetadata record. update_to_ctx = hg_util.get_changectx_for_changeset( repo, changeset_hash ) @@ -662,7 +662,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): def contact_owner( self, trans, id, **kwd ): message = escape( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) - repository = suc.get_repository_in_tool_shed( trans.app, id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, id ) metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans.app, id, repository.tip( trans.app ), @@ -688,7 +688,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): if operation == "include in docker image": repository_tups = [] for repository_id in repository_ids: - repository = suc.get_repository_by_id( trans.app, repository_id ) + repository = repository_util.get_repository_by_id( trans.app, repository_id ) repository_tups.append( ( str( repository.name ), str( repository.user.username ), str( repository.type ) ) ) @@ -709,7 +709,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): tool_shed_url = tool_shed_url = web.url_for( '/', qualified=True ) repository_string = '' for repository_id in repository_ids: - repository = suc.get_repository_by_id( trans.app, repository_id ) + repository = repository_util.get_repository_by_id( trans.app, repository_id ) template = basic_util.SELECTED_REPOSITORIES_TEMPLATE repository_template = \ string.Template( template ).safe_substitute( tool_shed_url=tool_shed_url, @@ -801,7 +801,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): message = escape( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) repository_id = kwd.get( 'id', None ) - repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id ) mark_deprecated = util.string_as_bool( kwd.get( 'mark_deprecated', False ) ) repository.deprecated = mark_deprecated trans.sa_session.add( repository ) @@ -835,10 +835,10 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): repository_id = kwd.get( 'repository_id', None ) relative_path_to_image_file = kwd.get( 'image_file', None ) if repository_id and relative_path_to_image_file: - repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id ) if repository: repo_files_dir = repository.repo_path( trans.app ) - path_to_file = suc.get_absolute_path_to_file_in_repository( repo_files_dir, relative_path_to_image_file ) + path_to_file = repository_util.get_absolute_path_to_file_in_repository( repo_files_dir, relative_path_to_image_file ) if os.path.exists( path_to_file ): file_name = os.path.basename( relative_path_to_image_file ) try: @@ -900,7 +900,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): """Download an archive of the repository files compressed as zip, gz or bz2.""" # FIXME: thgis will currently only download the repository tip, no matter which installable changeset_revision is being viewed. # This should be enhanced to use the export method below, which accounts for the currently viewed changeset_revision. - repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id ) # Allow hgweb to handle the download. This requires the tool shed # server account's .hgrc file to include the following setting: # [web] @@ -919,7 +919,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): message = escape( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) export_repository_dependencies = kwd.get( 'export_repository_dependencies', '' ) - repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id ) if kwd.get( 'export_repository_button', False ): # We'll currently support only gzip-compressed tar archives. export_repository_dependencies = CheckboxField.is_checked( export_repository_dependencies ) @@ -945,7 +945,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): repositories_archive_path, file_name = os.path.split( repositories_archive.name ) basic_util.remove_dir( repositories_archive_path ) return opened_archive - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) metadata = repository_metadata.metadata toolshed_base_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) # Initialize the repository dependency RelationBuilder. @@ -1007,7 +1007,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): # The received id is a RepositoryMetadata id, so we have to get the repository id. repository_metadata = metadata_util.get_repository_metadata_by_id( trans.app, item_id ) repository_id = trans.security.encode_id( repository_metadata.repository.id ) - repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id ) kwd[ 'id' ] = repository_id kwd[ 'changeset_revision' ] = repository_metadata.changeset_revision if trans.webapp.name == 'tool_shed' and ( is_admin or repository.user == trans.user ): @@ -1098,7 +1098,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): # The received id is a RepositoryMetadata id, so we have to get the repository id. repository_metadata = metadata_util.get_repository_metadata_by_id( trans.app, item_id ) repository_id = trans.security.encode_id( repository_metadata.repository.id ) - repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id ) kwd[ 'id' ] = repository_id kwd[ 'changeset_revision' ] = repository_metadata.changeset_revision if trans.webapp.name == 'tool_shed' and ( is_admin or repository.user == trans.user ): @@ -1205,7 +1205,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): repository_dependencies_dict = metadata.get( 'repository_dependencies', {} ) repository_dependencies = repository_dependencies_dict.get( 'repository_dependencies', [] ) has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td = \ - suc.get_repository_dependency_types( repository_dependencies ) + repository_util.get_repository_dependency_types( repository_dependencies ) has_galaxy_utilities_dict[ 'has_repository_dependencies' ] = has_repository_dependencies has_galaxy_utilities_dict[ 'has_repository_dependencies_only_if_compiling_contained_td' ] = \ has_repository_dependencies_only_if_compiling_contained_td @@ -1215,8 +1215,8 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): name = kwd.get( 'name', None ) owner = kwd.get( 'owner', None ) changeset_revision = kwd.get( 'changeset_revision', None ) - repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, + repository = repository_util.get_repository_by_name_and_owner( trans.app, name, owner ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, trans.security.encode_id( repository.id ), changeset_revision ) has_galaxy_utilities_dict = has_galaxy_utilities( repository_metadata ) @@ -1262,9 +1262,9 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): changeset_hash = str( repo.changectx( changeset ) ) hg_util.get_changectx_for_changeset( repo, changeset_hash ) if update_to_changeset_hash: - update_to_repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, - trans.security.encode_id( repository.id ), - changeset_hash ) + update_to_repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, + trans.security.encode_id( repository.id ), + changeset_hash ) if update_to_repository_metadata: has_galaxy_utilities_dict = has_galaxy_utilities( repository_metadata ) includes_data_managers = has_galaxy_utilities_dict[ 'includes_data_managers' ] @@ -1305,7 +1305,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): repository_name = kwd[ 'name' ] repository_owner = kwd[ 'owner' ] changeset_revision = kwd[ 'changeset_revision' ] - repository = suc.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) + repository = repository_util.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) if ctx: @@ -1330,10 +1330,10 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): repository_name = kwd.get( 'name', None ) repository_owner = kwd.get( 'owner', None ) if repository_name is not None and repository_owner is not None: - repository = suc.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) + repository = repository_util.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) if repository: repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) - return suc.get_latest_downloadable_changeset_revision( trans.app, repository, repo ) + return metadata_util.get_latest_downloadable_changeset_revision( trans.app, repository, repo ) return hg_util.INITIAL_CHANGELOG_HASH @web.json @@ -1346,12 +1346,12 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): repository_owner = kwd.get( 'owner', None ) changeset_revision = kwd.get( 'changeset_revision', None ) if repository_name is not None and repository_owner is not None and changeset_revision is not None: - repository = suc.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) + repository = repository_util.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) if repository: repository_metadata = \ - suc.get_repository_metadata_by_changeset_revision( trans.app, - trans.security.encode_id( repository.id ), - changeset_revision ) + metadata_util.get_repository_metadata_by_changeset_revision( trans.app, + trans.security.encode_id( repository.id ), + changeset_revision ) if repository_metadata: metadata = repository_metadata.metadata if metadata: @@ -1370,7 +1370,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): name = kwd.get( 'name', None ) owner = kwd.get( 'owner', None ) changeset_revision = kwd.get( 'changeset_revision', None ) - repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + repository = repository_util.get_repository_by_name_and_owner( trans.app, name, owner ) # get_repository_dependencies( self, app, changeset, toolshed_url ) dependencies = repository.get_repository_dependencies( trans.app, changeset_revision, web.url_for( '/', qualified=True ) ) if dependencies: @@ -1382,7 +1382,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): """Given a repository name and owner, return the encoded repository id.""" repository_name = kwd[ 'name' ] repository_owner = kwd[ 'owner' ] - repository = suc.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) + repository = repository_util.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) if repository: return trans.security.encode_id( repository.id ) return '' @@ -1427,7 +1427,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): """Given a repository name and owner, return the type.""" repository_name = kwd[ 'name' ] repository_owner = kwd[ 'owner' ] - repository = suc.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) + repository = repository_util.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) return str( repository.type ) @web.json @@ -1448,7 +1448,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): for required_repository_tup in decoded_required_repository_tups: tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ common_util.parse_repository_dependency_tuple( required_repository_tup ) - repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + repository = repository_util.get_repository_by_name_and_owner( trans.app, name, owner ) encoded_repository_ids.append( trans.security.encode_id( repository.id ) ) changeset_revisions.append( changeset_revision ) if encoded_repository_ids and changeset_revisions: @@ -1464,7 +1464,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): name = kwd.get( 'name', None ) owner = kwd.get( 'owner', None ) changeset_revision = kwd.get( 'changeset_revision', None ) - repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + repository = repository_util.get_repository_by_name_and_owner( trans.app, name, owner ) dependencies = repository.get_tool_dependencies( changeset_revision ) if dependencies is not None: return encoding_util.tool_shed_encode( dependencies ) @@ -1478,7 +1478,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): """ name = kwd.get( 'name', None ) owner = kwd.get( 'owner', None ) - repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + repository = repository_util.get_repository_by_name_and_owner( trans.app, name, owner ) # TODO: We're currently returning the tool_dependencies.xml file that is available on disk. We need # to enhance this process to retrieve older versions of the tool-dependencies.xml file from the repository # manafest. @@ -1501,12 +1501,12 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): """ repository_name = kwd[ 'name' ] repository_owner = kwd[ 'owner' ] - repository = suc.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) + repository = repository_util.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner ) encoded_id = trans.app.security.encode_id( repository.id ) repository_tip = repository.tip( trans.app ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, - encoded_id, - repository_tip ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, + encoded_id, + repository_tip ) return repository_metadata.metadata @web.expose @@ -1518,14 +1518,14 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): name = kwd[ 'name' ] owner = kwd[ 'owner' ] changeset_revision = kwd[ 'changeset_revision' ] - repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + repository = repository_util.get_repository_by_name_and_owner( trans.app, name, owner ) repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) tool_version_dicts = [] for changeset in repo.changelog: current_changeset_revision = str( repo.changectx( changeset ) ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, - trans.security.encode_id( repository.id ), - current_changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, + trans.security.encode_id( repository.id ), + current_changeset_revision ) if repository_metadata and repository_metadata.tool_versions: tool_version_dicts.append( repository_metadata.tool_versions ) if current_changeset_revision == changeset_revision: @@ -1540,18 +1540,18 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): Generate a dictionary that contains the information about a repository that is necessary for installing it into a local Galaxy instance. """ - repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + repository = repository_util.get_repository_by_name_and_owner( trans.app, name, owner ) repository_id = trans.security.encode_id( repository.id ) repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( trans.user, repository ) repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) if not repository_metadata: # The received changeset_revision is no longer associated with metadata, so get the next changeset_revision in the repository # changelog that is associated with metadata. - changeset_revision = suc.get_next_downloadable_changeset_revision( repository, - repo, - after_changeset_revision=changeset_revision ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) + changeset_revision = metadata_util.get_next_downloadable_changeset_revision( repository, + repo, + after_changeset_revision=changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) repo_info_dict = repository_util.create_repo_info_dict( app=trans.app, repository_clone_url=repository_clone_url, @@ -1592,13 +1592,13 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): includes_tool_dependencies = False for name, repo_info_tuple in repo_info_dict.items(): if not has_repository_dependencies or not has_repository_dependencies_only_if_compiling_contained_td or not includes_tool_dependencies: - description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \ - suc.get_repo_info_tuple_contents( repo_info_tuple ) + description, reposectory_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \ + repository_util.get_repo_info_tuple_contents( repo_info_tuple ) for rd_key, rd_tups in repository_dependencies.items(): if rd_key in [ 'root_key', 'description' ]: continue curr_has_repository_dependencies, curr_has_repository_dependencies_only_if_compiling_contained_td = \ - suc.get_repository_dependency_types( rd_tups ) + repository_util.get_repository_dependency_types( rd_tups ) if curr_has_repository_dependencies and not has_repository_dependencies: has_repository_dependencies = True if curr_has_repository_dependencies_only_if_compiling_contained_td and not has_repository_dependencies_only_if_compiling_contained_td: @@ -1726,7 +1726,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): name = kwd.get( 'name', None ) owner = kwd.get( 'owner', None ) if not repository_ids: - repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + repository = repository_util.get_repository_by_name_and_owner( trans.app, name, owner ) repository_ids = trans.security.encode_id( repository.id ) galaxy_url = common_util.handle_galaxy_url( trans, **kwd ) if galaxy_url: @@ -1832,13 +1832,13 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): def manage_repository( self, trans, id, **kwd ): message = escape( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) - repository = suc.get_repository_in_tool_shed( trans.app, id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, id ) repository_type = kwd.get( 'repository_type', str( repository.type ) ) repo_dir = repository.repo_path( trans.app ) repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) repo_name = kwd.get( 'repo_name', repository.name ) changeset_revision = kwd.get( 'changeset_revision', repository.tip( trans.app ) ) - repository.share_url = suc.generate_sharable_link_for_repository_in_tool_shed( repository, changeset_revision=changeset_revision ) + repository.share_url = repository_util.generate_sharable_link_for_repository_in_tool_shed( repository, changeset_revision=changeset_revision ) repository.clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( trans.user, repository ) remote_repository_url = kwd.get( 'remote_repository_url', repository.remote_repository_url ) homepage_url = kwd.get( 'homepage_url', repository.homepage_url ) @@ -1939,7 +1939,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): is_malicious = False repository_dependencies = None if changeset_revision != hg_util.INITIAL_CHANGELOG_HASH: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) if repository_metadata: revision_label = hg_util.get_revision_label( trans.app, repository, changeset_revision, include_date=False ) metadata = repository_metadata.metadata @@ -1950,7 +1950,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): previous_changeset_revision = \ metadata_util.get_previous_metadata_changeset_revision( repository, repo, changeset_revision, downloadable=False ) if previous_changeset_revision != hg_util.INITIAL_CHANGELOG_HASH: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, previous_changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, id, previous_changeset_revision ) if repository_metadata: revision_label = hg_util.get_revision_label( trans.app, repository, previous_changeset_revision, include_date=False ) metadata = repository_metadata.metadata @@ -2037,11 +2037,11 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): def manage_repository_admins( self, trans, id, **kwd ): message = escape( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) - repository = suc.get_repository_in_tool_shed( trans.app, id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, id ) changeset_revision = kwd.get( 'changeset_revision', repository.tip( trans.app ) ) metadata = None if changeset_revision != hg_util.INITIAL_CHANGELOG_HASH: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) if repository_metadata: metadata = repository_metadata.metadata else: @@ -2054,9 +2054,9 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): changeset_revision, downloadable=False ) if previous_changeset_revision != hg_util.INITIAL_CHANGELOG_HASH: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, - id, - previous_changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, + id, + previous_changeset_revision ) if repository_metadata: metadata = repository_metadata.metadata role = repository.admin_role @@ -2123,10 +2123,10 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): name = kwd.get( 'name', None ) owner = kwd.get( 'owner', None ) changeset_revision = kwd.get( 'changeset_revision', None ) - repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + repository = repository_util.get_repository_by_name_and_owner( trans.app, name, owner ) repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) # Get the next installable changeset_revision beyond the received changeset_revision. - next_changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) + next_changeset_revision = metadata_util.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) if next_changeset_revision and next_changeset_revision != changeset_revision: return next_changeset_revision return '' @@ -2143,9 +2143,9 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): def preview_tools_in_changeset( self, trans, repository_id, **kwd ): message = escape( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) - repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id ) changeset_revision = kwd.get( 'changeset_revision', repository.tip( trans.app ) ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) if repository_metadata: repository_metadata_id = trans.security.encode_id( repository_metadata.id ), metadata = repository_metadata.metadata @@ -2219,7 +2219,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): name = kwd.get( 'name', None ) owner = kwd.get( 'owner', None ) if name is not None and owner is not None: - repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + repository = repository_util.get_repository_by_name_and_owner( trans.app, name, owner ) from_tip = util.string_as_bool( from_tip ) if from_tip: changeset_revision = repository.tip( trans.app ) @@ -2255,7 +2255,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): action='browse_repositories', message='Select a repository to rate', status='error' ) ) - repository = suc.get_repository_in_tool_shed( trans.app, id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, id ) changeset_revision = repository.tip( trans.app ) if repository.user == trans.user: return trans.response.send_redirect( web.url_for( controller='repository', @@ -2291,7 +2291,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): def reset_all_metadata( self, trans, id, **kwd ): """Reset all metadata on the complete changelog for a single repository in the tool shed.""" # This method is called only from the ~/templates/webapps/tool_shed/repository/manage_repository.mako template. - repository = suc.get_repository_in_tool_shed( trans.app, id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, id ) rmm = repository_metadata_manager.RepositoryMetadataManager( app=trans.app, user=trans.user, repository=repository, @@ -2336,7 +2336,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): message = escape( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) commit_message = escape( kwd.get( 'commit_message', 'Deleted selected files' ) ) - repository = suc.get_repository_in_tool_shed( trans.app, id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, id ) repo_dir = repository.repo_path( trans.app ) repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) selected_files_to_delete = kwd.get( 'selected_files_to_delete', '' ) @@ -2414,7 +2414,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): @web.expose def send_to_owner( self, trans, id, message='' ): - repository = suc.get_repository_in_tool_shed( trans.app, id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, id ) if not message: message = 'Enter a message' status = 'error' @@ -2464,7 +2464,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): total_alerts_removed = 0 flush_needed = False for repository_id in repository_ids: - repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id ) if repository.email_alerts: email_alerts = json.loads( repository.email_alerts ) else: @@ -2496,7 +2496,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): def set_malicious( self, trans, id, ctx_str, **kwd ): malicious = kwd.get( 'malicious', '' ) if kwd.get( 'malicious_button', False ): - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, ctx_str ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, id, ctx_str ) malicious_checked = CheckboxField.is_checked( malicious ) repository_metadata.malicious = malicious_checked trans.sa_session.add( repository_metadata ) @@ -2518,7 +2518,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): def sharable_owner( self, trans, owner ): """Support for sharable URL for each repository owner's tools, e.g. http://example.org/view/owner.""" try: - user = suc.get_user_by_username( trans, owner ) + user = common_util.get_user_by_username( trans, owner ) except: user = None if user: @@ -2534,7 +2534,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): def sharable_repository( self, trans, owner, name ): """Support for sharable URL for a specified repository, e.g. http://example.org/view/owner/name.""" try: - repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + repository = repository_util.get_repository_by_name_and_owner( trans.app, name, owner ) except: repository = None if repository: @@ -2545,7 +2545,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): else: # If the owner is valid, then show all of their repositories. try: - user = suc.get_user_by_username( trans, owner ) + user = common_util.get_user_by_username( trans, owner ) except: user = None if user: @@ -2564,7 +2564,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): def sharable_repository_revision( self, trans, owner, name, changeset_revision ): """Support for sharable URL for a specified repository revision, e.g. http://example.org/view/owner/name/changeset_revision.""" try: - repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + repository = repository_util.get_repository_by_name_and_owner( trans.app, name, owner ) except: repository = None if repository: @@ -2575,7 +2575,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): if not repository_metadata: # Get updates to the received changeset_revision if any exist. repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) - upper_bound_changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) + upper_bound_changeset_revision = metadata_util.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) if upper_bound_changeset_revision and upper_bound_changeset_revision != changeset_revision: changeset_revision = upper_bound_changeset_revision repository_metadata = metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans.app, @@ -2610,11 +2610,11 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): name = kwd.get( 'name', None ) owner = kwd.get( 'owner', None ) changeset_revision = kwd.get( 'changeset_revision', None ) - repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + repository = repository_util.get_repository_by_name_and_owner( trans.app, name, owner ) if repository: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, - trans.security.encode_id( repository.id ), - changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, + trans.security.encode_id( repository.id ), + changeset_revision ) repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) tool_shed_status_dict = {} # Handle repository deprecation. @@ -2623,7 +2623,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): if changeset_revision == repository.tip( trans.app ): tool_shed_status_dict[ 'latest_installable_revision' ] = 'True' else: - next_installable_revision = suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) + next_installable_revision = metadata_util.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) if repository_metadata is None: if next_installable_revision and next_installable_revision != changeset_revision: tool_shed_status_dict[ 'latest_installable_revision' ] = 'True' @@ -2643,7 +2643,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): else: tool_shed_status_dict[ 'revision_update' ] = 'False' # Handle revision upgrades. - metadata_revisions = [ revision[ 1 ] for revision in suc.get_metadata_revisions( repository, repo ) ] + metadata_revisions = [ revision[ 1 ] for revision in metadata_util.get_metadata_revisions( repository, repo ) ] num_metadata_revisions = len( metadata_revisions ) for index, metadata_revision in enumerate( metadata_revisions ): if index == num_metadata_revisions: @@ -2673,7 +2673,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): owner = kwd.get( 'owner', None ) changeset_revision = kwd.get( 'changeset_revision', None ) if name and owner and changeset_revision: - return suc.get_updated_changeset_revisions( trans.app, name, owner, changeset_revision ) + return metadata_util.get_updated_changeset_revisions( trans.app, name, owner, changeset_revision ) return '' @web.expose @@ -2710,12 +2710,12 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): def view_changelog( self, trans, id, **kwd ): message = escape( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) - repository = suc.get_repository_in_tool_shed( trans.app, id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, id ) repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) changesets = [] for changeset in repo.changelog: ctx = repo.changectx( changeset ) - if suc.get_repository_metadata_by_changeset_revision( trans.app, id, str( ctx ) ): + if metadata_util.get_repository_metadata_by_changeset_revision( trans.app, id, str( ctx ) ): has_metadata = True else: has_metadata = False @@ -2745,7 +2745,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): def view_changeset( self, trans, id, ctx_str, **kwd ): message = escape( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) - repository = suc.get_repository_in_tool_shed( trans.app, id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, id ) repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) ctx = hg_util.get_changectx_for_changeset( repo, ctx_str ) if ctx is None: @@ -2818,7 +2818,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): def view_or_manage_repository( self, trans, **kwd ): repository_id = kwd.get( 'id', None ) if repository_id: - repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id ) user = trans.user if repository: if user is not None and ( trans.user_is_admin() or @@ -2837,11 +2837,11 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): def view_repository( self, trans, id, **kwd ): message = escape( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) - repository = suc.get_repository_in_tool_shed( trans.app, id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, id ) repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) avg_rating, num_ratings = self.get_ave_item_rating_data( trans.sa_session, repository, webapp_model=trans.model ) changeset_revision = kwd.get( 'changeset_revision', repository.tip( trans.app ) ) - repository.share_url = suc.generate_sharable_link_for_repository_in_tool_shed( repository, changeset_revision=changeset_revision ) + repository.share_url = repository_util.generate_sharable_link_for_repository_in_tool_shed( repository, changeset_revision=changeset_revision ) repository.clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( trans.user, repository ) display_reviews = kwd.get( 'display_reviews', False ) alerts = kwd.get( 'alerts', '' ) @@ -2875,7 +2875,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): add_id_to_name=False, downloadable=False ) revision_label = hg_util.get_revision_label( trans.app, repository, changeset_revision, include_date=False ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, id, changeset_revision ) if repository_metadata: metadata = repository_metadata.metadata # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. @@ -2928,7 +2928,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): message = escape( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) render_repository_actions_for = kwd.get( 'render_repository_actions_for', 'tool_shed' ) - repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id ) repo_files_dir = repository.repo_path( trans.app ) repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_files_dir, create=False ) tool_metadata_dict = {} @@ -2937,7 +2937,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): guid = None original_tool_data_path = trans.app.config.tool_data_path revision_label = hg_util.get_revision_label( trans.app, repository, changeset_revision, include_date=False ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) if repository_metadata: repository_metadata_id = trans.security.encode_id( repository_metadata.id ) metadata = repository_metadata.metadata @@ -3012,7 +3012,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): if workflow_name: workflow_name = encoding_util.tool_shed_decode( workflow_name ) repository_metadata = metadata_util.get_repository_metadata_by_id( trans.app, repository_metadata_id ) - repository = suc.get_repository_in_tool_shed( trans.app, trans.security.encode_id( repository_metadata.repository_id ) ) + repository = repository_util.get_repository_in_tool_shed( trans.app, trans.security.encode_id( repository_metadata.repository_id ) ) changeset_revision = repository_metadata.changeset_revision metadata = repository_metadata.metadata return trans.fill_template( "/webapps/tool_shed/repository/view_workflow.mako", diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository_review.py b/lib/galaxy/webapps/tool_shed/controllers/repository_review.py index eab995f4e52..ce53369879f 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/repository_review.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository_review.py @@ -12,6 +12,8 @@ from galaxy.web.form_builder import CheckboxField from galaxy.webapps.tool_shed.util import ratings_util from tool_shed.util import hg_util from tool_shed.util import review_util +from tool_shed.util import metadata_util +from tool_shed.util import repository_util from tool_shed.util import shed_util_common as suc from tool_shed.util.container_util import STRSEP from tool_shed.util.web_util import escape @@ -139,7 +141,7 @@ class RepositoryReviewController( BaseUIController, ratings_util.ItemRatings ): if repository_id: if changeset_revision: # Make sure there is not already a review of the revision by the user. - repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id ) if review_util.get_review_by_repository_id_changeset_revision_user_id( app=trans.app, repository_id=repository_id, changeset_revision=changeset_revision, @@ -156,7 +158,7 @@ class RepositoryReviewController( BaseUIController, ratings_util.ItemRatings ): **kwd ) ) # A review can be initially performed only on an installable revision of a repository, so make sure we have metadata associated # with the received changeset_revision. - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) if repository_metadata: metadata = repository_metadata.metadata if metadata: @@ -489,7 +491,7 @@ class RepositoryReviewController( BaseUIController, ratings_util.ItemRatings ): changeset_revision ) # Determine if the current user can add a review to this revision. can_add_review = trans.user not in [ repository_review.user for repository_review in repository_reviews ] - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision ) if repository_metadata: repository_metadata_reviews = util.listify( repository_metadata.reviews ) else: @@ -520,7 +522,7 @@ class RepositoryReviewController( BaseUIController, ratings_util.ItemRatings ): status = kwd.get( 'status', 'done' ) repository_id = kwd.get( 'id', None ) changeset_revision = kwd.get( 'changeset_revision', None ) - repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id ) repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) installable = changeset_revision in [ metadata_revision.changeset_revision for metadata_revision in repository.metadata_revisions ] rev, changeset_revision_label = hg_util.get_rev_label_from_changeset_revision( repo, changeset_revision ) @@ -582,7 +584,7 @@ class RepositoryReviewController( BaseUIController, ratings_util.ItemRatings ): # The value of the received id is the encoded repository id. message = escape( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) - repository = suc.get_repository_in_tool_shed( trans.app, kwd[ 'id' ] ) + repository = repository_util.get_repository_in_tool_shed( trans.app, kwd[ 'id' ] ) changeset_revision = kwd.get( 'changeset_revision', None ) repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) previous_reviews_dict = review_util.get_previous_repository_reviews( trans.app, @@ -600,7 +602,7 @@ class RepositoryReviewController( BaseUIController, ratings_util.ItemRatings ): @web.expose @web.require_login( "view or manage repository" ) def view_or_manage_repository( self, trans, **kwd ): - repository = suc.get_repository_in_tool_shed( trans.app, kwd[ 'id' ] ) + repository = repository_util.get_repository_in_tool_shed( trans.app, kwd[ 'id' ] ) if trans.user_is_admin() or repository.user == trans.user: return trans.response.send_redirect( web.url_for( controller='repository', action='manage_repository', diff --git a/lib/galaxy/webapps/tool_shed/controllers/upload.py b/lib/galaxy/webapps/tool_shed/controllers/upload.py index f2d0c5d7671..1d005c7f7f2 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -20,6 +20,7 @@ from tool_shed.tools import data_table_manager from tool_shed.util import basic_util from tool_shed.util import commit_util from tool_shed.util import hg_util +from tool_shed.util import repository_util from tool_shed.util import shed_util_common as suc from tool_shed.util import repository_content_util from tool_shed.util import xml_util @@ -38,7 +39,7 @@ class UploadController( BaseUIController ): status = kwd.get( 'status', 'done' ) commit_message = escape( kwd.get( 'commit_message', 'Uploaded' ) ) repository_id = kwd.get( 'repository_id', '' ) - repository = suc.get_repository_in_tool_shed( trans.app, repository_id ) + repository = repository_util.get_repository_in_tool_shed( trans.app, repository_id ) repo_dir = repository.repo_path( trans.app ) repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False ) uncompress_file = util.string_as_bool( kwd.get( 'uncompress_file', 'true' ) ) diff --git a/lib/galaxy/webapps/tool_shed/model/__init__.py b/lib/galaxy/webapps/tool_shed/model/__init__.py index 2bdd67f05f0..c4b07a97614 100644 --- a/lib/galaxy/webapps/tool_shed/model/__init__.py +++ b/lib/galaxy/webapps/tool_shed/model/__init__.py @@ -10,7 +10,7 @@ from galaxy.util.hash_util import new_secure_hash from galaxy.util.dictifiable import Dictifiable import tool_shed.repository_types.util as rt_util from tool_shed.dependencies.repository import relation_builder -from tool_shed.util import shed_util_common as suc +from tool_shed.util import metadata_util from mercurial import hg from mercurial import ui @@ -230,9 +230,9 @@ class Repository( object, Dictifiable ): # have repository dependencies. However, if a readme file is uploaded, or some other change # is made that does not create a new downloadable changeset revision but updates the existing # one, we still want to be able to get repository dependencies. - repository_metadata = suc.get_current_repository_metadata_for_changeset_revision( app, - self, - changeset ) + repository_metadata = metadata_util.get_current_repository_metadata_for_changeset_revision( app, + self, + changeset ) if repository_metadata: metadata = repository_metadata.metadata if metadata: @@ -252,9 +252,9 @@ class Repository( object, Dictifiable ): return [] def installable_revisions( self, app, sort_revisions=True ): - return suc.get_metadata_revisions( self, - hg.repository( ui.ui(), self.repo_path( app ) ), - sort_revisions=sort_revisions ) + return metadata_util.get_metadata_revisions( self, + hg.repository( ui.ui(), self.repo_path( app ) ), + sort_revisions=sort_revisions ) def is_new( self, app ): repo = hg.repository( ui.ui(), self.repo_path( app ) ) diff --git a/lib/tool_shed/capsule/capsule_manager.py b/lib/tool_shed/capsule/capsule_manager.py index dc1d6483071..02b4176281b 100644 --- a/lib/tool_shed/capsule/capsule_manager.py +++ b/lib/tool_shed/capsule/capsule_manager.py @@ -27,6 +27,7 @@ from tool_shed.util import commit_util from tool_shed.util import common_util from tool_shed.util import encoding_util from tool_shed.util import hg_util +from tool_shed.util import metadata_util from tool_shed.util import repository_util from tool_shed.util import shed_util_common as suc from tool_shed.util import xml_util @@ -68,9 +69,9 @@ class ExportRepositoryManager( object ): ordered_changeset_revisions = [] if self.repository: repository_metadata = \ - suc.get_current_repository_metadata_for_changeset_revision( self.app, - self.repository, - self.changeset_revision ) + metadata_util.get_current_repository_metadata_for_changeset_revision( self.app, + self.repository, + self.changeset_revision ) if repository_metadata: ordered_repository_ids = [ self.repository_id ] ordered_repositories = [ self.repository ] @@ -214,11 +215,11 @@ class ExportRepositoryManager( object ): # There should only be one entry in the received repo_info_dict. description, repository_clone_url, changeset_revision, ctx_rev, \ repository_owner, repository_dependencies, tool_dependencies = \ - suc.get_repo_info_tuple_contents( repo_info_tup ) - repository = suc.get_repository_by_name_and_owner( self.app, repository_name, repository_owner ) - repository_metadata = suc.get_current_repository_metadata_for_changeset_revision( self.app, - repository, - changeset_revision ) + repository_util.get_repo_info_tuple_contents( repo_info_tup ) + repository = repository_util.get_repository_by_name_and_owner( self.app, repository_name, repository_owner ) + repository_metadata = metadata_util.get_current_repository_metadata_for_changeset_revision( self.app, + repository, + changeset_revision ) if repository_metadata: return repository, repository_metadata.changeset_revision return None, None @@ -242,10 +243,10 @@ class ExportRepositoryManager( object ): associated with self.repository_id. """ rdim = RepositoryDependencyInstallManager( self.app ) - repository = suc.get_repository_in_tool_shed( self.app, self.repository_id ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( self.app, - self.repository_id, - self.changeset_revision ) + repository = repository_util.get_repository_in_tool_shed( self.app, self.repository_id ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( self.app, + self.repository_id, + self.changeset_revision ) # Get a dictionary of all repositories upon which the contents of the current # repository_metadata record depend. toolshed_base_url = str( web.url_for( '/', qualified=True ) ).rstrip( '/' ) @@ -305,8 +306,8 @@ class ExportRepositoryManager( object ): description, repository_clone_url, changeset_revision, \ ctx_rev, repository_owner, repository_dependencies, \ tool_dependencies = \ - suc.get_repo_info_tuple_contents( repo_info_tup ) - repository = suc.get_repository_by_name_and_owner( self.app, repository_name, repository_owner ) + repository_util.get_repo_info_tuple_contents( repo_info_tup ) + repository = repository_util.get_repository_by_name_and_owner( self.app, repository_name, repository_owner ) repository_ids.append( self.app.security.encode_id( repository.id ) ) return repository_ids @@ -328,9 +329,9 @@ class ExportRepositoryManager( object ): # Create a dictionary whose keys are the received repository_ids and whose values are a list of # repository_ids, each of which is contained in the received list of repository_ids and whose associated # repository must be imported prior to the repository associated with the repository_id key. - prior_import_required_dict = suc.get_prior_import_or_install_required_dict( self.app, - repository_ids, - repo_info_dicts ) + prior_import_required_dict = repository_util.get_prior_import_or_install_required_dict( self.app, + repository_ids, + repo_info_dicts ) processed_repository_ids = [] # Process the list of repository dependencies defined for the primary exported repository. while len( processed_repository_ids ) != len( prior_import_required_dict.keys() ): @@ -403,13 +404,13 @@ class ImportRepositoryManager( object ): ok, name_owner, message = import_results_tup name, owner = name_owner if not ok: - repository = suc.get_repository_by_name_and_owner( self.app, name, owner ) + repository = repository_util.get_repository_by_name_and_owner( self.app, name, owner ) if repository is not None: # Do not allow the repository to be automatically installed if population resulted in errors. tip_changeset_revision = repository.tip( self.app ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( self.app, - self.app.security.encode_id( repository.id ), - tip_changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( self.app, + self.app.security.encode_id( repository.id ), + tip_changeset_revision ) if repository_metadata: if repository_metadata.downloadable: repository_metadata.downloadable = False @@ -451,7 +452,7 @@ class ImportRepositoryManager( object ): long_description = repository_archive_dict.get( 'long_description', '' ) # The owner entry in the repository_archive_dict is the public username of the user associated with # the exported repository archive. - user = suc.get_user_by_username( self.app, username ) + user = common_util.get_user_by_username( self.app, username ) if user is None: ok = False results_message += 'Import failed: repository owner %s does not have an account in this Tool Shed.' % \ @@ -595,15 +596,15 @@ class ImportRepositoryManager( object ): if defined_repository_metadata is None: # The defined changeset_revision is not associated with a repository_metadata # record, so updates must be necessary. - defined_repository = suc.get_repository_by_name_and_owner( self.app, name, owner ) + defined_repository = repository_util.get_repository_by_name_and_owner( self.app, name, owner ) defined_repo = hg_util.get_repo_for_repository( self.app, repository=defined_repository, repo_path=None, create=False ) updated_changeset_revision = \ - suc.get_next_downloadable_changeset_revision( defined_repository, - defined_repo, - changeset_revision ) + metadata_util.get_next_downloadable_changeset_revision( defined_repository, + defined_repo, + changeset_revision ) if updated_changeset_revision == rm_changeset_revision and updated_changeset_revision != changeset_revision: dependent_downloadable_revisions.append( downloadable_revision ) return dependent_downloadable_revisions @@ -691,9 +692,9 @@ class ImportRepositoryManager( object ): """ repository_status_info_dicts = [] for repository_info_dict in repository_info_dicts: - repository = suc.get_repository_by_name_and_owner( self.app, - repository_info_dict[ 'name' ], - repository_info_dict[ 'owner' ] ) + repository = repository_util.get_repository_by_name_and_owner( self.app, + repository_info_dict[ 'name' ], + repository_info_dict[ 'owner' ] ) if repository: if repository.deleted: repository_info_dict[ 'status' ] = 'Exists, deleted' diff --git a/lib/tool_shed/dependencies/attribute_handlers.py b/lib/tool_shed/dependencies/attribute_handlers.py index 68c592ff316..8b9a896cb6c 100644 --- a/lib/tool_shed/dependencies/attribute_handlers.py +++ b/lib/tool_shed/dependencies/attribute_handlers.py @@ -9,7 +9,8 @@ from tool_shed.dependencies.tool import tag_attribute_handler from tool_shed.repository_types.util import REPOSITORY_DEPENDENCY_DEFINITION_FILENAME from tool_shed.repository_types.util import TOOL_DEPENDENCY_DEFINITION_FILENAME from tool_shed.util import hg_util -from tool_shed.util import shed_util_common as suc +from tool_shed.util import metadata_util +from tool_shed.util import repository_util from tool_shed.util import xml_util log = logging.getLogger( __name__ ) @@ -106,14 +107,14 @@ class RepositoryDependencyAttributeHandler( object ): # Populate the changeset_revision attribute with the latest installable metadata revision for # the defined repository. We use the latest installable revision instead of the latest metadata # revision to ensure that the contents of the revision are valid. - repository = suc.get_repository_by_name_and_owner( self.app, name, owner ) + repository = repository_util.get_repository_by_name_and_owner( self.app, name, owner ) if repository: repo = hg_util.get_repo_for_repository( self.app, repository=repository, repo_path=None, create=False ) lastest_installable_changeset_revision = \ - suc.get_latest_downloadable_changeset_revision( self.app, repository, repo ) + metadata_util.get_latest_downloadable_changeset_revision( self.app, repository, repo ) if lastest_installable_changeset_revision != hg_util.INITIAL_CHANGELOG_HASH: elem.attrib[ 'changeset_revision' ] = lastest_installable_changeset_revision altered = True diff --git a/lib/tool_shed/dependencies/repository/relation_builder.py b/lib/tool_shed/dependencies/repository/relation_builder.py index 6bc2e578aac..056c8f79b20 100644 --- a/lib/tool_shed/dependencies/repository/relation_builder.py +++ b/lib/tool_shed/dependencies/repository/relation_builder.py @@ -2,6 +2,7 @@ import logging from galaxy.util import asbool, listify +import tool_shed.util.repository_util from tool_shed.util import common_util, container_util, hg_util, metadata_util from tool_shed.util import shed_util_common as suc @@ -80,10 +81,10 @@ class RelationBuilder( object ): current_changeset_revision = str( self.repository_metadata.changeset_revision ) # Get the changeset revision to which the current value of required_repository_changeset_revision # should be updated if it's not current. - text = suc.get_updated_changeset_revisions( self.app, - name=str( self.repository.name ), - owner=str( self.repository.user.username ), - changeset_revision=current_changeset_revision ) + text = metadata_util.get_updated_changeset_revisions( self.app, + name=str( self.repository.name ), + owner=str( self.repository.user.username ), + changeset_revision=current_changeset_revision ) if text: valid_changeset_revisions = listify( text ) if current_changeset_revision not in valid_changeset_revisions: @@ -181,7 +182,7 @@ class RelationBuilder( object ): rd_only_if_compiling_contained_td = \ common_util.parse_repository_dependency_tuple( repository_dependency ) if suc.tool_shed_is_this_tool_shed( rd_toolshed ): - repository = suc.get_repository_by_name_and_owner( self.app, rd_name, rd_owner ) + repository = tool_shed.util.repository_util.get_repository_by_name_and_owner( self.app, rd_name, rd_owner ) if repository: repository_id = self.app.security.encode_id( repository.id ) repository_metadata = \ @@ -196,7 +197,7 @@ class RelationBuilder( object ): else: # The repository changeset_revision is no longer installable, so see if there's been an update. repo = hg_util.get_repo_for_repository( self.app, repository=repository, repo_path=None, create=False ) - changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, rd_changeset_revision ) + changeset_revision = metadata_util.get_next_downloadable_changeset_revision( repository, repo, rd_changeset_revision ) if changeset_revision != rd_changeset_revision: repository_metadata = \ metadata_util.get_repository_metadata_by_repository_id_changeset_revision( self.app, @@ -215,7 +216,7 @@ class RelationBuilder( object ): updated_key_rd_dicts.append( new_key_rd_dict ) else: repository_components_tuple = container_util.get_components_from_key( key ) - components_list = suc.extract_components_from_tuple( repository_components_tuple ) + components_list = tool_shed.util.repository_util.extract_components_from_tuple( repository_components_tuple ) toolshed, repository_name, repository_owner, repository_changeset_revision = components_list[ 0:4 ] # For backward compatibility to the 12/20/12 Galaxy release. if len( components_list ) in (4, 5): @@ -226,7 +227,7 @@ class RelationBuilder( object ): log.debug( message ) else: repository_components_tuple = container_util.get_components_from_key( key ) - components_list = suc.extract_components_from_tuple( repository_components_tuple ) + components_list = tool_shed.util.repository_util.extract_components_from_tuple( repository_components_tuple ) toolshed, repository_name, repository_owner, repository_changeset_revision = components_list[ 0:4 ] message = "The revision %s defined for repository %s owned by %s is invalid, so repository " % \ ( str( rd_changeset_revision ), str( rd_name ), str( rd_owner ) ) @@ -260,7 +261,7 @@ class RelationBuilder( object ): toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ common_util.parse_repository_dependency_tuple( repository_dependency ) if suc.tool_shed_is_this_tool_shed( toolshed ): - required_repository = suc.get_repository_by_name_and_owner( self.app, name, owner ) + required_repository = tool_shed.util.repository_util.get_repository_by_name_and_owner( self.app, name, owner ) self.repository = required_repository repository_id = self.app.security.encode_id( required_repository.id ) required_repository_metadata = \ diff --git a/lib/tool_shed/galaxy_install/dependency_display.py b/lib/tool_shed/galaxy_install/dependency_display.py index ea6407c4adf..997822411e8 100644 --- a/lib/tool_shed/galaxy_install/dependency_display.py +++ b/lib/tool_shed/galaxy_install/dependency_display.py @@ -5,8 +5,11 @@ import threading from galaxy import util from tool_shed.galaxy_install.utility_containers import GalaxyUtilityContainerManager -from tool_shed.util import common_util, container_util, readme_util -from tool_shed.util import shed_util_common as suc, tool_dependency_util +from tool_shed.util import common_util +from tool_shed.util import container_util +from tool_shed.util import readme_util +from tool_shed.util import repository_util +from tool_shed.util import tool_dependency_util from tool_shed.utility_containers import utility_container_manager log = logging.getLogger( __name__ ) @@ -286,7 +289,7 @@ class DependencyDisplayer( object ): folder_id += 1 repository_components_tuple = \ container_util.get_components_from_key( old_container_repository_dependencies_folder.key ) - components_list = suc.extract_components_from_tuple( repository_components_tuple ) + components_list = repository_util.extract_components_from_tuple( repository_components_tuple ) name = components_list[ 1 ] # Generate the label by retrieving the repository name. old_container_repository_dependencies_folder.label = str( name ) @@ -553,17 +556,17 @@ class DependencyDisplayer( object ): description, repository_clone_url, changeset_revision, \ ctx_rev, repository_owner, repository_dependencies, \ tool_dependencies = \ - suc.get_repo_info_tuple_contents( repo_info_tuple ) + repository_util.get_repo_info_tuple_contents( repo_info_tuple ) if tool_dependencies: # Add the install_dir attribute to the tool_dependencies. tool_dependencies = self.add_installation_directories_to_tool_dependencies( tool_dependencies ) # The required_repository may have been installed with a different changeset revision. required_repository, installed_changeset_revision = \ - suc.repository_was_previously_installed( self.app, - tool_shed_url, - name, - repo_info_tuple, - from_tip=False ) + repository_util.repository_was_previously_installed( self.app, + tool_shed_url, + name, + repo_info_tuple, + from_tip=False ) if required_repository: required_repository_installed_tool_dependencies, required_repository_missing_tool_dependencies = \ self.get_installed_and_missing_tool_dependencies_for_installed_repository( required_repository, diff --git a/lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py b/lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py index 581feb7f056..35352cb4897 100644 --- a/lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py +++ b/lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py @@ -2,12 +2,12 @@ import logging from sqlalchemy import false, or_ -import tool_shed.util.shed_util_common as suc from galaxy import util from galaxy.model import tool_shed_install from galaxy.web import url_for from galaxy.web.framework.helpers import iff, grids from tool_shed.util import tool_dependency_util +from tool_shed.util import repository_util log = logging.getLogger( __name__ ) @@ -104,7 +104,7 @@ class InstalledRepositoryGrid( grids.Grid ): class StatusColumn( grids.TextColumn ): def get_value( self, trans, grid, tool_shed_repository ): - return suc.get_tool_shed_repository_status_label( trans.app, tool_shed_repository ) + return repository_util.get_tool_shed_repository_status_label( trans.app, tool_shed_repository ) class ToolShedColumn( grids.TextColumn ): diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py index 4f31d032f6d..0f0ea4713d1 100644 --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -10,7 +10,7 @@ from six import string_types from sqlalchemy import or_ from galaxy import exceptions, util -from tool_shed.util import basic_util, common_util, encoding_util, hg_util +from tool_shed.util import basic_util, common_util, encoding_util, hg_util, repository_util from tool_shed.util import shed_util_common as suc, tool_dependency_util from tool_shed.util import tool_util, xml_util @@ -517,7 +517,7 @@ class InstallRepositoryManager( object ): irmm_metadata_dict = irmm.get_metadata_dict() tool_shed_repository.metadata = irmm_metadata_dict # Update the tool_shed_repository.tool_shed_status column in the database. - tool_shed_status_dict = suc.get_tool_shed_status_for_installed_repository( self.app, tool_shed_repository ) + tool_shed_status_dict = repository_util.get_tool_shed_status_for_installed_repository( self.app, tool_shed_repository ) if tool_shed_status_dict: tool_shed_repository.tool_shed_status = tool_shed_status_dict self.install_model.context.add( tool_shed_repository ) @@ -864,8 +864,8 @@ class InstallRepositoryManager( object ): self.install_model.ToolShedRepository.installation_status.CLONING ) repo_info_tuple = repo_info_dict[ tool_shed_repository.name ] description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = repo_info_tuple - relative_clone_dir = suc.generate_tool_shed_repository_install_dir( repository_clone_url, - tool_shed_repository.installed_changeset_revision ) + relative_clone_dir = repository_util.generate_tool_shed_repository_install_dir( repository_clone_url, + tool_shed_repository.installed_changeset_revision ) relative_install_dir = os.path.join( relative_clone_dir, tool_shed_repository.name ) install_dir = os.path.join( tool_path, relative_install_dir ) cloned_ok, error_message = hg_util.clone_repository( repository_clone_url, os.path.abspath( install_dir ), ctx_rev ) @@ -927,13 +927,13 @@ class InstallRepositoryManager( object ): self.app.installed_repository_manager.handle_repository_install( tool_shed_repository ) else: # An error occurred while cloning the repository, so reset everything necessary to enable another attempt. - suc.set_repository_attributes( self.app, - tool_shed_repository, - status=self.install_model.ToolShedRepository.installation_status.ERROR, - error_message=error_message, - deleted=False, - uninstalled=False, - remove_from_disk=True ) + repository_util.set_repository_attributes( self.app, + tool_shed_repository, + status=self.install_model.ToolShedRepository.installation_status.ERROR, + error_message=error_message, + deleted=False, + uninstalled=False, + remove_from_disk=True ) def order_components_for_installation( self, tsr_ids, repo_info_dicts, tool_panel_section_keys ): """ @@ -952,9 +952,9 @@ class InstallRepositoryManager( object ): # Create a dictionary whose keys are the received tsr_ids and whose values are a list of # tsr_ids, each of which is contained in the received list of tsr_ids and whose associated # repository must be installed prior to the repository associated with the tsr_id key. - prior_install_required_dict = suc.get_prior_import_or_install_required_dict( self.app, - tsr_ids, - repo_info_dicts ) + prior_install_required_dict = repository_util.get_prior_import_or_install_required_dict( self.app, + tsr_ids, + repo_info_dicts ) processed_tsr_ids = [] while len( processed_tsr_ids ) != len( prior_install_required_dict.keys() ): tsr_id = suc.get_next_prior_import_or_install_required_dict_entry( prior_install_required_dict, diff --git a/lib/tool_shed/galaxy_install/installed_repository_manager.py b/lib/tool_shed/galaxy_install/installed_repository_manager.py index f9f942424db..76f42eeb2bf 100644 --- a/lib/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/tool_shed/galaxy_install/installed_repository_manager.py @@ -10,6 +10,7 @@ from sqlalchemy import and_, false, true from galaxy import util from tool_shed.util import common_util from tool_shed.util import container_util +from tool_shed.util import repository_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util from tool_shed.util import xml_util @@ -244,7 +245,7 @@ class InstalledRepositoryManager( object ): name = repo_info_dict.keys()[ 0 ] repo_info_tuple = repo_info_dict[ name ] description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \ - suc.get_repo_info_tuple_contents( repo_info_tuple ) + repository_util.get_repo_info_tuple_contents( repo_info_tuple ) if tool_dependencies: if not includes_tool_dependencies: includes_tool_dependencies = True @@ -256,11 +257,11 @@ class InstalledRepositoryManager( object ): if repository_dependencies: # We have a repository with one or more defined repository dependencies. if not repository: - repository = suc.get_repository_for_dependency_relationship( self.app, - tool_shed_url, - name, - repository_owner, - changeset_revision ) + repository = repository_util.get_repository_for_dependency_relationship( self.app, + tool_shed_url, + name, + repository_owner, + changeset_revision ) if not updating and repository and repository.metadata: installed_rd, missing_rd = self.get_installed_and_missing_repository_dependencies( repository ) else: @@ -282,7 +283,7 @@ class InstalledRepositoryManager( object ): for name, repo_info_tuple in rid.items(): description, repository_clone_url, changeset_revision, ctx_rev, \ repository_owner, rid_repository_dependencies, rid_tool_dependencies = \ - suc.get_repo_info_tuple_contents( repo_info_tuple ) + repository_util.get_repo_info_tuple_contents( repo_info_tuple ) if rid_tool_dependencies: for td_key, td_dict in rid_tool_dependencies.items(): if td_key not in required_tool_dependencies: @@ -403,7 +404,7 @@ class InstalledRepositoryManager( object ): missing_rd_tups = [] installed_rd_tups = [] ( description, repository_clone_url, changeset_revision, ctx_rev, - repository_owner, repository_dependencies, tool_dependencies ) = suc.get_repo_info_tuple_contents( repo_info_tuple ) + repository_owner, repository_dependencies, tool_dependencies ) = repository_util.get_repo_info_tuple_contents( repo_info_tuple ) if repository_dependencies: description = repository_dependencies[ 'description' ] root_key = repository_dependencies[ 'root_key' ] @@ -422,11 +423,11 @@ class InstalledRepositoryManager( object ): # repository_dependencies, installed_td ) tmp_clone_url = common_util.generate_clone_url_from_repo_info_tup( self.app, rd_tup ) tmp_repo_info_tuple = ( None, tmp_clone_url, changeset_revision, None, owner, None, None ) - repository, installed_changeset_revision = suc.repository_was_previously_installed( self.app, - tool_shed, - name, - tmp_repo_info_tuple, - from_tip=False ) + repository, installed_changeset_revision = repository_util.repository_was_previously_installed( self.app, + tool_shed, + name, + tmp_repo_info_tuple, + from_tip=False ) if repository: new_rd_tup = [ tool_shed, name, diff --git a/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py b/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py index e03e07ffce3..bfa7cc7f6fa 100644 --- a/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py +++ b/lib/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py @@ -10,7 +10,6 @@ from tool_shed.galaxy_install.tools import tool_panel_manager from tool_shed.metadata import metadata_generator from tool_shed.util import common_util from tool_shed.util import repository_util -from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_util from tool_shed.util import xml_util @@ -162,7 +161,7 @@ class InstalledRepositoryMetadataManager( metadata_generator.MetadataGenerator ) owner = self.repository.owner if not owner: cleaned_repository_clone_url = common_util.remove_protocol_and_user_from_clone_url( clone_url ) - owner = suc.get_repository_owner( cleaned_repository_clone_url ) + owner = repository_util.get_repository_owner( cleaned_repository_clone_url ) guid_to_tool_elem_dict = {} for tool_config_filename, guid, tool in repository_tools_tups: guid_to_tool_elem_dict[ guid ] = self.tpm.generate_tool_elem( tool_shed, diff --git a/lib/tool_shed/galaxy_install/repair_repository_manager.py b/lib/tool_shed/galaxy_install/repair_repository_manager.py index bb84241f8e8..e59c0c7fbed 100644 --- a/lib/tool_shed/galaxy_install/repair_repository_manager.py +++ b/lib/tool_shed/galaxy_install/repair_repository_manager.py @@ -30,22 +30,22 @@ class RepairRepositoryManager(): # rd_key is something like: 'http://localhost:9009__ESEP__package_rdkit_2012_12__ESEP__test__ESEP__d635ffb9c665__ESEP__True' # rd_val is something like: [['http://localhost:9009', 'package_numpy_1_7', 'test', 'cddd64ecd985', 'True']] repository_components_tuple = container_util.get_components_from_key( rd_key ) - components_list = suc.extract_components_from_tuple( repository_components_tuple ) + components_list = repository_util.extract_components_from_tuple( repository_components_tuple ) tool_shed, name, owner, changeset_revision = components_list[ 0:4 ] - installed_repository = suc.get_installed_repository( self.app, - tool_shed=tool_shed, - name=name, - owner=owner, - changeset_revision=changeset_revision ) + installed_repository = repository_util.get_installed_repository( self.app, + tool_shed=tool_shed, + name=name, + owner=owner, + changeset_revision=changeset_revision ) if ( installed_repository ) and ( installed_repository not in installed_repositories ): installed_repositories.append( installed_repository ) for rd_val in rd_vals: tool_shed, name, owner, changeset_revision = rd_val[ 0:4 ] - installed_repository = suc.get_installed_repository( self.app, - tool_shed=tool_shed, - name=name, - owner=owner, - changeset_revision=changeset_revision ) + installed_repository = repository_util.get_installed_repository( self.app, + tool_shed=tool_shed, + name=name, + owner=owner, + changeset_revision=changeset_revision ) if ( installed_repository ) and ( installed_repository not in installed_repositories ): installed_repositories.append( installed_repository ) return installed_repositories @@ -180,13 +180,13 @@ class RepairRepositoryManager(): else: # The tools will be loaded outside of any sections in the tool panel. tool_panel_section_key = None - suc.set_repository_attributes( self.app, - repository, - status=self.app.install_model.ToolShedRepository.installation_status.NEW, - error_message=None, - deleted=False, - uninstalled=False, - remove_from_disk=True ) + repository_util.set_repository_attributes( self.app, + repository, + status=self.app.install_model.ToolShedRepository.installation_status.NEW, + error_message=None, + deleted=False, + uninstalled=False, + remove_from_disk=True ) irm = install_manager.InstallRepositoryManager( self.app, tpm ) irm.install_tool_shed_repository( repository, repo_info_dict, diff --git a/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py b/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py index 6108a15272b..5730739a319 100644 --- a/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py +++ b/lib/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py @@ -15,6 +15,7 @@ from tool_shed.galaxy_install.tools import tool_panel_manager from tool_shed.util import common_util from tool_shed.util import container_util from tool_shed.util import encoding_util +from tool_shed.util import repository_util from tool_shed.util import shed_util_common as suc log = logging.getLogger( __name__ ) @@ -44,14 +45,14 @@ class RepositoryDependencyInstallManager( object ): repository_owner, \ repository_dependencies, \ tool_dependencies = \ - suc.get_repo_info_tuple_contents( repo_info_tuple ) + repository_util.get_repo_info_tuple_contents( repo_info_tuple ) if repository_dependencies: for key, val in repository_dependencies.items(): if key in [ 'root_key', 'description' ]: continue d_repository = None repository_components_tuple = container_util.get_components_from_key( key ) - components_list = suc.extract_components_from_tuple( repository_components_tuple ) + components_list = repository_util.extract_components_from_tuple( repository_components_tuple ) d_toolshed, d_name, d_owner, d_changeset_revision = components_list[ 0:4 ] for tsr in tool_shed_repositories: # Get the the tool_shed_repository defined by name, owner and changeset_revision. This is @@ -165,10 +166,10 @@ class RepositoryDependencyInstallManager( object ): repository_owner, \ repository_dependencies, \ tool_dependencies = \ - suc.get_repo_info_tuple_contents( repo_info_tuple ) + repository_util.get_repo_info_tuple_contents( repo_info_tuple ) # See if the repository has an existing record in the database. repository_db_record, installed_changeset_revision = \ - suc.repository_was_previously_installed( self.app, tool_shed_url, name, repo_info_tuple, from_tip=False ) + repository_util.repository_was_previously_installed( self.app, tool_shed_url, name, repo_info_tuple, from_tip=False ) if repository_db_record: if repository_db_record.status in [ install_model.ToolShedRepository.installation_status.INSTALLED, install_model.ToolShedRepository.installation_status.CLONING, @@ -199,7 +200,7 @@ class RepositoryDependencyInstallManager( object ): # No additional updates to the database record are necessary. can_update_db_record = False elif repository_db_record.status not in [ install_model.ToolShedRepository.installation_status.NEW ]: - # Set changeset_revision here so suc.create_or_update_tool_shed_repository will find + # Set changeset_revision here so repository_util.create_or_update_tool_shed_repository will find # the previously installed and uninstalled repository instead of creating a new record. changeset_revision = repository_db_record.installed_changeset_revision self.reset_previously_installed_repository( repository_db_record ) @@ -226,17 +227,17 @@ class RepositoryDependencyInstallManager( object ): tool_panel_section_id=tool_panel_section_id, new_tool_panel_section_label=new_tool_panel_section_label ) tool_shed_repository = \ - suc.create_or_update_tool_shed_repository( app=self.app, - name=name, - description=description, - installed_changeset_revision=installed_changeset_revision, - ctx_rev=ctx_rev, - repository_clone_url=repository_clone_url, - metadata_dict={}, - status=install_model.ToolShedRepository.installation_status.NEW, - current_changeset_revision=changeset_revision, - owner=repository_owner, - dist_to_shed=False ) + repository_util.create_or_update_tool_shed_repository( app=self.app, + name=name, + description=description, + installed_changeset_revision=installed_changeset_revision, + ctx_rev=ctx_rev, + repository_clone_url=repository_clone_url, + metadata_dict={}, + status=install_model.ToolShedRepository.installation_status.NEW, + current_changeset_revision=changeset_revision, + owner=repository_owner, + dist_to_shed=False ) if tool_shed_repository not in all_created_or_updated_tool_shed_repositories: all_created_or_updated_tool_shed_repositories.append( tool_shed_repository ) # Only append the tool shed repository to the list of created_or_updated_tool_shed_repositories if @@ -262,22 +263,22 @@ class RepositoryDependencyInstallManager( object ): # We store the port in the database. tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) # This method is used only in Galaxy, not the tool shed. - repository = suc.get_repository_for_dependency_relationship( self.app, tool_shed, name, owner, changeset_revision ) + repository = repository_util.get_repository_for_dependency_relationship( self.app, tool_shed, name, owner, changeset_revision ) if not repository: tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, tool_shed ) repository_clone_url = os.path.join( tool_shed_url, 'repos', owner, name ) ctx_rev = suc.get_ctx_rev( self.app, tool_shed_url, name, owner, changeset_revision ) - repository = suc.create_or_update_tool_shed_repository( app=self.app, - name=name, - description=None, - installed_changeset_revision=changeset_revision, - ctx_rev=ctx_rev, - repository_clone_url=repository_clone_url, - metadata_dict={}, - status=install_model.ToolShedRepository.installation_status.NEW, - current_changeset_revision=None, - owner=owner, - dist_to_shed=False ) + repository = repository_util.create_or_update_tool_shed_repository( app=self.app, + name=name, + description=None, + installed_changeset_revision=changeset_revision, + ctx_rev=ctx_rev, + repository_clone_url=repository_clone_url, + metadata_dict={}, + status=install_model.ToolShedRepository.installation_status.NEW, + current_changeset_revision=None, + owner=owner, + dist_to_shed=False ) return repository def get_repository_dependencies_for_installed_tool_shed_repository( self, app, repository ): @@ -331,13 +332,13 @@ class RepositoryDependencyInstallManager( object ): repository_owner, \ repository_dependencies, \ tool_dependencies = \ - suc.get_repo_info_tuple_contents( repo_info_tup ) + repository_util.get_repo_info_tuple_contents( repo_info_tup ) if repository_dependencies: for key, val in repository_dependencies.items(): if key in [ 'root_key', 'description' ]: continue repository_components_tuple = container_util.get_components_from_key( key ) - components_list = suc.extract_components_from_tuple( repository_components_tuple ) + components_list = repository_util.extract_components_from_tuple( repository_components_tuple ) # Skip listing a repository dependency if it is required only to compile a tool dependency # defined for the dependent repository since in this case, the repository dependency is really # a dependency of the dependent repository's contained tool dependency, and only if that @@ -374,7 +375,7 @@ class RepositoryDependencyInstallManager( object ): encoded_required_repository_tups.append( encoding_util.encoding_sep.join( required_repository_tup ) ) encoded_required_repository_str = encoding_util.encoding_sep2.join( encoded_required_repository_tups ) encoded_required_repository_str = encoding_util.tool_shed_encode( encoded_required_repository_str ) - if suc.is_tool_shed_client( self.app ): + if repository_util.is_tool_shed_client( self.app ): # Handle secure / insecure Tool Shed URL protocol changes and port changes. tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, tool_shed_url ) pathspec = [ 'repository', 'get_required_repo_info_dict' ] diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/env_manager.py b/lib/tool_shed/galaxy_install/tool_dependencies/env_manager.py index ddcfe36d55f..9aa9de3c4a9 100644 --- a/lib/tool_shed/galaxy_install/tool_dependencies/env_manager.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/env_manager.py @@ -1,8 +1,8 @@ import logging import os -import tool_shed.util.shed_util_common as suc from tool_shed.util import common_util +from tool_shed.util import repository_util log = logging.getLogger( __name__ ) @@ -67,11 +67,11 @@ class EnvManager( object ): if toolshed and repository_name and repository_owner and changeset_revision: # The protocol is not stored, but the port is if it exists. toolshed = common_util.remove_protocol_from_tool_shed_url( toolshed ) - repository = suc.get_repository_for_dependency_relationship( self.app, - toolshed, - repository_name, - repository_owner, - changeset_revision ) + repository = repository_util.get_repository_for_dependency_relationship( self.app, + toolshed, + repository_name, + repository_owner, + changeset_revision ) if repository: for sub_elem in elem: tool_dependency_type = sub_elem.tag diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py index 517ccf4a956..2f906e60a4a 100644 --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py @@ -6,7 +6,8 @@ from galaxy.tools.deps.resolvers import INDETERMINATE_DEPENDENCY from galaxy.util import listify, url_get from tool_shed.util import basic_util from tool_shed.util import common_util -from tool_shed.util import shed_util_common as suc +from tool_shed.util import metadata_util +from tool_shed.util import repository_util from tool_shed.util import tool_dependency_util from tool_shed.util import xml_util @@ -372,10 +373,10 @@ class Repository( RecipeTag, SyncDatabase ): required_repository_name = elem.attrib[ 'name' ] required_repository_owner = elem.attrib[ 'owner' ] default_required_repository_changeset_revision = elem.attrib[ 'changeset_revision' ] - required_repository = suc.get_repository_for_dependency_relationship( self.app, tool_shed_url, - required_repository_name, - required_repository_owner, - default_required_repository_changeset_revision ) + required_repository = repository_util.get_repository_for_dependency_relationship( self.app, tool_shed_url, + required_repository_name, + required_repository_owner, + default_required_repository_changeset_revision ) tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed_url ) tmp_filename = None if required_repository: @@ -436,7 +437,7 @@ class Repository( RecipeTag, SyncDatabase ): message = "Unable to locate the repository directory for revision %s of installed repository %s owned by %s." % \ ( str( required_repository.changeset_revision ), str( required_repository.name ), str( required_repository.owner ) ) raise Exception( message ) - tool_dependencies_config = suc.get_absolute_path_to_file_in_repository( repo_files_dir, 'tool_dependencies.xml' ) + tool_dependencies_config = repository_util.get_absolute_path_to_file_in_repository( repo_files_dir, 'tool_dependencies.xml' ) if tool_dependencies_config: config_to_use = tool_dependencies_config else: @@ -446,11 +447,11 @@ class Repository( RecipeTag, SyncDatabase ): else: # Make a call to the tool shed to get the changeset revision to which the current value of required_repository_changeset_revision # should be updated if it's not current. - text = suc.get_updated_changeset_revisions_from_tool_shed( app=self.app, - tool_shed_url=tool_shed, - name=required_repository_name, - owner=required_repository_owner, - changeset_revision=required_repository_changeset_revision ) + text = metadata_util.get_updated_changeset_revisions_from_tool_shed( app=self.app, + tool_shed_url=tool_shed, + name=required_repository_name, + owner=required_repository_owner, + changeset_revision=required_repository_changeset_revision ) if text: updated_changeset_revisions = listify( text ) # The list of changeset revisions is in reverse order, so the newest will be first. diff --git a/lib/tool_shed/galaxy_install/tool_migration_manager.py b/lib/tool_shed/galaxy_install/tool_migration_manager.py index 2d75d940f1f..39c33470230 100644 --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -25,6 +25,7 @@ from tool_shed.tools import tool_version_manager from tool_shed.util import basic_util from tool_shed.util import common_util from tool_shed.util import hg_util +from tool_shed.util import repository_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util @@ -173,17 +174,17 @@ class ToolMigrationManager( object ): if not self.__iscloned( clone_dir ): repository_clone_url = os.path.join( self.tool_shed_url, 'repos', owner, name ) ctx_rev = suc.get_ctx_rev( self.app, self.tool_shed_url, name, owner, changeset_revision ) - tool_shed_repository = suc.create_or_update_tool_shed_repository( app=self.app, - name=name, - description=description, - installed_changeset_revision=changeset_revision, - ctx_rev=ctx_rev, - repository_clone_url=repository_clone_url, - metadata_dict={}, - status=self.app.install_model.ToolShedRepository.installation_status.NEW, - current_changeset_revision=None, - owner=self.repository_owner, - dist_to_shed=True ) + tool_shed_repository = repository_util.create_or_update_tool_shed_repository( app=self.app, + name=name, + description=description, + installed_changeset_revision=changeset_revision, + ctx_rev=ctx_rev, + repository_clone_url=repository_clone_url, + metadata_dict={}, + status=self.app.install_model.ToolShedRepository.installation_status.NEW, + current_changeset_revision=None, + owner=self.repository_owner, + dist_to_shed=True ) return tool_shed_repository return None diff --git a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py index 2db27488dc9..204ed0f2749 100644 --- a/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -5,6 +5,7 @@ from xml.etree import ElementTree as XmlET from tool_shed.util import basic_util from tool_shed.util import common_util +from tool_shed.util import repository_util from tool_shed.util import shed_util_common as suc from tool_shed.util import xml_util @@ -220,7 +221,7 @@ class ToolPanelManager( object ): tool_elem = None cleaned_repository_clone_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) if not owner: - owner = suc.get_repository_owner( cleaned_repository_clone_url ) + owner = repository_util.get_repository_owner( cleaned_repository_clone_url ) tool_shed = cleaned_repository_clone_url.split( '/repos/' )[ 0 ].rstrip( '/' ) for guid, tool_section_dicts in tool_panel_dict.items(): for tool_section_dict in tool_section_dicts: diff --git a/lib/tool_shed/galaxy_install/update_repository_manager.py b/lib/tool_shed/galaxy_install/update_repository_manager.py index 3861bb4e2cf..6f487aaf5f0 100644 --- a/lib/tool_shed/galaxy_install/update_repository_manager.py +++ b/lib/tool_shed/galaxy_install/update_repository_manager.py @@ -9,6 +9,7 @@ from sqlalchemy import false import tool_shed.util.shed_util_common as suc from tool_shed.util import common_util from tool_shed.util import encoding_util +from tool_shed.util import repository_util from galaxy import util log = logging.getLogger( __name__ ) @@ -84,7 +85,7 @@ class UpdateRepositoryManager( object ): # has been deprecated in the Tool Shed. for repository in self.context.query( self.app.install_model.ToolShedRepository ) \ .filter( self.app.install_model.ToolShedRepository.table.c.deleted == false() ): - tool_shed_status_dict = suc.get_tool_shed_status_for_installed_repository( self.app, repository ) + tool_shed_status_dict = repository_util.get_tool_shed_status_for_installed_repository( self.app, repository ) if tool_shed_status_dict: if tool_shed_status_dict != repository.tool_shed_status: repository.tool_shed_status = tool_shed_status_dict @@ -115,7 +116,7 @@ class UpdateRepositoryManager( object ): repository.changeset_revision = updated_changeset_revision repository.ctx_rev = updated_ctx_rev # Update the repository.tool_shed_status column in the database. - tool_shed_status_dict = suc.get_tool_shed_status_for_installed_repository( self.app, repository ) + tool_shed_status_dict = repository_util.get_tool_shed_status_for_installed_repository( self.app, repository ) if tool_shed_status_dict: repository.tool_shed_status = tool_shed_status_dict else: diff --git a/lib/tool_shed/grids/repository_grids.py b/lib/tool_shed/grids/repository_grids.py index b90f555938f..4d2fc88f726 100644 --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -9,7 +9,7 @@ import tool_shed.repository_types.util as rt_util import tool_shed.util.shed_util_common as suc from galaxy.web.framework.helpers import grids from galaxy.webapps.tool_shed import model -from tool_shed.util import hg_util, metadata_util +from tool_shed.util import hg_util, metadata_util, repository_util log = logging.getLogger( __name__ ) @@ -1007,7 +1007,7 @@ class RepositoryDependenciesGrid( RepositoryMetadataGrid ): for rd_tup in sorted_rd_tups: name, owner, changeset_revision = rd_tup[1:4] rd_line = '' - required_repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + required_repository = repository_util.get_repository_by_name_and_owner( trans.app, name, owner ) if required_repository and not required_repository.deleted: required_repository_id = trans.security.encode_id( required_repository.id ) required_repository_metadata = \ @@ -1020,9 +1020,9 @@ class RepositoryDependenciesGrid( RepositoryMetadataGrid ): repo_path=None, create=False ) updated_changeset_revision = \ - suc.get_next_downloadable_changeset_revision( required_repository, - repo, - changeset_revision ) + metadata_util.get_next_downloadable_changeset_revision( required_repository, + repo, + changeset_revision ) required_repository_metadata = \ metadata_util.get_repository_metadata_by_repository_id_changeset_revision( trans.app, required_repository_id, diff --git a/lib/tool_shed/grids/util.py b/lib/tool_shed/grids/util.py index 03201e15f68..04e1b74bbd6 100644 --- a/lib/tool_shed/grids/util.py +++ b/lib/tool_shed/grids/util.py @@ -1,7 +1,7 @@ import logging from galaxy.web.form_builder import SelectField -from tool_shed.util import hg_util, metadata_util, shed_util_common as suc +from tool_shed.util import hg_util, metadata_util log = logging.getLogger( __name__ ) @@ -118,7 +118,7 @@ def get_latest_downloadable_repository_metadata( trans, repository ): tip_ctx = str( repo.changectx( repo.changelog.tip() ) ) repository_metadata = None try: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, encoded_repository_id, tip_ctx ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, encoded_repository_id, tip_ctx ) if repository_metadata is not None and repository_metadata.downloadable: return repository_metadata return None @@ -129,9 +129,9 @@ def get_latest_downloadable_repository_metadata( trans, repository ): downloadable=True ) if latest_downloadable_revision == hg_util.INITIAL_CHANGELOG_HASH: return None - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, - encoded_repository_id, - latest_downloadable_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, + encoded_repository_id, + latest_downloadable_revision ) if repository_metadata is not None and repository_metadata.downloadable: return repository_metadata return None @@ -159,7 +159,7 @@ def get_latest_repository_metadata( trans, repository ): repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False ) tip_ctx = str( repo.changectx( repo.changelog.tip() ) ) try: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, encoded_repository_id, tip_ctx ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, encoded_repository_id, tip_ctx ) return repository_metadata except: latest_downloadable_revision = metadata_util.get_previous_metadata_changeset_revision( repository, @@ -168,9 +168,9 @@ def get_latest_repository_metadata( trans, repository ): downloadable=False ) if latest_downloadable_revision == hg_util.INITIAL_CHANGELOG_HASH: return None - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, - encoded_repository_id, - latest_downloadable_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( trans.app, + encoded_repository_id, + latest_downloadable_revision ) return repository_metadata diff --git a/lib/tool_shed/metadata/metadata_generator.py b/lib/tool_shed/metadata/metadata_generator.py index 65f3bfe47ff..2b3b76bf668 100644 --- a/lib/tool_shed/metadata/metadata_generator.py +++ b/lib/tool_shed/metadata/metadata_generator.py @@ -16,6 +16,8 @@ from tool_shed.util import basic_util from tool_shed.util import common_util from tool_shed.util import hg_util from tool_shed.util import readme_util +from tool_shed.util import metadata_util +from tool_shed.util import repository_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_dependency_util from tool_shed.util import tool_util @@ -903,30 +905,30 @@ class MetadataGenerator( object ): # the changeset_revision defined in the repository_elem (it may be outdated). If # we're successful in locating an installed repository with the attributes defined # in the repository_elem, we know it is valid. - repository = suc.get_repository_for_dependency_relationship( self.app, - toolshed, - name, - owner, - changeset_revision ) + repository = repository_util.get_repository_for_dependency_relationship( self.app, + toolshed, + name, + owner, + changeset_revision ) if repository: return repository_dependency_tup, is_valid, error_message else: # Send a request to the tool shed to retrieve appropriate additional changeset # revisions with which the repository # may have been installed. - text = suc.get_updated_changeset_revisions_from_tool_shed( self.app, - toolshed, - name, - owner, - changeset_revision ) - if text: - updated_changeset_revisions = util.listify( text ) - for updated_changeset_revision in updated_changeset_revisions: - repository = suc.get_repository_for_dependency_relationship( self.app, + text = metadata_util.get_updated_changeset_revisions_from_tool_shed( self.app, toolshed, name, owner, - updated_changeset_revision ) + changeset_revision ) + if text: + updated_changeset_revisions = util.listify( text ) + for updated_changeset_revision in updated_changeset_revisions: + repository = repository_util.get_repository_for_dependency_relationship( self.app, + toolshed, + name, + owner, + updated_changeset_revision ) if repository: return repository_dependency_tup, is_valid, error_message if self.updating_installed_repository: diff --git a/lib/tool_shed/metadata/repository_metadata_manager.py b/lib/tool_shed/metadata/repository_metadata_manager.py index c2584dc88ab..543b70e8057 100644 --- a/lib/tool_shed/metadata/repository_metadata_manager.py +++ b/lib/tool_shed/metadata/repository_metadata_manager.py @@ -14,6 +14,7 @@ from tool_shed.util import basic_util from tool_shed.util import common_util from tool_shed.util import hg_util from tool_shed.util import metadata_util +from tool_shed.util import repository_util from tool_shed.util import shed_util_common as suc from tool_shed.util import tool_util @@ -340,7 +341,7 @@ class RepositoryMetadataManager( metadata_generator.MetadataGenerator ): repository_dependencies_dict = metadata_dict.get( 'repository_dependencies', {} ) repository_dependencies = repository_dependencies_dict.get( 'repository_dependencies', [] ) has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td = \ - suc.get_repository_dependency_types( repository_dependencies ) + repository_util.get_repository_dependency_types( repository_dependencies ) if 'datatypes' in metadata_dict: includes_datatypes = True if 'tools' in metadata_dict: @@ -356,9 +357,9 @@ class RepositoryMetadataManager( metadata_generator.MetadataGenerator ): downloadable = True else: downloadable = False - repository_metadata = suc.get_repository_metadata_by_changeset_revision( self.app, - self.app.security.encode_id( self.repository.id ), - changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( self.app, + self.app.security.encode_id( self.repository.id ), + changeset_revision ) if repository_metadata: repository_metadata.metadata = metadata_dict repository_metadata.downloadable = downloadable @@ -400,7 +401,7 @@ class RepositoryMetadataManager( metadata_generator.MetadataGenerator ): cleaned_tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) if cleaned_rd_tool_shed == cleaned_tool_shed and rd_name == name and rd_owner == owner: # Determine if the repository represented by the dependency tuple is an instance of the repository type TipOnly. - required_repository = suc.get_repository_by_name_and_owner( self.app, name, owner ) + required_repository = repository_util.get_repository_by_name_and_owner( self.app, name, owner ) repository_type_class = self.app.repository_types_registry.get_class_by_label( required_repository.type ) return isinstance( repository_type_class, TipOnly ) return False @@ -410,7 +411,7 @@ class RepositoryMetadataManager( metadata_generator.MetadataGenerator ): # Compare from most recent to oldest. changeset_revisions.reverse() for changeset_revision in changeset_revisions: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( self.app, id, changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( self.app, id, changeset_revision ) metadata = repository_metadata.metadata tools_dicts = metadata.get( 'tools', [] ) for tool_dict in tools_dicts: @@ -823,9 +824,9 @@ class RepositoryMetadataManager( metadata_generator.MetadataGenerator ): changeset_revisions_that_contain_tools = [] for changeset in repo.changelog: changeset_revision = str( repo.changectx( changeset ) ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( self.app, - encoded_repository_id, - changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( self.app, + encoded_repository_id, + changeset_revision ) if repository_metadata: metadata = repository_metadata.metadata if metadata: @@ -836,9 +837,9 @@ class RepositoryMetadataManager( metadata_generator.MetadataGenerator ): # { 'tool id' : 'parent tool id' } pairs for each tool in each changeset revision. for index, changeset_revision in enumerate( changeset_revisions_that_contain_tools ): tool_versions_dict = {} - repository_metadata = suc.get_repository_metadata_by_changeset_revision( self.app, - encoded_repository_id, - changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( self.app, + encoded_repository_id, + changeset_revision ) metadata = repository_metadata.metadata tool_dicts = metadata[ 'tools' ] if index == 0: @@ -873,7 +874,7 @@ class RepositoryMetadataManager( metadata_generator.MetadataGenerator ): unsuccessful_count = 0 for repository_id in repository_ids: try: - repository = suc.get_repository_in_tool_shed( self.app, repository_id ) + repository = repository_util.get_repository_in_tool_shed( self.app, repository_id ) self.set_repository( repository ) self.resetting_all_metadata_on_repository = True self.reset_all_metadata_on_repository_in_tool_shed() @@ -953,7 +954,7 @@ class RepositoryMetadataManager( metadata_generator.MetadataGenerator ): repository_dependencies_dict = self.metadata_dict.get( 'repository_dependencies', {} ) repository_dependencies = repository_dependencies_dict.get( 'repository_dependencies', [] ) has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td = \ - suc.get_repository_dependency_types( repository_dependencies ) + repository_util.get_repository_dependency_types( repository_dependencies ) repository_metadata.has_repository_dependencies = has_repository_dependencies if 'tool_dependencies' in self.metadata_dict: repository_metadata.includes_tool_dependencies = True @@ -980,7 +981,7 @@ class RepositoryMetadataManager( metadata_generator.MetadataGenerator ): changeset_revisions = [] for changeset in repo.changelog: changeset_revision = str( repo.changectx( changeset ) ) - if suc.get_repository_metadata_by_changeset_revision( self.app, encoded_id, changeset_revision ): + if metadata_util.get_repository_metadata_by_changeset_revision( self.app, encoded_id, changeset_revision ): changeset_revisions.append( changeset_revision ) self.add_tool_versions( encoded_id, repository_metadata, changeset_revisions ) elif len( repo ) == 1 and not self.invalid_file_tups: diff --git a/lib/tool_shed/repository_registry.py b/lib/tool_shed/repository_registry.py index 0554a4e4bad..fce6b601da7 100644 --- a/lib/tool_shed/repository_registry.py +++ b/lib/tool_shed/repository_registry.py @@ -5,7 +5,7 @@ from sqlalchemy import and_, false, or_ import tool_shed.repository_types.util as rt_util from galaxy.webapps.tool_shed import model from tool_shed.util import hg_util -from tool_shed.util import shed_util_common as suc +from tool_shed.util import metadata_util log = logging.getLogger( __name__ ) @@ -163,12 +163,12 @@ class Registry( object ): return ( None, False ) repo = hg_util.get_repo_for_repository( self.app, repository=repository, repo_path=None, create=False ) # Get the latest installable changeset revision since that is all that is currently configured for testing. - latest_installable_changeset_revision = suc.get_latest_downloadable_changeset_revision( self.app, repository, repo ) + latest_installable_changeset_revision = metadata_util.get_latest_downloadable_changeset_revision( self.app, repository, repo ) if latest_installable_changeset_revision not in [ None, hg_util.INITIAL_CHANGELOG_HASH ]: encoded_repository_id = self.app.security.encode_id( repository.id ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( self.app, - encoded_repository_id, - latest_installable_changeset_revision ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( self.app, + encoded_repository_id, + latest_installable_changeset_revision ) if repository_metadata: # No repository_metadata. return ( latest_installable_changeset_revision, True ) @@ -264,9 +264,9 @@ class Registry( object ): is_valid = self.is_valid( repository ) encoded_repository_id = self.app.security.encode_id( repository.id ) tip_changeset_hash = repository.tip( self.app ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( self.app, - encoded_repository_id, - tip_changeset_hash ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( self.app, + encoded_repository_id, + tip_changeset_hash ) self.viewable_repositories_and_suites_by_category[ category_name ] += 1 if is_valid: self.viewable_valid_repositories_and_suites_by_category[ category_name ] += 1 diff --git a/lib/tool_shed/tools/tool_validator.py b/lib/tool_shed/tools/tool_validator.py index faff4f9d165..d7873ac7dfa 100644 --- a/lib/tool_shed/tools/tool_validator.py +++ b/lib/tool_shed/tools/tool_validator.py @@ -11,7 +11,7 @@ from tool_shed.tools import data_table_manager from tool_shed.util import basic_util from tool_shed.util import hg_util -from tool_shed.util import shed_util_common as suc +from tool_shed.util import repository_util from tool_shed.util import tool_util from tool_shed.util import xml_util @@ -244,13 +244,13 @@ class ToolValidator( object ): searching backwards. """ original_tool_data_path = self.app.config.tool_data_path - repository = suc.get_repository_in_tool_shed( self.app, repository_id ) + repository = repository_util.get_repository_in_tool_shed( self.app, repository_id ) repo_files_dir = repository.repo_path( self.app ) repo = hg_util.get_repo_for_repository( self.app, repository=None, repo_path=repo_files_dir, create=False ) message = '' tool = None can_use_disk_file = False - tool_config_filepath = suc.get_absolute_path_to_file_in_repository( repo_files_dir, tool_config_filename ) + tool_config_filepath = repository_util.get_absolute_path_to_file_in_repository( repo_files_dir, tool_config_filename ) work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-ltfcr" ) can_use_disk_file = self.can_use_tool_config_disk_file( repository, repo, diff --git a/lib/tool_shed/tools/tool_version_manager.py b/lib/tool_shed/tools/tool_version_manager.py index 8041d43dfbf..6b9c593e86c 100644 --- a/lib/tool_shed/tools/tool_version_manager.py +++ b/lib/tool_shed/tools/tool_version_manager.py @@ -3,7 +3,8 @@ import logging from sqlalchemy import and_, or_ from tool_shed.util import hg_util -from tool_shed.util import shed_util_common as suc +from tool_shed.util import metadata_util +from tool_shed.util import repository_util log = logging.getLogger( __name__ ) @@ -36,7 +37,7 @@ class ToolVersionManager( object ): guid contained in the received repsitory_metadata.tool_versions. This function is called only from the Tool Shed. """ - repository = suc.get_repository_by_id( self.app, repository_id ) + repository = repository_util.get_repository_by_id( self.app, repository_id ) repo = hg_util.get_repo_for_repository( self.app, repository=repository, repo_path=None, create=False ) # Initialize the tool lineage version_lineage = [ guid ] @@ -44,7 +45,7 @@ class ToolVersionManager( object ): current_child_guid = guid for changeset in hg_util.reversed_upper_bounded_changelog( repo, repository_metadata.changeset_revision ): ctx = repo.changectx( changeset ) - rm = suc.get_repository_metadata_by_changeset_revision( self.app, repository_id, str( ctx ) ) + rm = metadata_util.get_repository_metadata_by_changeset_revision( self.app, repository_id, str( ctx ) ) if rm: parent_guid = rm.tool_versions.get( current_child_guid, None ) if parent_guid: @@ -56,7 +57,7 @@ class ToolVersionManager( object ): repository_metadata.changeset_revision, repository.tip( self.app ) ): ctx = repo.changectx( changeset ) - rm = suc.get_repository_metadata_by_changeset_revision( self.app, repository_id, str( ctx ) ) + rm = metadata_util.get_repository_metadata_by_changeset_revision( self.app, repository_id, str( ctx ) ) if rm: tool_versions = rm.tool_versions for child_guid, parent_guid in tool_versions.items(): diff --git a/lib/tool_shed/util/common_util.py b/lib/tool_shed/util/common_util.py index dde8cfb342f..0302227c312 100644 --- a/lib/tool_shed/util/common_util.py +++ b/lib/tool_shed/util/common_util.py @@ -242,6 +242,18 @@ def get_tool_shed_url_from_tool_shed_registry( app, tool_shed ): return None +def get_user_by_username( app, username ): + """Get a user from the database by username.""" + sa_session = app.model.context.current + try: + user = sa_session.query( app.model.User ) \ + .filter( app.model.User.table.c.username == username ) \ + .one() + return user + except Exception: + return None + + def handle_galaxy_url( trans, **kwd ): galaxy_url = kwd.get( 'galaxy_url', None ) if galaxy_url: diff --git a/lib/tool_shed/util/metadata_util.py b/lib/tool_shed/util/metadata_util.py index cf4f3361d61..f452f941e01 100644 --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -1,6 +1,10 @@ import logging -from tool_shed.util import common_util, hg_util, shed_util_common as suc +from tool_shed.util import common_util +from tool_shed.util import hg_util +from sqlalchemy import and_ +from operator import itemgetter +import tool_shed.util.repository_util log = logging.getLogger( __name__ ) @@ -35,27 +39,62 @@ def get_all_dependencies( app, metadata_entry, processed_dependency_links=[] ): return returned_dependencies +def get_current_repository_metadata_for_changeset_revision( app, repository, changeset_revision ): + encoded_repository_id = app.security.encode_id( repository.id ) + repository_metadata = get_repository_metadata_by_changeset_revision( app, + encoded_repository_id, + changeset_revision ) + if repository_metadata: + return repository_metadata + # The installable changeset_revision may have been changed because it was "moved ahead" + # in the repository changelog. + repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) + updated_changeset_revision = get_next_downloadable_changeset_revision( repository, + repo, + after_changeset_revision=changeset_revision ) + if updated_changeset_revision and updated_changeset_revision != changeset_revision: + repository_metadata = get_repository_metadata_by_changeset_revision( app, + encoded_repository_id, + updated_changeset_revision ) + if repository_metadata: + return repository_metadata + return None + + def get_dependencies_for_metadata_revision( app, metadata ): dependencies = [] for shed, name, owner, changeset, prior, _ in metadata[ 'repository_dependencies' ]: - required_repository = suc.get_repository_by_name_and_owner( app, name, owner ) + required_repository = tool_shed.util.repository_util.get_repository_by_name_and_owner( app, name, owner ) repo = hg_util.get_repo_for_repository( app, repository=required_repository, repo_path=None, create=False ) - updated_changeset = suc.get_next_downloadable_changeset_revision( required_repository, repo, changeset ) + updated_changeset = get_next_downloadable_changeset_revision( required_repository, repo, changeset ) if updated_changeset is None: continue - metadata_entry = suc.get_repository_metadata_by_changeset_revision( app, app.security.encode_id( required_repository.id ), updated_changeset ) + metadata_entry = get_repository_metadata_by_changeset_revision( app, app.security.encode_id( required_repository.id ), updated_changeset ) dependencies.append( metadata_entry ) return dependencies def get_latest_changeset_revision( app, repository, repo ): repository_tip = repository.tip( app ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, - app.security.encode_id( repository.id ), - repository_tip ) + repository_metadata = get_repository_metadata_by_changeset_revision( app, + app.security.encode_id( repository.id ), + repository_tip ) if repository_metadata and repository_metadata.downloadable: return repository_tip - changeset_revisions = [ revision[ 1 ] for revision in suc.get_metadata_revisions( repository, repo ) ] + changeset_revisions = [ revision[ 1 ] for revision in get_metadata_revisions( repository, repo ) ] + if changeset_revisions: + return changeset_revisions[ -1 ] + return hg_util.INITIAL_CHANGELOG_HASH + + +def get_latest_downloadable_changeset_revision( app, repository, repo=None ): + if repo is None: + repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) + repository_tip = repository.tip( app ) + repository_metadata = get_repository_metadata_by_changeset_revision( app, app.security.encode_id( repository.id ), repository_tip ) + if repository_metadata and repository_metadata.downloadable: + return repository_tip + changeset_revisions = [ revision[ 1 ] for revision in get_metadata_revisions( repository, repo ) ] if changeset_revisions: return changeset_revisions[ -1 ] return hg_util.INITIAL_CHANGELOG_HASH @@ -67,12 +106,12 @@ def get_latest_repository_metadata( app, decoded_repository_id, downloadable=Fal repository = sa_session.query( app.model.Repository ).get( decoded_repository_id ) repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) if downloadable: - changeset_revision = suc.get_latest_downloadable_changeset_revision( app, repository, repo ) + changeset_revision = get_latest_downloadable_changeset_revision( app, repository, repo ) else: changeset_revision = get_latest_changeset_revision( app, repository, repo ) - return suc.get_repository_metadata_by_changeset_revision( app, - app.security.encode_id( repository.id ), - changeset_revision ) + return get_repository_metadata_by_changeset_revision( app, + app.security.encode_id( repository.id ), + changeset_revision ) def get_metadata_by_id( app, metadata_id ): @@ -80,13 +119,72 @@ def get_metadata_by_id( app, metadata_id ): return sa_session.query( app.model.RepositoryMetadata ).filter( app.model.RepositoryMetadata.table.c.id == metadata_id ).one() +def get_metadata_changeset_revisions( repository, repo ): + """ + Return an unordered list of changeset_revisions and changeset numbers that are defined as installable. + """ + changeset_tups = [] + for repository_metadata in repository.downloadable_revisions: + ctx = hg_util.get_changectx_for_changeset( repo, repository_metadata.changeset_revision ) + if ctx: + rev = ctx.rev() + else: + rev = -1 + changeset_tups.append( ( rev, repository_metadata.changeset_revision ) ) + return sorted( changeset_tups ) + + +def get_metadata_revisions( repository, repo, sort_revisions=True, reverse=False, downloadable=True ): + """ + Return a list of changesets for the provided repository. + """ + if downloadable: + metadata_revisions = repository.downloadable_revisions + else: + metadata_revisions = repository.metadata_revisions + changeset_tups = [] + for repository_metadata in metadata_revisions: + ctx = hg_util.get_changectx_for_changeset( repo, repository_metadata.changeset_revision ) + if ctx: + rev = '%04d' % ctx.rev() + else: + rev = -1 + changeset_tups.append( ( rev, repository_metadata.changeset_revision ) ) + if sort_revisions: + changeset_tups.sort( key=itemgetter( 0 ), reverse=reverse ) + return changeset_tups + + +def get_next_downloadable_changeset_revision( repository, repo, after_changeset_revision ): + """ + Return the installable changeset_revision in the repository changelog after the changeset to which + after_changeset_revision refers. If there isn't one, return None. If there is only one installable + changeset, and that matches the requested revision, return it. + """ + changeset_revisions = [ revision[ 1 ] for revision in get_metadata_revisions( repository, repo ) ] + if len( changeset_revisions ) == 1: + changeset_revision = changeset_revisions[ 0 ] + if changeset_revision == after_changeset_revision: + return after_changeset_revision + found_after_changeset_revision = False + for changeset in repo.changelog: + changeset_revision = str( repo.changectx( changeset ) ) + if found_after_changeset_revision: + if changeset_revision in changeset_revisions: + return changeset_revision + elif not found_after_changeset_revision and changeset_revision == after_changeset_revision: + # We've found the changeset in the changelog for which we need to get the next downloadable changeset. + found_after_changeset_revision = True + return None + + def get_previous_metadata_changeset_revision( repository, repo, before_changeset_revision, downloadable=True ): """ Return the changeset_revision in the repository changelog that has associated metadata prior to the changeset to which before_changeset_revision refers. If there isn't one, return the hash value of an empty repository changelog, hg_util.INITIAL_CHANGELOG_HASH. """ - changeset_revisions = [ revision[ 1 ] for revision in suc.get_metadata_revisions( repository, repo ) ] + changeset_revisions = [ revision[ 1 ] for revision in get_metadata_revisions( repository, repo ) ] if len( changeset_revisions ) == 1: changeset_revision = changeset_revisions[ 0 ] if changeset_revision == before_changeset_revision: @@ -128,7 +226,7 @@ def get_repository_dependency_tups_from_repository_metadata( app, repository_met for repository_dependency_tup in repository_dependency_tups: toolshed, name, owner, changeset_revision, pir, oicct = \ common_util.parse_repository_dependency_tuple( repository_dependency_tup ) - repository = suc.get_repository_by_name_and_owner( app, name, owner ) + repository = tool_shed.util.repository_util.get_repository_by_name_and_owner( app, name, owner ) if repository: if deprecated_only: if repository.deprecated: @@ -141,6 +239,28 @@ def get_repository_dependency_tups_from_repository_metadata( app, repository_met return dependency_tups +def get_repository_metadata_by_changeset_revision( app, id, changeset_revision ): + """Get metadata for a specified repository change set from the database.""" + # Make sure there are no duplicate records, and return the single unique record for the changeset_revision. + # Duplicate records were somehow created in the past. The cause of this issue has been resolved, but we'll + # leave this method as is for a while longer to ensure all duplicate records are removed. + sa_session = app.model.context.current + all_metadata_records = sa_session.query( app.model.RepositoryMetadata ) \ + .filter( and_( app.model.RepositoryMetadata.table.c.repository_id == app.security.decode_id( id ), + app.model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) ) \ + .order_by( app.model.RepositoryMetadata.table.c.update_time.desc() ) \ + .all() + if len( all_metadata_records ) > 1: + # Delete all records older than the last one updated. + for repository_metadata in all_metadata_records[ 1: ]: + sa_session.delete( repository_metadata ) + sa_session.flush() + return all_metadata_records[ 0 ] + elif all_metadata_records: + return all_metadata_records[ 0 ] + return None + + def get_repository_metadata_by_id( app, id ): """Get repository metadata from the database""" sa_session = app.model.context.current @@ -150,11 +270,11 @@ def get_repository_metadata_by_id( app, id ): def get_repository_metadata_by_repository_id_changeset_revision( app, id, changeset_revision, metadata_only=False ): """Get a specified metadata record for a specified repository in the tool shed.""" if metadata_only: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, id, changeset_revision ) + repository_metadata = get_repository_metadata_by_changeset_revision( app, id, changeset_revision ) if repository_metadata and repository_metadata.metadata: return repository_metadata.metadata return None - return suc.get_repository_metadata_by_changeset_revision( app, id, changeset_revision ) + return get_repository_metadata_by_changeset_revision( app, id, changeset_revision ) def get_repository_metadata_revisions_for_review( repository, reviewed=True ): @@ -178,6 +298,28 @@ def get_repository_metadata_revisions_for_review( repository, reviewed=True ): return repository_metadata_revisions +def get_updated_changeset_revisions( app, name, owner, changeset_revision ): + """ + Return a string of comma-separated changeset revision hashes for all available updates to the received changeset + revision for the repository defined by the received name and owner. + """ + repository = tool_shed.util.repository_util.get_repository_by_name_and_owner( app, name, owner ) + repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) + # Get the upper bound changeset revision. + upper_bound_changeset_revision = get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) + # Build the list of changeset revision hashes defining each available update up to, but excluding + # upper_bound_changeset_revision. + changeset_hashes = [] + for changeset in hg_util.reversed_lower_upper_bounded_changelog( repo, changeset_revision, upper_bound_changeset_revision ): + # Make sure to exclude upper_bound_changeset_revision. + if changeset != upper_bound_changeset_revision: + changeset_hashes.append( str( repo.changectx( changeset ) ) ) + if changeset_hashes: + changeset_hashes_str = ','.join( changeset_hashes ) + return changeset_hashes_str + return '' + + def is_downloadable( metadata_dict ): # NOTE: although repository README files are considered Galaxy utilities, they have no # effect on determining if a revision is installable. See the comments in the @@ -202,7 +344,7 @@ def is_downloadable( metadata_dict ): def is_malicious( app, id, changeset_revision, **kwd ): """Check the malicious flag in repository metadata for a specified change set revision.""" - repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, id, changeset_revision ) + repository_metadata = get_repository_metadata_by_changeset_revision( app, id, changeset_revision ) if repository_metadata: return repository_metadata.malicious return False diff --git a/lib/tool_shed/util/readme_util.py b/lib/tool_shed/util/readme_util.py index 61442e3c1ed..d4364557094 100644 --- a/lib/tool_shed/util/readme_util.py +++ b/lib/tool_shed/util/readme_util.py @@ -8,7 +8,11 @@ from mako.template import Template import tool_shed.util.shed_util_common as suc from galaxy import web from galaxy.util import rst_to_html, unicodify, url_get -from tool_shed.util import basic_util, common_util, hg_util +from tool_shed.util import basic_util +from tool_shed.util import common_util +from tool_shed.util import hg_util +from tool_shed.util import metadata_util +from tool_shed.util import repository_util log = logging.getLogger( __name__ ) @@ -23,7 +27,7 @@ def build_readme_files_dict( app, repository, changeset_revision, metadata, tool can_use_disk_files = True else: repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) - latest_downloadable_changeset_revision = suc.get_latest_downloadable_changeset_revision( app, repository, repo ) + latest_downloadable_changeset_revision = metadata_util.get_latest_downloadable_changeset_revision( app, repository, repo ) can_use_disk_files = changeset_revision == latest_downloadable_changeset_revision readme_files_dict = {} if metadata: @@ -92,7 +96,7 @@ def get_readme_files_dict_for_display( app, tool_shed_url, repo_info_dict ): name = repo_info_dict.keys()[ 0 ] repo_info_tuple = repo_info_dict[ name ] description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td = \ - suc.get_repo_info_tuple_contents( repo_info_tuple ) + repository_util.get_repo_info_tuple_contents( repo_info_tuple ) # Handle changing HTTP protocols over time. tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url ) params = dict( name=name, owner=repository_owner, changeset_revision=changeset_revision ) diff --git a/lib/tool_shed/util/repository_util.py b/lib/tool_shed/util/repository_util.py index 59f95f7b0aa..db92c846f01 100644 --- a/lib/tool_shed/util/repository_util.py +++ b/lib/tool_shed/util/repository_util.py @@ -2,18 +2,22 @@ import ConfigParser import logging import os import re +import shutil +from urllib2 import HTTPError -from sqlalchemy import false +from sqlalchemy import false, and_, or_ from galaxy import util from galaxy import web from galaxy.web.form_builder import build_select_field +from tool_shed.util import basic_util +from tool_shed.util import common_util +from tool_shed.util import encoding_util +from tool_shed.util import hg_util +import tool_shed.util.metadata_util as metadata_util +from tool_shed.util.web_util import escape from galaxy.webapps.tool_shed.model import directory_hash_id from tool_shed.dependencies.repository import relation_builder -from tool_shed.util import common_util -from tool_shed.util import hg_util -from tool_shed.util import shed_util_common as suc -from tool_shed.util.web_util import escape log = logging.getLogger( __name__ ) @@ -44,9 +48,46 @@ def change_repository_name_in_hgrc_file( hgrc_file, new_name ): new_file.close() +def check_for_updates( app, model, repository_id=None ): + message = '' + status = 'ok' + if repository_id is None: + success_count = 0 + repository_names_not_updated = [] + updated_count = 0 + for repository in model.context.query( model.ToolShedRepository ) \ + .filter( model.ToolShedRepository.table.c.deleted == false() ): + ok, updated = \ + check_or_update_tool_shed_status_for_installed_repository( app, repository ) + if ok: + success_count += 1 + else: + repository_names_not_updated.append( '%s' % escape( str( repository.name ) ) ) + if updated: + updated_count += 1 + message = "Checked the status in the tool shed for %d repositories. " % success_count + message += "Updated the tool shed status for %d repositories. " % updated_count + if repository_names_not_updated: + message += "Unable to retrieve status from the tool shed for the following repositories:\n" + message += ", ".join( repository_names_not_updated ) + else: + repository = get_tool_shed_repository_by_id( app, repository_id ) + ok, updated = \ + check_or_update_tool_shed_status_for_installed_repository( app, repository ) + if ok: + if updated: + message = "The tool shed status for repository %s has been updated." % escape( str( repository.name ) ) + else: + message = "The status has not changed in the tool shed for repository %s." % escape( str( repository.name ) ) + else: + message = "Unable to retrieve status from the tool shed for repository %s." % escape( str( repository.name ) ) + status = 'error' + return message, status + + def check_or_update_tool_shed_status_for_installed_repository( app, repository ): updated = False - tool_shed_status_dict = suc.get_tool_shed_status_for_installed_repository( app, repository ) + tool_shed_status_dict = get_tool_shed_status_for_installed_repository( app, repository ) if tool_shed_status_dict: ok = True if tool_shed_status_dict != repository.tool_shed_status: @@ -59,6 +100,71 @@ def check_or_update_tool_shed_status_for_installed_repository( app, repository ) return ok, updated +def create_or_update_tool_shed_repository( app, name, description, installed_changeset_revision, ctx_rev, repository_clone_url, + metadata_dict, status, current_changeset_revision=None, owner='', dist_to_shed=False ): + """ + Update a tool shed repository record in the Galaxy database with the new information received. + If a record defined by the received tool shed, repository name and owner does not exist, create + a new record with the received information. + """ + # The received value for dist_to_shed will be True if the ToolMigrationManager is installing a repository + # that contains tools or datatypes that used to be in the Galaxy distribution, but have been moved + # to the main Galaxy tool shed. + if current_changeset_revision is None: + # The current_changeset_revision is not passed if a repository is being installed for the first + # time. If a previously installed repository was later uninstalled, this value should be received + # as the value of that change set to which the repository had been updated just prior to it being + # uninstalled. + current_changeset_revision = installed_changeset_revision + context = app.install_model.context + tool_shed = get_tool_shed_from_clone_url( repository_clone_url ) + if not owner: + owner = get_repository_owner_from_clone_url( repository_clone_url ) + includes_datatypes = 'datatypes' in metadata_dict + if status in [ app.install_model.ToolShedRepository.installation_status.DEACTIVATED ]: + deleted = True + uninstalled = False + elif status in [ app.install_model.ToolShedRepository.installation_status.UNINSTALLED ]: + deleted = True + uninstalled = True + else: + deleted = False + uninstalled = False + tool_shed_repository = \ + get_installed_repository( app, tool_shed=tool_shed, name=name, owner=owner, installed_changeset_revision=installed_changeset_revision ) + if tool_shed_repository: + log.debug( "Updating an existing row for repository '%s' in the tool_shed_repository table, status set to '%s'." % + ( str( name ), str( status ) ) ) + tool_shed_repository.description = description + tool_shed_repository.changeset_revision = current_changeset_revision + tool_shed_repository.ctx_rev = ctx_rev + tool_shed_repository.metadata = metadata_dict + tool_shed_repository.includes_datatypes = includes_datatypes + tool_shed_repository.deleted = deleted + tool_shed_repository.uninstalled = uninstalled + tool_shed_repository.status = status + else: + log.debug( "Adding new row for repository '%s' in the tool_shed_repository table, status set to '%s'." % + ( str( name ), str( status ) ) ) + tool_shed_repository = \ + app.install_model.ToolShedRepository( tool_shed=tool_shed, + name=name, + description=description, + owner=owner, + installed_changeset_revision=installed_changeset_revision, + changeset_revision=current_changeset_revision, + ctx_rev=ctx_rev, + metadata=metadata_dict, + includes_datatypes=includes_datatypes, + dist_to_shed=dist_to_shed, + deleted=deleted, + uninstalled=uninstalled, + status=status ) + context.add( tool_shed_repository ) + context.flush() + return tool_shed_repository + + def create_repo_info_dict( app, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_name=None, repository=None, repository_metadata=None, tool_dependencies=None, repository_dependencies=None ): """ @@ -84,10 +190,10 @@ def create_repo_info_dict( app, repository_clone_url, changeset_revision, ctx_re repository_dependencies will be None. """ repo_info_dict = {} - repository = suc.get_repository_by_name_and_owner( app, repository_name, repository_owner ) + repository = get_repository_by_name_and_owner( app, repository_name, repository_owner ) if app.name == 'tool_shed': # We're in the tool shed. - repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( app, app.security.encode_id( repository.id ), changeset_revision ) if repository_metadata: @@ -126,6 +232,26 @@ def create_repo_info_dict( app, repository_clone_url, changeset_revision, ctx_re return repo_info_dict +def create_repository_admin_role( app, repository ): + """ + Create a new role with name-spaced name based on the repository name and its owner's public user + name. This will ensure that the tole name is unique. + """ + sa_session = app.model.context.current + name = get_repository_admin_role_name( str( repository.name ), str( repository.user.username ) ) + description = 'A user or group member with this role can administer this repository.' + role = app.model.Role( name=name, description=description, type=app.model.Role.types.SYSTEM ) + sa_session.add( role ) + sa_session.flush() + # Associate the role with the repository owner. + app.model.UserRoleAssociation( repository.user, role ) + # Associate the role with the repository. + rra = app.model.RepositoryRoleAssociation( repository, role ) + sa_session.add( rra ) + sa_session.flush() + return role + + def create_repository( app, name, type, description, long_description, user_id, category_ids=[], remote_repository_url=None, homepage_url=None ): """Create a new ToolShed repository""" sa_session = app.model.context.current @@ -176,6 +302,651 @@ def create_repository( app, name, type, description, long_description, user_id, return repository, message +def extract_components_from_tuple( repository_components_tuple ): + '''Extract the repository components from the provided tuple in a backward-compatible manner.''' + toolshed = repository_components_tuple[ 0 ] + name = repository_components_tuple[ 1 ] + owner = repository_components_tuple[ 2 ] + changeset_revision = repository_components_tuple[ 3 ] + components_list = [ toolshed, name, owner, changeset_revision ] + if len( repository_components_tuple ) == 5: + toolshed, name, owner, changeset_revision, prior_installation_required = repository_components_tuple + components_list = [ toolshed, name, owner, changeset_revision, prior_installation_required ] + elif len( repository_components_tuple ) == 6: + toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = repository_components_tuple + components_list = [ toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td ] + return components_list + + +def generate_sharable_link_for_repository_in_tool_shed( repository, changeset_revision=None ): + """Generate the URL for sharing a repository that is in the tool shed.""" + base_url = web.url_for( '/', qualified=True ).rstrip( '/' ) + protocol, base = base_url.split( '://' ) + sharable_url = '%s://%s/view/%s/%s' % ( protocol, base, repository.user.username, repository.name ) + if changeset_revision: + sharable_url += '/%s' % changeset_revision + return sharable_url + + +def generate_tool_shed_repository_install_dir( repository_clone_url, changeset_revision ): + """ + Generate a repository installation directory that guarantees repositories with the same + name will always be installed in different directories. The tool path will be of the form: + /repos/// + """ + tmp_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) + # Now tmp_url is something like: bx.psu.edu:9009/repos/some_username/column + items = tmp_url.split( '/repos/' ) + tool_shed_url = items[ 0 ] + repo_path = items[ 1 ] + tool_shed_url = common_util.remove_port_from_tool_shed_url( tool_shed_url ) + return '/'.join( [ tool_shed_url, 'repos', repo_path, changeset_revision ] ) + + +def get_absolute_path_to_file_in_repository( repo_files_dir, file_name ): + """Return the absolute path to a specified disk file contained in a repository.""" + stripped_file_name = basic_util.strip_path( file_name ) + file_path = None + for root, dirs, files in os.walk( repo_files_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name == stripped_file_name: + return os.path.abspath( os.path.join( root, name ) ) + return file_path + + +def get_ids_of_tool_shed_repositories_being_installed( app, as_string=False ): + installing_repository_ids = [] + new_status = app.install_model.ToolShedRepository.installation_status.NEW + cloning_status = app.install_model.ToolShedRepository.installation_status.CLONING + setting_tool_versions_status = app.install_model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS + installing_dependencies_status = app.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES + loading_datatypes_status = app.install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES + for tool_shed_repository in \ + app.install_model.context.query( app.install_model.ToolShedRepository ) \ + .filter( or_( app.install_model.ToolShedRepository.status == new_status, + app.install_model.ToolShedRepository.status == cloning_status, + app.install_model.ToolShedRepository.status == setting_tool_versions_status, + app.install_model.ToolShedRepository.status == installing_dependencies_status, + app.install_model.ToolShedRepository.status == loading_datatypes_status ) ): + installing_repository_ids.append( app.security.encode_id( tool_shed_repository.id ) ) + if as_string: + return ','.join( installing_repository_ids ) + return installing_repository_ids + + +def get_installed_repository( app, tool_shed, name, owner, changeset_revision=None, installed_changeset_revision=None ): + """ + Return a tool shed repository database record defined by the combination of a toolshed, repository name, + repository owner and either current or originally installed changeset_revision. + """ + query = app.install_model.context.query( app.install_model.ToolShedRepository ) + # We store the port, if one exists, in the database. + tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) + clause_list = [ app.install_model.ToolShedRepository.table.c.tool_shed == tool_shed, + app.install_model.ToolShedRepository.table.c.name == name, + app.install_model.ToolShedRepository.table.c.owner == owner ] + if changeset_revision is not None: + clause_list.append( app.install_model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) + if installed_changeset_revision is not None: + clause_list.append( app.install_model.ToolShedRepository.table.c.installed_changeset_revision == installed_changeset_revision ) + return query.filter( and_( *clause_list ) ).first() + + +def get_installed_tool_shed_repository( app, id ): + """Get a tool shed repository record from the Galaxy database defined by the id.""" + rval = [] + if isinstance( id, list ): + return_list = True + else: + id = [ id ] + return_list = False + for i in id: + rval.append( app.install_model.context.query( app.install_model.ToolShedRepository ).get( app.security.decode_id( i ) ) ) + if return_list: + return rval + return rval[0] + + +def get_prior_import_or_install_required_dict( app, tsr_ids, repo_info_dicts ): + """ + This method is used in the Tool Shed when exporting a repository and its dependencies, + and in Galaxy when a repository and its dependencies are being installed. Return a + dictionary whose keys are the received tsr_ids and whose values are a list of tsr_ids, + each of which is contained in the received list of tsr_ids and whose associated repository + must be imported or installed prior to the repository associated with the tsr_id key. + """ + # Initialize the dictionary. + prior_import_or_install_required_dict = {} + for tsr_id in tsr_ids: + prior_import_or_install_required_dict[ tsr_id ] = [] + # Inspect the repository dependencies for each repository about to be installed and populate the dictionary. + for repo_info_dict in repo_info_dicts: + repository, repository_dependencies = get_repository_and_repository_dependencies_from_repo_info_dict( app, repo_info_dict ) + if repository: + encoded_repository_id = app.security.encode_id( repository.id ) + if encoded_repository_id in tsr_ids: + # We've located the database table record for one of the repositories we're about to install, so find out if it has any repository + # dependencies that require prior installation. + prior_import_or_install_ids = get_repository_ids_requiring_prior_import_or_install( app, tsr_ids, repository_dependencies ) + prior_import_or_install_required_dict[ encoded_repository_id ] = prior_import_or_install_ids + return prior_import_or_install_required_dict + + +def get_repo_info_dict( app, user, repository_id, changeset_revision ): + repository = get_repository_in_tool_shed( app, repository_id ) + repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) + repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( user, repository ) + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( app, + repository_id, + changeset_revision ) + if not repository_metadata: + # The received changeset_revision is no longer installable, so get the next changeset_revision + # in the repository's changelog. This generally occurs only with repositories of type + # repository_suite_definition or tool_dependency_definition. + next_downloadable_changeset_revision = \ + metadata_util.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) + if next_downloadable_changeset_revision and next_downloadable_changeset_revision != changeset_revision: + repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision( app, + repository_id, + next_downloadable_changeset_revision ) + if repository_metadata: + # For now, we'll always assume that we'll get repository_metadata, but if we discover our assumption + # is not valid we'll have to enhance the callers to handle repository_metadata values of None in the + # returned repo_info_dict. + metadata = repository_metadata.metadata + if 'tools' in metadata: + includes_tools = True + else: + includes_tools = False + includes_tools_for_display_in_tool_panel = repository_metadata.includes_tools_for_display_in_tool_panel + repository_dependencies_dict = metadata.get( 'repository_dependencies', {} ) + repository_dependencies = repository_dependencies_dict.get( 'repository_dependencies', [] ) + has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td = \ + get_repository_dependency_types( repository_dependencies ) + if 'tool_dependencies' in metadata: + includes_tool_dependencies = True + else: + includes_tool_dependencies = False + else: + # Here's where we may have to handle enhancements to the callers. See above comment. + includes_tools = False + has_repository_dependencies = False + has_repository_dependencies_only_if_compiling_contained_td = False + includes_tool_dependencies = False + includes_tools_for_display_in_tool_panel = False + ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) + repo_info_dict = create_repo_info_dict( app=app, + repository_clone_url=repository_clone_url, + changeset_revision=changeset_revision, + ctx_rev=str( ctx.rev() ), + repository_owner=repository.user.username, + repository_name=repository.name, + repository=repository, + repository_metadata=repository_metadata, + tool_dependencies=None, + repository_dependencies=None ) + return repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, \ + has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td + + +def get_repo_info_tuple_contents( repo_info_tuple ): + """Take care in handling the repo_info_tuple as it evolves over time as new tool shed features are introduced.""" + if len( repo_info_tuple ) == 6: + description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, tool_dependencies = repo_info_tuple + repository_dependencies = None + elif len( repo_info_tuple ) == 7: + description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = repo_info_tuple + return description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies + + +def get_repositories_by_category( app, category_id ): + sa_session = app.model.context.current + resultset = sa_session.query( app.model.Category ).get( category_id ) + repositories = [] + default_value_mapper = { 'id': app.security.encode_id, 'user_id': app.security.encode_id } + for row in resultset.repositories: + repository_dict = row.repository.to_dict( value_mapper=default_value_mapper ) + repository_dict[ 'metadata' ] = {} + for changeset, changehash in row.repository.installable_revisions( app ): + encoded_id = app.security.encode_id( row.repository.id ) + metadata = metadata_util.get_repository_metadata_by_changeset_revision( app, encoded_id, changehash ) + repository_dict[ 'metadata' ][ '%s:%s' % ( changeset, changehash ) ] = metadata.to_dict( value_mapper=default_value_mapper ) + repositories.append( repository_dict ) + return repositories + + +def get_repository_admin_role_name( repository_name, repository_owner ): + return '%s_%s_admin' % ( str( repository_name ), str( repository_owner ) ) + + +def get_repository_and_repository_dependencies_from_repo_info_dict( app, repo_info_dict ): + """Return a tool_shed_repository or repository record defined by the information in the received repo_info_dict.""" + repository_name = repo_info_dict.keys()[ 0 ] + repo_info_tuple = repo_info_dict[ repository_name ] + description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \ + get_repo_info_tuple_contents( repo_info_tuple ) + if hasattr( app, "install_model" ): + # In a tool shed client (Galaxy, or something install repositories like Galaxy) + tool_shed = get_tool_shed_from_clone_url( repository_clone_url ) + repository = get_repository_for_dependency_relationship( app, tool_shed, repository_name, repository_owner, changeset_revision ) + else: + # We're in the tool shed. + repository = get_repository_by_name_and_owner( app, repository_name, repository_owner ) + return repository, repository_dependencies + + +def get_repository_by_id( app, id ): + """Get a repository from the database via id.""" + if is_tool_shed_client( app ): + return app.install_model.context.query( app.install_model.ToolShedRepository ).get( app.security.decode_id( id ) ) + else: + sa_session = app.model.context.current + return sa_session.query( app.model.Repository ).get( app.security.decode_id( id ) ) + + +def get_repository_by_name_and_owner( app, name, owner ): + """Get a repository from the database via name and owner""" + repository_query = get_repository_query( app ) + if is_tool_shed_client( app ): + return repository_query \ + .filter( and_( app.install_model.ToolShedRepository.table.c.name == name, + app.install_model.ToolShedRepository.table.c.owner == owner ) ) \ + .first() + # We're in the tool shed. + user = common_util.get_user_by_username( app, owner ) + if user: + return repository_query \ + .filter( and_( app.model.Repository.table.c.name == name, + app.model.Repository.table.c.user_id == user.id ) ) \ + .first() + return None + + +def get_repository_by_name( app, name ): + """Get a repository from the database via name.""" + return get_repository_query( app ).filter_by( name=name ).first() + + +def get_repository_dependency_types( repository_dependencies ): + """ + Inspect the received list of repository_dependencies tuples and return boolean values + for has_repository_dependencies and has_repository_dependencies_only_if_compiling_contained_td. + """ + # Set has_repository_dependencies, which will be True only if at least one repository_dependency + # is defined with the value of + # only_if_compiling_contained_td as False. + has_repository_dependencies = False + for rd_tup in repository_dependencies: + tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( rd_tup ) + if not util.asbool( only_if_compiling_contained_td ): + has_repository_dependencies = True + break + # Set has_repository_dependencies_only_if_compiling_contained_td, which will be True only if at + # least one repository_dependency is defined with the value of only_if_compiling_contained_td as True. + has_repository_dependencies_only_if_compiling_contained_td = False + for rd_tup in repository_dependencies: + tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( rd_tup ) + if util.asbool( only_if_compiling_contained_td ): + has_repository_dependencies_only_if_compiling_contained_td = True + break + return has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td + + +def get_repository_for_dependency_relationship( app, tool_shed, name, owner, changeset_revision ): + """ + Return an installed tool_shed_repository database record that is defined by either the current changeset + revision or the installed_changeset_revision. + """ + # This method is used only in Galaxy, not the Tool Shed. We store the port (if one exists) in the database. + tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) + if tool_shed is None or name is None or owner is None or changeset_revision is None: + message = "Unable to retrieve the repository record from the database because one or more of the following " + message += "required parameters is None: tool_shed: %s, name: %s, owner: %s, changeset_revision: %s " % \ + ( str( tool_shed ), str( name ), str( owner ), str( changeset_revision ) ) + raise Exception( message ) + repository = get_installed_repository( app=app, + tool_shed=tool_shed, + name=name, + owner=owner, + installed_changeset_revision=changeset_revision ) + if not repository: + repository = get_installed_repository( app=app, + tool_shed=tool_shed, + name=name, + owner=owner, + changeset_revision=changeset_revision ) + if not repository: + tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed ) + repository_clone_url = os.path.join( tool_shed_url, 'repos', owner, name ) + repo_info_tuple = (None, repository_clone_url, changeset_revision, None, owner, None, None) + repository, pcr = repository_was_previously_installed( app, tool_shed_url, name, repo_info_tuple ) + if not repository: + # The received changeset_revision is no longer installable, so get the next changeset_revision + # in the repository's changelog in the tool shed that is associated with repository_metadata. + tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed ) + params = dict( name=name, owner=owner, changeset_revision=changeset_revision ) + pathspec = [ 'repository', 'next_installable_changeset_revision' ] + text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) + if text: + repository = get_installed_repository( app=app, + tool_shed=tool_shed, + name=name, + owner=owner, + changeset_revision=text ) + return repository + + +def get_repository_ids_requiring_prior_import_or_install( app, tsr_ids, repository_dependencies ): + """ + This method is used in the Tool Shed when exporting a repository and its dependencies, + and in Galaxy when a repository and its dependencies are being installed. Inspect the + received repository_dependencies and determine if the encoded id of each required + repository is in the received tsr_ids. If so, then determine whether that required + repository should be imported / installed prior to its dependent repository. Return a + list of encoded repository ids, each of which is contained in the received list of tsr_ids, + and whose associated repositories must be imported / installed prior to the dependent + repository associated with the received repository_dependencies. + """ + prior_tsr_ids = [] + if repository_dependencies: + for key, rd_tups in repository_dependencies.items(): + if key in [ 'description', 'root_key' ]: + continue + for rd_tup in rd_tups: + tool_shed, \ + name, \ + owner, \ + changeset_revision, \ + prior_installation_required, \ + only_if_compiling_contained_td = \ + common_util.parse_repository_dependency_tuple( rd_tup ) + # If only_if_compiling_contained_td is False, then the repository dependency + # is not required to be installed prior to the dependent repository even if + # prior_installation_required is True. This is because the only meaningful + # content of the repository dependency is its contained tool dependency, which + # is required in order to compile the dependent repository's tool dependency. + # In the scenario where the repository dependency is not installed prior to the + # dependent repository's tool dependency compilation process, the tool dependency + # compilation framework will install the repository dependency prior to compilation + # of the dependent repository's tool dependency. + if not util.asbool( only_if_compiling_contained_td ): + if util.asbool( prior_installation_required ): + if is_tool_shed_client( app ): + # We store the port, if one exists, in the database. + tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) + repository = get_repository_for_dependency_relationship( app, + tool_shed, + name, + owner, + changeset_revision ) + else: + repository = get_repository_by_name_and_owner( app, name, owner ) + if repository: + encoded_repository_id = app.security.encode_id( repository.id ) + if encoded_repository_id in tsr_ids: + prior_tsr_ids.append( encoded_repository_id ) + return prior_tsr_ids + + +def get_repository_in_tool_shed( app, id ): + """Get a repository on the tool shed side from the database via id.""" + return get_repository_query( app ).get( app.security.decode_id( id ) ) + + +def get_repository_owner( cleaned_repository_url ): + """Gvien a "cleaned" repository clone URL, return the owner of the repository.""" + items = cleaned_repository_url.split( '/repos/' ) + repo_path = items[ 1 ] + if repo_path.startswith( '/' ): + repo_path = repo_path.replace( '/', '', 1 ) + return repo_path.lstrip( '/' ).split( '/' )[ 0 ] + + +def get_repository_owner_from_clone_url( repository_clone_url ): + """Given a repository clone URL, return the owner of the repository.""" + tmp_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) + return get_repository_owner( tmp_url ) + + +def get_repository_query( app ): + if is_tool_shed_client( app ): + query = app.install_model.context.query( app.install_model.ToolShedRepository ) + else: + query = app.model.context.query( app.model.Repository ) + return query + + +def get_role_by_id( app, role_id ): + """Get a Role from the database by id.""" + sa_session = app.model.context.current + return sa_session.query( app.model.Role ).get( app.security.decode_id( role_id ) ) + + +def get_tool_shed_from_clone_url( repository_clone_url ): + tmp_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) + return tmp_url.split( '/repos/' )[ 0 ].rstrip( '/' ) + + +def get_tool_shed_repository_by_id( app, repository_id ): + """Return a tool shed repository database record defined by the id.""" + # This method is used only in Galaxy, not the tool shed. + return app.install_model.context.query( app.install_model.ToolShedRepository ) \ + .filter( app.install_model.ToolShedRepository.table.c.id == app.security.decode_id( repository_id ) ) \ + .first() + + +def get_tool_shed_repository_status_label( app, tool_shed_repository=None, name=None, owner=None, changeset_revision=None, repository_clone_url=None ): + """Return a color-coded label for the status of the received tool-shed_repository installed into Galaxy.""" + if tool_shed_repository is None: + if name is not None and owner is not None and repository_clone_url is not None: + tool_shed = get_tool_shed_from_clone_url( repository_clone_url ) + tool_shed_repository = get_installed_repository( app, + tool_shed=tool_shed, + name=name, + owner=owner, + installed_changeset_revision=changeset_revision ) + if tool_shed_repository: + status_label = tool_shed_repository.status + if tool_shed_repository.status in [ app.install_model.ToolShedRepository.installation_status.CLONING, + app.install_model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS, + app.install_model.ToolShedRepository.installation_status.INSTALLING_REPOSITORY_DEPENDENCIES, + app.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES, + app.install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES ]: + bgcolor = app.install_model.ToolShedRepository.states.INSTALLING + elif tool_shed_repository.status in [ app.install_model.ToolShedRepository.installation_status.NEW, + app.install_model.ToolShedRepository.installation_status.UNINSTALLED ]: + bgcolor = app.install_model.ToolShedRepository.states.UNINSTALLED + elif tool_shed_repository.status in [ app.install_model.ToolShedRepository.installation_status.ERROR ]: + bgcolor = app.install_model.ToolShedRepository.states.ERROR + elif tool_shed_repository.status in [ app.install_model.ToolShedRepository.installation_status.DEACTIVATED ]: + bgcolor = app.install_model.ToolShedRepository.states.WARNING + elif tool_shed_repository.status in [ app.install_model.ToolShedRepository.installation_status.INSTALLED ]: + if tool_shed_repository.repository_dependencies_being_installed: + bgcolor = app.install_model.ToolShedRepository.states.WARNING + status_label = '%s, %s' % ( status_label, + app.install_model.ToolShedRepository.installation_status.INSTALLING_REPOSITORY_DEPENDENCIES ) + elif tool_shed_repository.missing_repository_dependencies: + bgcolor = app.install_model.ToolShedRepository.states.WARNING + status_label = '%s, missing repository dependencies' % status_label + elif tool_shed_repository.tool_dependencies_being_installed: + bgcolor = app.install_model.ToolShedRepository.states.WARNING + status_label = '%s, %s' % ( status_label, + app.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) + elif tool_shed_repository.missing_tool_dependencies: + bgcolor = app.install_model.ToolShedRepository.states.WARNING + status_label = '%s, missing tool dependencies' % status_label + else: + bgcolor = app.install_model.ToolShedRepository.states.OK + else: + bgcolor = app.install_model.ToolShedRepository.states.ERROR + else: + bgcolor = app.install_model.ToolShedRepository.states.WARNING + status_label = 'unknown status' + return '
%s
' % ( bgcolor, status_label ) + + +def get_tool_shed_status_for_installed_repository( app, repository ): + """ + Send a request to the tool shed to retrieve information about newer installable repository revisions, + current revision updates, whether the repository revision is the latest downloadable revision, and + whether the repository has been deprecated in the tool shed. The received repository is a ToolShedRepository + object from Galaxy. + """ + tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, str( repository.tool_shed ) ) + params = dict( name=repository.name, owner=repository.owner, changeset_revision=repository.changeset_revision ) + pathspec = [ 'repository', 'status_for_installed_repository' ] + try: + encoded_tool_shed_status_dict = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) + tool_shed_status_dict = encoding_util.tool_shed_decode( encoded_tool_shed_status_dict ) + return tool_shed_status_dict + except HTTPError, e: + # This should handle backward compatility to the Galaxy 12/20/12 release. We used to only handle updates for an installed revision + # using a boolean value. + log.debug( "Error attempting to get tool shed status for installed repository %s: %s\nAttempting older 'check_for_updates' method.\n" % + ( str( repository.name ), str( e ) ) ) + pathspec = [ 'repository', 'check_for_updates' ] + params[ 'from_update_manager' ] = True + try: + # The value of text will be 'true' or 'false', depending upon whether there is an update available for the installed revision. + text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) + return dict( revision_update=text ) + except Exception, e: + # The required tool shed may be unavailable, so default the revision_update value to 'false'. + return dict( revision_update='false' ) + except Exception, e: + log.exception( "Error attempting to get tool shed status for installed repository %s: %s" % ( str( repository.name ), str( e ) ) ) + return {} + + +def handle_role_associations( app, role, repository, **kwd ): + sa_session = app.model.context.current + message = escape( kwd.get( 'message', '' ) ) + status = kwd.get( 'status', 'done' ) + repository_owner = repository.user + if kwd.get( 'manage_role_associations_button', False ): + in_users_list = util.listify( kwd.get( 'in_users', [] ) ) + in_users = [ sa_session.query( app.model.User ).get( x ) for x in in_users_list ] + # Make sure the repository owner is always associated with the repostory's admin role. + owner_associated = False + for user in in_users: + if user.id == repository_owner.id: + owner_associated = True + break + if not owner_associated: + in_users.append( repository_owner ) + message += "The repository owner must always be associated with the repository's administrator role. " + status = 'error' + in_groups_list = util.listify( kwd.get( 'in_groups', [] ) ) + in_groups = [ sa_session.query( app.model.Group ).get( x ) for x in in_groups_list ] + in_repositories = [ repository ] + app.security_agent.set_entity_role_associations( roles=[ role ], + users=in_users, + groups=in_groups, + repositories=in_repositories ) + sa_session.refresh( role ) + message += "Role %s has been associated with %d users, %d groups and %d repositories. " % \ + ( escape( str( role.name ) ), len( in_users ), len( in_groups ), len( in_repositories ) ) + in_users = [] + out_users = [] + in_groups = [] + out_groups = [] + for user in sa_session.query( app.model.User ) \ + .filter( app.model.User.table.c.deleted == false() ) \ + .order_by( app.model.User.table.c.email ): + if user in [ x.user for x in role.users ]: + in_users.append( ( user.id, user.email ) ) + else: + out_users.append( ( user.id, user.email ) ) + for group in sa_session.query( app.model.Group ) \ + .filter( app.model.Group.table.c.deleted == false() ) \ + .order_by( app.model.Group.table.c.name ): + if group in [ x.group for x in role.groups ]: + in_groups.append( ( group.id, group.name ) ) + else: + out_groups.append( ( group.id, group.name ) ) + associations_dict = dict( in_users=in_users, + out_users=out_users, + in_groups=in_groups, + out_groups=out_groups, + message=message, + status=status ) + return associations_dict + + +def is_tool_shed_client( app ): + """ + The tool shed and clients to the tool (i.e. Galaxy) require a lot + of similar functionality in this file but with small differences. This + method should determine if the app performing the action is the tool shed + or a client of the tool shed. + """ + return hasattr( app, "install_model" ) + + +def repository_was_previously_installed( app, tool_shed_url, repository_name, repo_info_tuple, from_tip=False ): + """ + Find out if a repository is already installed into Galaxy - there are several scenarios where this + is necessary. For example, this method will handle the case where the repository was previously + installed using an older changeset_revsion, but later the repository was updated in the tool shed + and now we're trying to install the latest changeset revision of the same repository instead of + updating the one that was previously installed. We'll look in the database instead of on disk since + the repository may be currently uninstalled. + """ + tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url ) + description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \ + get_repo_info_tuple_contents( repo_info_tuple ) + tool_shed = get_tool_shed_from_clone_url( repository_clone_url ) + # See if we can locate the repository using the value of changeset_revision. + tool_shed_repository = get_installed_repository( app, + tool_shed=tool_shed, + name=repository_name, + owner=repository_owner, + installed_changeset_revision=changeset_revision ) + if tool_shed_repository: + return tool_shed_repository, changeset_revision + # Get all previous changeset revisions from the tool shed for the repository back to, but excluding, + # the previous valid changeset revision to see if it was previously installed using one of them. + params = dict( galaxy_url=web.url_for( '/', qualified=True ), + name=repository_name, + owner=repository_owner, + changeset_revision=changeset_revision, + from_tip=str( from_tip ) ) + pathspec = [ 'repository', 'previous_changeset_revisions' ] + text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) + if text: + changeset_revisions = util.listify( text ) + for previous_changeset_revision in changeset_revisions: + tool_shed_repository = get_installed_repository( app, + tool_shed=tool_shed, + name=repository_name, + owner=repository_owner, + installed_changeset_revision=previous_changeset_revision ) + if tool_shed_repository: + return tool_shed_repository, previous_changeset_revision + return None, None + + +def set_repository_attributes( app, repository, status, error_message, deleted, uninstalled, remove_from_disk=False ): + if remove_from_disk: + relative_install_dir = repository.repo_path( app ) + if relative_install_dir: + clone_dir = os.path.abspath( relative_install_dir ) + try: + shutil.rmtree( clone_dir ) + log.debug( "Removed repository installation directory: %s" % str( clone_dir ) ) + except Exception, e: + log.debug( "Error removing repository installation directory %s: %s" % ( str( clone_dir ), str( e ) ) ) + repository.error_message = error_message + repository.status = status + repository.deleted = deleted + repository.uninstalled = uninstalled + app.install_model.context.add( repository ) + app.install_model.context.flush() + + def update_repository( app, trans, id, **kwds ): """Update an existing ToolShed repository""" message = None @@ -251,200 +1022,6 @@ def update_repository( app, trans, id, **kwds ): return repository, message -def create_repository_admin_role( app, repository ): - """ - Create a new role with name-spaced name based on the repository name and its owner's public user - name. This will ensure that the tole name is unique. - """ - sa_session = app.model.context.current - name = get_repository_admin_role_name( str( repository.name ), str( repository.user.username ) ) - description = 'A user or group member with this role can administer this repository.' - role = app.model.Role( name=name, description=description, type=app.model.Role.types.SYSTEM ) - sa_session.add( role ) - sa_session.flush() - # Associate the role with the repository owner. - app.model.UserRoleAssociation( repository.user, role ) - # Associate the role with the repository. - rra = app.model.RepositoryRoleAssociation( repository, role ) - sa_session.add( rra ) - sa_session.flush() - return role - - -def get_installed_tool_shed_repository( app, id ): - """Get a tool shed repository record from the Galaxy database defined by the id.""" - rval = [] - if isinstance( id, list ): - return_list = True - else: - id = [ id ] - return_list = False - for i in id: - rval.append( app.install_model.context.query( app.install_model.ToolShedRepository ).get( app.security.decode_id( i ) ) ) - if return_list: - return rval - return rval[0] - - -def get_repo_info_dict( app, user, repository_id, changeset_revision ): - repository = suc.get_repository_in_tool_shed( app, repository_id ) - repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) - repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( user, repository ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, - repository_id, - changeset_revision ) - if not repository_metadata: - # The received changeset_revision is no longer installable, so get the next changeset_revision - # in the repository's changelog. This generally occurs only with repositories of type - # repository_suite_definition or tool_dependency_definition. - next_downloadable_changeset_revision = \ - suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) - if next_downloadable_changeset_revision and next_downloadable_changeset_revision != changeset_revision: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( app, - repository_id, - next_downloadable_changeset_revision ) - if repository_metadata: - # For now, we'll always assume that we'll get repository_metadata, but if we discover our assumption - # is not valid we'll have to enhance the callers to handle repository_metadata values of None in the - # returned repo_info_dict. - metadata = repository_metadata.metadata - if 'tools' in metadata: - includes_tools = True - else: - includes_tools = False - includes_tools_for_display_in_tool_panel = repository_metadata.includes_tools_for_display_in_tool_panel - repository_dependencies_dict = metadata.get( 'repository_dependencies', {} ) - repository_dependencies = repository_dependencies_dict.get( 'repository_dependencies', [] ) - has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td = \ - suc.get_repository_dependency_types( repository_dependencies ) - if 'tool_dependencies' in metadata: - includes_tool_dependencies = True - else: - includes_tool_dependencies = False - else: - # Here's where we may have to handle enhancements to the callers. See above comment. - includes_tools = False - has_repository_dependencies = False - has_repository_dependencies_only_if_compiling_contained_td = False - includes_tool_dependencies = False - includes_tools_for_display_in_tool_panel = False - ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision ) - repo_info_dict = create_repo_info_dict( app=app, - repository_clone_url=repository_clone_url, - changeset_revision=changeset_revision, - ctx_rev=str( ctx.rev() ), - repository_owner=repository.user.username, - repository_name=repository.name, - repository=repository, - repository_metadata=repository_metadata, - tool_dependencies=None, - repository_dependencies=None ) - return repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, \ - has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td - - -def get_repository_admin_role_name( repository_name, repository_owner ): - return '%s_%s_admin' % ( str( repository_name ), str( repository_owner ) ) - - -def get_role_by_id( app, role_id ): - """Get a Role from the database by id.""" - sa_session = app.model.context.current - return sa_session.query( app.model.Role ).get( app.security.decode_id( role_id ) ) - - -def handle_role_associations( app, role, repository, **kwd ): - sa_session = app.model.context.current - message = escape( kwd.get( 'message', '' ) ) - status = kwd.get( 'status', 'done' ) - repository_owner = repository.user - if kwd.get( 'manage_role_associations_button', False ): - in_users_list = util.listify( kwd.get( 'in_users', [] ) ) - in_users = [ sa_session.query( app.model.User ).get( x ) for x in in_users_list ] - # Make sure the repository owner is always associated with the repostory's admin role. - owner_associated = False - for user in in_users: - if user.id == repository_owner.id: - owner_associated = True - break - if not owner_associated: - in_users.append( repository_owner ) - message += "The repository owner must always be associated with the repository's administrator role. " - status = 'error' - in_groups_list = util.listify( kwd.get( 'in_groups', [] ) ) - in_groups = [ sa_session.query( app.model.Group ).get( x ) for x in in_groups_list ] - in_repositories = [ repository ] - app.security_agent.set_entity_role_associations( roles=[ role ], - users=in_users, - groups=in_groups, - repositories=in_repositories ) - sa_session.refresh( role ) - message += "Role %s has been associated with %d users, %d groups and %d repositories. " % \ - ( escape( str( role.name ) ), len( in_users ), len( in_groups ), len( in_repositories ) ) - in_users = [] - out_users = [] - in_groups = [] - out_groups = [] - for user in sa_session.query( app.model.User ) \ - .filter( app.model.User.table.c.deleted == false() ) \ - .order_by( app.model.User.table.c.email ): - if user in [ x.user for x in role.users ]: - in_users.append( ( user.id, user.email ) ) - else: - out_users.append( ( user.id, user.email ) ) - for group in sa_session.query( app.model.Group ) \ - .filter( app.model.Group.table.c.deleted == false() ) \ - .order_by( app.model.Group.table.c.name ): - if group in [ x.group for x in role.groups ]: - in_groups.append( ( group.id, group.name ) ) - else: - out_groups.append( ( group.id, group.name ) ) - associations_dict = dict( in_users=in_users, - out_users=out_users, - in_groups=in_groups, - out_groups=out_groups, - message=message, - status=status ) - return associations_dict - - -def check_for_updates( app, model, repository_id=None ): - message = '' - status = 'ok' - if repository_id is None: - success_count = 0 - repository_names_not_updated = [] - updated_count = 0 - for repository in model.context.query( model.ToolShedRepository ) \ - .filter( model.ToolShedRepository.table.c.deleted == false() ): - ok, updated = \ - check_or_update_tool_shed_status_for_installed_repository( app, repository ) - if ok: - success_count += 1 - else: - repository_names_not_updated.append( '%s' % escape( str( repository.name ) ) ) - if updated: - updated_count += 1 - message = "Checked the status in the tool shed for %d repositories. " % success_count - message += "Updated the tool shed status for %d repositories. " % updated_count - if repository_names_not_updated: - message += "Unable to retrieve status from the tool shed for the following repositories:\n" - message += ", ".join( repository_names_not_updated ) - else: - repository = suc.get_tool_shed_repository_by_id( app, repository_id ) - ok, updated = \ - check_or_update_tool_shed_status_for_installed_repository( app, repository ) - if ok: - if updated: - message = "The tool shed status for repository %s has been updated." % escape( str( repository.name ) ) - else: - message = "The status has not changed in the tool shed for repository %s." % escape( str( repository.name ) ) - else: - message = "Unable to retrieve status from the tool shed for repository %s." % escape( str( repository.name ) ) - status = 'error' - return message, status - - def validate_repository_name( app, name, user ): """ Validate whether the given name qualifies as a new TS repo name. @@ -455,7 +1032,7 @@ def validate_repository_name( app, name, user ): return 'Enter the required repository name.' if name in [ 'repos' ]: return "The term %s is a reserved word in the tool shed, so it cannot be used as a repository name." % name - check_existing = suc.get_repository_by_name_and_owner( app, name, user.username ) + check_existing = get_repository_by_name_and_owner( app, name, user.username ) if check_existing is not None: if check_existing.deleted: return 'You own a deleted repository named %s, please choose a different name.' % escape( name ) diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index 1437d597f22..6efa2b8449f 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -2,21 +2,18 @@ import json import logging import os import re -import shutil import socket import string -from urllib2 import HTTPError -from operator import itemgetter import sqlalchemy.orm.exc -from sqlalchemy import and_, false, or_, true +from sqlalchemy import and_, false, true +import tool_shed.util.repository_util from galaxy import util from galaxy.web import url_for from galaxy.util import checkers from tool_shed.util import basic_util from tool_shed.util import common_util -from tool_shed.util import encoding_util from tool_shed.util import hg_util log = logging.getLogger( __name__ ) @@ -149,97 +146,6 @@ def clean_dependency_relationships(trans, metadata_dict, tool_shed_repository, t trans.install_model.context.flush() -def create_or_update_tool_shed_repository( app, name, description, installed_changeset_revision, ctx_rev, repository_clone_url, - metadata_dict, status, current_changeset_revision=None, owner='', dist_to_shed=False ): - """ - Update a tool shed repository record in the Galaxy database with the new information received. - If a record defined by the received tool shed, repository name and owner does not exist, create - a new record with the received information. - """ - # The received value for dist_to_shed will be True if the ToolMigrationManager is installing a repository - # that contains tools or datatypes that used to be in the Galaxy distribution, but have been moved - # to the main Galaxy tool shed. - if current_changeset_revision is None: - # The current_changeset_revision is not passed if a repository is being installed for the first - # time. If a previously installed repository was later uninstalled, this value should be received - # as the value of that change set to which the repository had been updated just prior to it being - # uninstalled. - current_changeset_revision = installed_changeset_revision - context = app.install_model.context - tool_shed = get_tool_shed_from_clone_url( repository_clone_url ) - if not owner: - owner = get_repository_owner_from_clone_url( repository_clone_url ) - includes_datatypes = 'datatypes' in metadata_dict - if status in [ app.install_model.ToolShedRepository.installation_status.DEACTIVATED ]: - deleted = True - uninstalled = False - elif status in [ app.install_model.ToolShedRepository.installation_status.UNINSTALLED ]: - deleted = True - uninstalled = True - else: - deleted = False - uninstalled = False - tool_shed_repository = \ - get_installed_repository( app, tool_shed=tool_shed, name=name, owner=owner, installed_changeset_revision=installed_changeset_revision ) - if tool_shed_repository: - log.debug( "Updating an existing row for repository '%s' in the tool_shed_repository table, status set to '%s'." % - ( str( name ), str( status ) ) ) - tool_shed_repository.description = description - tool_shed_repository.changeset_revision = current_changeset_revision - tool_shed_repository.ctx_rev = ctx_rev - tool_shed_repository.metadata = metadata_dict - tool_shed_repository.includes_datatypes = includes_datatypes - tool_shed_repository.deleted = deleted - tool_shed_repository.uninstalled = uninstalled - tool_shed_repository.status = status - else: - log.debug( "Adding new row for repository '%s' in the tool_shed_repository table, status set to '%s'." % - ( str( name ), str( status ) ) ) - tool_shed_repository = \ - app.install_model.ToolShedRepository( tool_shed=tool_shed, - name=name, - description=description, - owner=owner, - installed_changeset_revision=installed_changeset_revision, - changeset_revision=current_changeset_revision, - ctx_rev=ctx_rev, - metadata=metadata_dict, - includes_datatypes=includes_datatypes, - dist_to_shed=dist_to_shed, - deleted=deleted, - uninstalled=uninstalled, - status=status ) - context.add( tool_shed_repository ) - context.flush() - return tool_shed_repository - - -def extract_components_from_tuple( repository_components_tuple ): - '''Extract the repository components from the provided tuple in a backward-compatible manner.''' - toolshed = repository_components_tuple[ 0 ] - name = repository_components_tuple[ 1 ] - owner = repository_components_tuple[ 2 ] - changeset_revision = repository_components_tuple[ 3 ] - components_list = [ toolshed, name, owner, changeset_revision ] - if len( repository_components_tuple ) == 5: - toolshed, name, owner, changeset_revision, prior_installation_required = repository_components_tuple - components_list = [ toolshed, name, owner, changeset_revision, prior_installation_required ] - elif len( repository_components_tuple ) == 6: - toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = repository_components_tuple - components_list = [ toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td ] - return components_list - - -def generate_sharable_link_for_repository_in_tool_shed( repository, changeset_revision=None ): - """Generate the URL for sharing a repository that is in the tool shed.""" - base_url = url_for( '/', qualified=True ).rstrip( '/' ) - protocol, base = base_url.split( '://' ) - sharable_url = '%s://%s/view/%s/%s' % ( protocol, base, repository.user.username, repository.name ) - if changeset_revision: - sharable_url += '/%s' % changeset_revision - return sharable_url - - def generate_tool_guid( repository_clone_url, tool ): """ Generate a guid for the installed tool. It is critical that this guid matches the guid for @@ -250,33 +156,6 @@ def generate_tool_guid( repository_clone_url, tool ): return '%s/%s/%s' % ( tmp_url, tool.id, tool.version ) -def generate_tool_shed_repository_install_dir( repository_clone_url, changeset_revision ): - """ - Generate a repository installation directory that guarantees repositories with the same - name will always be installed in different directories. The tool path will be of the form: - /repos/// - """ - tmp_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) - # Now tmp_url is something like: bx.psu.edu:9009/repos/some_username/column - items = tmp_url.split( '/repos/' ) - tool_shed_url = items[ 0 ] - repo_path = items[ 1 ] - tool_shed_url = common_util.remove_port_from_tool_shed_url( tool_shed_url ) - return '/'.join( [ tool_shed_url, 'repos', repo_path, changeset_revision ] ) - - -def get_absolute_path_to_file_in_repository( repo_files_dir, file_name ): - """Return the absolute path to a specified disk file contained in a repository.""" - stripped_file_name = basic_util.strip_path( file_name ) - file_path = None - for root, dirs, files in os.walk( repo_files_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name == stripped_file_name: - return os.path.abspath( os.path.join( root, name ) ) - return file_path - - def get_categories( app ): """Get all categories from the database.""" sa_session = app.model.context.current @@ -313,112 +192,6 @@ def get_ctx_rev( app, tool_shed_url, name, owner, changeset_revision ): return ctx_rev -def get_current_repository_metadata_for_changeset_revision( app, repository, changeset_revision ): - encoded_repository_id = app.security.encode_id( repository.id ) - repository_metadata = get_repository_metadata_by_changeset_revision( app, - encoded_repository_id, - changeset_revision ) - if repository_metadata: - return repository_metadata - # The installable changeset_revision may have been changed because it was "moved ahead" - # in the repository changelog. - repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) - updated_changeset_revision = get_next_downloadable_changeset_revision( repository, - repo, - after_changeset_revision=changeset_revision ) - if updated_changeset_revision and updated_changeset_revision != changeset_revision: - repository_metadata = get_repository_metadata_by_changeset_revision( app, - encoded_repository_id, - updated_changeset_revision ) - if repository_metadata: - return repository_metadata - return None - - -def get_ids_of_tool_shed_repositories_being_installed( app, as_string=False ): - installing_repository_ids = [] - new_status = app.install_model.ToolShedRepository.installation_status.NEW - cloning_status = app.install_model.ToolShedRepository.installation_status.CLONING - setting_tool_versions_status = app.install_model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS - installing_dependencies_status = app.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES - loading_datatypes_status = app.install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES - for tool_shed_repository in \ - app.install_model.context.query( app.install_model.ToolShedRepository ) \ - .filter( or_( app.install_model.ToolShedRepository.status == new_status, - app.install_model.ToolShedRepository.status == cloning_status, - app.install_model.ToolShedRepository.status == setting_tool_versions_status, - app.install_model.ToolShedRepository.status == installing_dependencies_status, - app.install_model.ToolShedRepository.status == loading_datatypes_status ) ): - installing_repository_ids.append( app.security.encode_id( tool_shed_repository.id ) ) - if as_string: - return ','.join( installing_repository_ids ) - return installing_repository_ids - - -def get_latest_downloadable_changeset_revision( app, repository, repo=None ): - if repo is None: - repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) - repository_tip = repository.tip( app ) - repository_metadata = get_repository_metadata_by_changeset_revision( app, app.security.encode_id( repository.id ), repository_tip ) - if repository_metadata and repository_metadata.downloadable: - return repository_tip - changeset_revisions = [ revision[ 1 ] for revision in get_metadata_revisions( repository, repo ) ] - if changeset_revisions: - return changeset_revisions[ -1 ] - return hg_util.INITIAL_CHANGELOG_HASH - - -def get_tool_dependency_definition_metadata_from_tool_shed( app, tool_shed_url, name, owner ): - """ - Send a request to the tool shed to retrieve the current metadata for a - repository of type tool_dependency_definition defined by the combination - of a name and owner. - """ - tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url ) - params = dict( name=name, owner=owner ) - pathspec = [ 'repository', 'get_tool_dependency_definition_metadata' ] - metadata = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) - return metadata - - -def get_metadata_changeset_revisions( repository, repo ): - """ - Return an unordered list of changeset_revisions and changeset numbers that are defined as installable. - """ - changeset_tups = [] - for repository_metadata in repository.downloadable_revisions: - ctx = hg_util.get_changectx_for_changeset( repo, repository_metadata.changeset_revision ) - if ctx: - rev = ctx.rev() - else: - rev = -1 - changeset_tups.append( ( rev, repository_metadata.changeset_revision ) ) - return sorted( changeset_tups ) - - -def get_next_downloadable_changeset_revision( repository, repo, after_changeset_revision ): - """ - Return the installable changeset_revision in the repository changelog after the changeset to which - after_changeset_revision refers. If there isn't one, return None. If there is only one installable - changeset, and that matches the requested revision, return it. - """ - changeset_revisions = [ revision[ 1 ] for revision in get_metadata_revisions( repository, repo ) ] - if len( changeset_revisions ) == 1: - changeset_revision = changeset_revisions[ 0 ] - if changeset_revision == after_changeset_revision: - return after_changeset_revision - found_after_changeset_revision = False - for changeset in repo.changelog: - changeset_revision = str( repo.changectx( changeset ) ) - if found_after_changeset_revision: - if changeset_revision in changeset_revisions: - return changeset_revision - elif not found_after_changeset_revision and changeset_revision == after_changeset_revision: - # We've found the changeset in the changelog for which we need to get the next downloadable changeset. - found_after_changeset_revision = True - return None - - def get_next_prior_import_or_install_required_dict_entry( prior_required_dict, processed_tsr_ids ): """ This method is used in the Tool Shed when exporting a repository and its dependencies, and in Galaxy @@ -453,196 +226,11 @@ def get_next_prior_import_or_install_required_dict_entry( prior_required_dict, p return key -def get_metadata_revisions( repository, repo, sort_revisions=True, reverse=False, downloadable=True ): - """ - Return a list of changesets for the provided repository. - """ - if downloadable: - metadata_revisions = repository.downloadable_revisions - else: - metadata_revisions = repository.metadata_revisions - changeset_tups = [] - for repository_metadata in metadata_revisions: - ctx = hg_util.get_changectx_for_changeset( repo, repository_metadata.changeset_revision ) - if ctx: - rev = '%04d' % ctx.rev() - else: - rev = -1 - changeset_tups.append( ( rev, repository_metadata.changeset_revision ) ) - if sort_revisions: - changeset_tups.sort( key=itemgetter( 0 ), reverse=reverse ) - return changeset_tups - - -def get_prior_import_or_install_required_dict( app, tsr_ids, repo_info_dicts ): - """ - This method is used in the Tool Shed when exporting a repository and its dependencies, - and in Galaxy when a repository and its dependencies are being installed. Return a - dictionary whose keys are the received tsr_ids and whose values are a list of tsr_ids, - each of which is contained in the received list of tsr_ids and whose associated repository - must be imported or installed prior to the repository associated with the tsr_id key. - """ - # Initialize the dictionary. - prior_import_or_install_required_dict = {} - for tsr_id in tsr_ids: - prior_import_or_install_required_dict[ tsr_id ] = [] - # Inspect the repository dependencies for each repository about to be installed and populate the dictionary. - for repo_info_dict in repo_info_dicts: - repository, repository_dependencies = get_repository_and_repository_dependencies_from_repo_info_dict( app, repo_info_dict ) - if repository: - encoded_repository_id = app.security.encode_id( repository.id ) - if encoded_repository_id in tsr_ids: - # We've located the database table record for one of the repositories we're about to install, so find out if it has any repository - # dependencies that require prior installation. - prior_import_or_install_ids = get_repository_ids_requiring_prior_import_or_install( app, tsr_ids, repository_dependencies ) - prior_import_or_install_required_dict[ encoded_repository_id ] = prior_import_or_install_ids - return prior_import_or_install_required_dict - - -def get_repo_info_tuple_contents( repo_info_tuple ): - """Take care in handling the repo_info_tuple as it evolves over time as new tool shed features are introduced.""" - if len( repo_info_tuple ) == 6: - description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, tool_dependencies = repo_info_tuple - repository_dependencies = None - elif len( repo_info_tuple ) == 7: - description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = repo_info_tuple - return description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies - - -def get_repositories_by_category( app, category_id ): +def get_repository_categories( app, id ): + """Get categories of a repository on the tool shed side from the database via id""" sa_session = app.model.context.current - resultset = sa_session.query( app.model.Category ).get( category_id ) - repositories = [] - default_value_mapper = { 'id': app.security.encode_id, 'user_id': app.security.encode_id } - for row in resultset.repositories: - repository_dict = row.repository.to_dict( value_mapper=default_value_mapper ) - repository_dict[ 'metadata' ] = {} - for changeset, changehash in row.repository.installable_revisions( app ): - encoded_id = app.security.encode_id( row.repository.id ) - metadata = get_repository_metadata_by_changeset_revision( app, encoded_id, changehash ) - repository_dict[ 'metadata' ][ '%s:%s' % ( changeset, changehash ) ] = metadata.to_dict( value_mapper=default_value_mapper ) - repositories.append( repository_dict ) - return repositories - - -def get_repository_and_repository_dependencies_from_repo_info_dict( app, repo_info_dict ): - """Return a tool_shed_repository or repository record defined by the information in the received repo_info_dict.""" - repository_name = repo_info_dict.keys()[ 0 ] - repo_info_tuple = repo_info_dict[ repository_name ] - description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \ - get_repo_info_tuple_contents( repo_info_tuple ) - if hasattr( app, "install_model" ): - # In a tool shed client (Galaxy, or something install repositories like Galaxy) - tool_shed = get_tool_shed_from_clone_url( repository_clone_url ) - repository = get_repository_for_dependency_relationship( app, tool_shed, repository_name, repository_owner, changeset_revision ) - else: - # We're in the tool shed. - repository = get_repository_by_name_and_owner( app, repository_name, repository_owner ) - return repository, repository_dependencies - - -def get_repository_by_id( app, id ): - """Get a repository from the database via id.""" - if is_tool_shed_client( app ): - return app.install_model.context.query( app.install_model.ToolShedRepository ).get( app.security.decode_id( id ) ) - else: - sa_session = app.model.context.current - return sa_session.query( app.model.Repository ).get( app.security.decode_id( id ) ) - - -def get_repository_by_name( app, name ): - """Get a repository from the database via name.""" - repository_query = get_repository_query( app ) - return repository_query.filter_by( name=name ).first() - - -def get_repository_by_name_and_owner( app, name, owner ): - """Get a repository from the database via name and owner""" - repository_query = get_repository_query( app ) - if is_tool_shed_client( app ): - return repository_query \ - .filter( and_( app.install_model.ToolShedRepository.table.c.name == name, - app.install_model.ToolShedRepository.table.c.owner == owner ) ) \ - .first() - # We're in the tool shed. - user = get_user_by_username( app, owner ) - if user: - return repository_query \ - .filter( and_( app.model.Repository.table.c.name == name, - app.model.Repository.table.c.user_id == user.id ) ) \ - .first() - return None - - -def get_repository_dependency_types( repository_dependencies ): - """ - Inspect the received list of repository_dependencies tuples and return boolean values - for has_repository_dependencies and has_repository_dependencies_only_if_compiling_contained_td. - """ - # Set has_repository_dependencies, which will be True only if at least one repository_dependency - # is defined with the value of - # only_if_compiling_contained_td as False. - has_repository_dependencies = False - for rd_tup in repository_dependencies: - tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( rd_tup ) - if not util.asbool( only_if_compiling_contained_td ): - has_repository_dependencies = True - break - # Set has_repository_dependencies_only_if_compiling_contained_td, which will be True only if at - # least one repository_dependency is defined with the value of only_if_compiling_contained_td as True. - has_repository_dependencies_only_if_compiling_contained_td = False - for rd_tup in repository_dependencies: - tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( rd_tup ) - if util.asbool( only_if_compiling_contained_td ): - has_repository_dependencies_only_if_compiling_contained_td = True - break - return has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td - - -def get_repository_for_dependency_relationship( app, tool_shed, name, owner, changeset_revision ): - """ - Return an installed tool_shed_repository database record that is defined by either the current changeset - revision or the installed_changeset_revision. - """ - # This method is used only in Galaxy, not the Tool Shed. We store the port (if one exists) in the database. - tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) - if tool_shed is None or name is None or owner is None or changeset_revision is None: - message = "Unable to retrieve the repository record from the database because one or more of the following " - message += "required parameters is None: tool_shed: %s, name: %s, owner: %s, changeset_revision: %s " % \ - ( str( tool_shed ), str( name ), str( owner ), str( changeset_revision ) ) - raise Exception( message ) - repository = get_installed_repository( app=app, - tool_shed=tool_shed, - name=name, - owner=owner, - installed_changeset_revision=changeset_revision ) - if not repository: - repository = get_installed_repository( app=app, - tool_shed=tool_shed, - name=name, - owner=owner, - changeset_revision=changeset_revision ) - if not repository: - tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed ) - repository_clone_url = os.path.join( tool_shed_url, 'repos', owner, name ) - repo_info_tuple = (None, repository_clone_url, changeset_revision, None, owner, None, None) - repository, pcr = repository_was_previously_installed( app, tool_shed_url, name, repo_info_tuple ) - if not repository: - # The received changeset_revision is no longer installable, so get the next changeset_revision - # in the repository's changelog in the tool shed that is associated with repository_metadata. - tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed ) - params = dict( name=name, owner=owner, changeset_revision=changeset_revision ) - pathspec = [ 'repository', 'next_installable_changeset_revision' ] - text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) - if text: - repository = get_installed_repository( app=app, - tool_shed=tool_shed, - name=name, - owner=owner, - changeset_revision=text ) - return repository + return sa_session.query( app.model.RepositoryCategoryAssociation ) \ + .filter(app.model.RepositoryCategoryAssociation.table.c.repository_id == app.security.decode_id( id )) def get_repository_file_contents( app, file_path, repository_id, is_admin=False ): @@ -712,123 +300,13 @@ def get_repository_from_refresh_on_change( app, **kwd ): changeset_revision_str = 'changeset_revision_' if k.startswith( changeset_revision_str ): repository_id = app.security.encode_id( int( k.lstrip( changeset_revision_str ) ) ) - repository = get_repository_in_tool_shed( app, repository_id ) + repository = tool_shed.util.repository_util.get_repository_in_tool_shed( app, repository_id ) if repository.tip( app ) != v: return v, repository # This should never be reached - raise an exception? return v, None -def get_repository_ids_requiring_prior_import_or_install( app, tsr_ids, repository_dependencies ): - """ - This method is used in the Tool Shed when exporting a repository and its dependencies, - and in Galaxy when a repository and its dependencies are being installed. Inspect the - received repository_dependencies and determine if the encoded id of each required - repository is in the received tsr_ids. If so, then determine whether that required - repository should be imported / installed prior to its dependent repository. Return a - list of encoded repository ids, each of which is contained in the received list of tsr_ids, - and whose associated repositories must be imported / installed prior to the dependent - repository associated with the received repository_dependencies. - """ - prior_tsr_ids = [] - if repository_dependencies: - for key, rd_tups in repository_dependencies.items(): - if key in [ 'description', 'root_key' ]: - continue - for rd_tup in rd_tups: - tool_shed, \ - name, \ - owner, \ - changeset_revision, \ - prior_installation_required, \ - only_if_compiling_contained_td = \ - common_util.parse_repository_dependency_tuple( rd_tup ) - # If only_if_compiling_contained_td is False, then the repository dependency - # is not required to be installed prior to the dependent repository even if - # prior_installation_required is True. This is because the only meaningful - # content of the repository dependency is its contained tool dependency, which - # is required in order to compile the dependent repository's tool dependency. - # In the scenario where the repository dependency is not installed prior to the - # dependent repository's tool dependency compilation process, the tool dependency - # compilation framework will install the repository dependency prior to compilation - # of the dependent repository's tool dependency. - if not util.asbool( only_if_compiling_contained_td ): - if util.asbool( prior_installation_required ): - if is_tool_shed_client( app ): - # We store the port, if one exists, in the database. - tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) - repository = get_repository_for_dependency_relationship( app, - tool_shed, - name, - owner, - changeset_revision ) - else: - repository = get_repository_by_name_and_owner( app, name, owner ) - if repository: - encoded_repository_id = app.security.encode_id( repository.id ) - if encoded_repository_id in tsr_ids: - prior_tsr_ids.append( encoded_repository_id ) - return prior_tsr_ids - - -def get_repository_in_tool_shed( app, id ): - """Get a repository on the tool shed side from the database via id.""" - sa_session = app.model.context.current - return sa_session.query( app.model.Repository ).get( app.security.decode_id( id ) ) - - -def get_repository_categories( app, id ): - """Get categories of a repository on the tool shed side from the database via id""" - sa_session = app.model.context.current - return sa_session.query( app.model.RepositoryCategoryAssociation ) \ - .filter(app.model.RepositoryCategoryAssociation.table.c.repository_id == app.security.decode_id( id )) - - -def get_repository_metadata_by_changeset_revision( app, id, changeset_revision ): - """Get metadata for a specified repository change set from the database.""" - # Make sure there are no duplicate records, and return the single unique record for the changeset_revision. - # Duplicate records were somehow created in the past. The cause of this issue has been resolved, but we'll - # leave this method as is for a while longer to ensure all duplicate records are removed. - sa_session = app.model.context.current - all_metadata_records = sa_session.query( app.model.RepositoryMetadata ) \ - .filter( and_( app.model.RepositoryMetadata.table.c.repository_id == app.security.decode_id( id ), - app.model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) ) \ - .order_by( app.model.RepositoryMetadata.table.c.update_time.desc() ) \ - .all() - if len( all_metadata_records ) > 1: - # Delete all records older than the last one updated. - for repository_metadata in all_metadata_records[ 1: ]: - sa_session.delete( repository_metadata ) - sa_session.flush() - return all_metadata_records[ 0 ] - elif all_metadata_records: - return all_metadata_records[ 0 ] - return None - - -def get_repository_owner( cleaned_repository_url ): - """Gvien a "cleaned" repository clone URL, return the owner of the repository.""" - items = cleaned_repository_url.split( '/repos/' ) - repo_path = items[ 1 ] - if repo_path.startswith( '/' ): - repo_path = repo_path.replace( '/', '', 1 ) - return repo_path.lstrip( '/' ).split( '/' )[ 0 ] - - -def get_repository_owner_from_clone_url( repository_clone_url ): - """Given a repository clone URL, return the owner of the repository.""" - tmp_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) - return get_repository_owner( tmp_url ) - - -def get_repository_query( app ): - if is_tool_shed_client( app ): - query = app.install_model.context.query( app.install_model.ToolShedRepository ) - else: - query = app.model.context.query( app.model.Repository ) - return query - - def get_repository_type_from_tool_shed( app, tool_shed_url, name, owner ): """ Send a request to the tool shed to retrieve the type for a repository defined by the @@ -841,6 +319,19 @@ def get_repository_type_from_tool_shed( app, tool_shed_url, name, owner ): return repository_type +def get_tool_dependency_definition_metadata_from_tool_shed( app, tool_shed_url, name, owner ): + """ + Send a request to the tool shed to retrieve the current metadata for a + repository of type tool_dependency_definition defined by the combination + of a name and owner. + """ + tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url ) + params = dict( name=name, owner=owner ) + pathspec = [ 'repository', 'get_tool_dependency_definition_metadata' ] + metadata = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) + return metadata + + def get_tool_panel_config_tool_path_install_dir( app, repository ): """ Return shed-related tool panel config, the tool_path configured in it, and the relative path to @@ -881,177 +372,12 @@ def get_tool_path_by_shed_tool_conf_filename( app, shed_tool_conf ): return None -def get_tool_shed_from_clone_url( repository_clone_url ): - tmp_url = common_util.remove_protocol_and_user_from_clone_url( repository_clone_url ) - return tmp_url.split( '/repos/' )[ 0 ].rstrip( '/' ) - - -def get_installed_repository( app, tool_shed, name, owner, changeset_revision=None, installed_changeset_revision=None ): - """ - Return a tool shed repository database record defined by the combination of a toolshed, repository name, - repository owner and either current or originally installed changeset_revision. - """ - query = app.install_model.context.query( app.install_model.ToolShedRepository ) - # We store the port, if one exists, in the database. - tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed ) - clause_list = [ app.install_model.ToolShedRepository.table.c.tool_shed == tool_shed, - app.install_model.ToolShedRepository.table.c.name == name, - app.install_model.ToolShedRepository.table.c.owner == owner ] - if changeset_revision is not None: - clause_list.append( app.install_model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) - if installed_changeset_revision is not None: - clause_list.append( app.install_model.ToolShedRepository.table.c.installed_changeset_revision == installed_changeset_revision ) - return query.filter( and_( *clause_list ) ).first() - - -def get_tool_shed_repository_by_id( app, repository_id ): - """Return a tool shed repository database record defined by the id.""" - # This method is used only in Galaxy, not the tool shed. - return app.install_model.context.query( app.install_model.ToolShedRepository ) \ - .filter( app.install_model.ToolShedRepository.table.c.id == app.security.decode_id( repository_id ) ) \ - .first() - - -def get_tool_shed_status_for_installed_repository( app, repository ): - """ - Send a request to the tool shed to retrieve information about newer installable repository revisions, - current revision updates, whether the repository revision is the latest downloadable revision, and - whether the repository has been deprecated in the tool shed. The received repository is a ToolShedRepository - object from Galaxy. - """ - tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, str( repository.tool_shed ) ) - params = dict( name=repository.name, owner=repository.owner, changeset_revision=repository.changeset_revision ) - pathspec = [ 'repository', 'status_for_installed_repository' ] - try: - encoded_tool_shed_status_dict = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) - tool_shed_status_dict = encoding_util.tool_shed_decode( encoded_tool_shed_status_dict ) - return tool_shed_status_dict - except HTTPError as e: - # This should handle backward compatility to the Galaxy 12/20/12 release. We used to only handle updates for an installed revision - # using a boolean value. - log.debug( "Error attempting to get tool shed status for installed repository %s: %s\nAttempting older 'check_for_updates' method.\n" % - ( str( repository.name ), str( e ) ) ) - pathspec = [ 'repository', 'check_for_updates' ] - params[ 'from_update_manager' ] = True - try: - # The value of text will be 'true' or 'false', depending upon whether there is an update available for the installed revision. - text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) - return dict( revision_update=text ) - except Exception as e: - # The required tool shed may be unavailable, so default the revision_update value to 'false'. - return dict( revision_update='false' ) - except Exception as e: - log.exception( "Error attempting to get tool shed status for installed repository %s: %s" % ( str( repository.name ), str( e ) ) ) - return {} - - -def get_tool_shed_repository_status_label( app, tool_shed_repository=None, name=None, owner=None, changeset_revision=None, repository_clone_url=None ): - """Return a color-coded label for the status of the received tool-shed_repository installed into Galaxy.""" - if tool_shed_repository is None: - if name is not None and owner is not None and repository_clone_url is not None: - tool_shed = get_tool_shed_from_clone_url( repository_clone_url ) - tool_shed_repository = get_installed_repository( app, - tool_shed=tool_shed, - name=name, - owner=owner, - installed_changeset_revision=changeset_revision ) - if tool_shed_repository: - status_label = tool_shed_repository.status - if tool_shed_repository.status in [ app.install_model.ToolShedRepository.installation_status.CLONING, - app.install_model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS, - app.install_model.ToolShedRepository.installation_status.INSTALLING_REPOSITORY_DEPENDENCIES, - app.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES, - app.install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES ]: - bgcolor = app.install_model.ToolShedRepository.states.INSTALLING - elif tool_shed_repository.status in [ app.install_model.ToolShedRepository.installation_status.NEW, - app.install_model.ToolShedRepository.installation_status.UNINSTALLED ]: - bgcolor = app.install_model.ToolShedRepository.states.UNINSTALLED - elif tool_shed_repository.status in [ app.install_model.ToolShedRepository.installation_status.ERROR ]: - bgcolor = app.install_model.ToolShedRepository.states.ERROR - elif tool_shed_repository.status in [ app.install_model.ToolShedRepository.installation_status.DEACTIVATED ]: - bgcolor = app.install_model.ToolShedRepository.states.WARNING - elif tool_shed_repository.status in [ app.install_model.ToolShedRepository.installation_status.INSTALLED ]: - if tool_shed_repository.repository_dependencies_being_installed: - bgcolor = app.install_model.ToolShedRepository.states.WARNING - status_label = '%s, %s' % ( status_label, - app.install_model.ToolShedRepository.installation_status.INSTALLING_REPOSITORY_DEPENDENCIES ) - elif tool_shed_repository.missing_repository_dependencies: - bgcolor = app.install_model.ToolShedRepository.states.WARNING - status_label = '%s, missing repository dependencies' % status_label - elif tool_shed_repository.tool_dependencies_being_installed: - bgcolor = app.install_model.ToolShedRepository.states.WARNING - status_label = '%s, %s' % ( status_label, - app.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) - elif tool_shed_repository.missing_tool_dependencies: - bgcolor = app.install_model.ToolShedRepository.states.WARNING - status_label = '%s, missing tool dependencies' % status_label - else: - bgcolor = app.install_model.ToolShedRepository.states.OK - else: - bgcolor = app.install_model.ToolShedRepository.states.ERROR - else: - bgcolor = app.install_model.ToolShedRepository.states.WARNING - status_label = 'unknown status' - return '
%s
' % ( bgcolor, status_label ) - - -def get_updated_changeset_revisions( app, name, owner, changeset_revision ): - """ - Return a string of comma-separated changeset revision hashes for all available updates to the received changeset - revision for the repository defined by the received name and owner. - """ - repository = get_repository_by_name_and_owner( app, name, owner ) - repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) - # Get the upper bound changeset revision. - upper_bound_changeset_revision = get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) - # Build the list of changeset revision hashes defining each available update up to, but excluding - # upper_bound_changeset_revision. - changeset_hashes = [] - for changeset in hg_util.reversed_lower_upper_bounded_changelog( repo, changeset_revision, upper_bound_changeset_revision ): - # Make sure to exclude upper_bound_changeset_revision. - if changeset != upper_bound_changeset_revision: - changeset_hashes.append( str( repo.changectx( changeset ) ) ) - if changeset_hashes: - changeset_hashes_str = ','.join( changeset_hashes ) - return changeset_hashes_str - return '' - - -def get_updated_changeset_revisions_from_tool_shed( app, tool_shed_url, name, owner, changeset_revision ): - """ - Get all appropriate newer changeset revisions for the repository defined by - the received tool_shed_url / name / owner combination. - """ - tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url ) - if tool_shed_url is None or name is None or owner is None or changeset_revision is None: - message = "Unable to get updated changeset revisions from the Tool Shed because one or more of the following " - message += "required parameters is None: tool_shed_url: %s, name: %s, owner: %s, changeset_revision: %s " % \ - ( str( tool_shed_url ), str( name ), str( owner ), str( changeset_revision ) ) - raise Exception( message ) - params = dict( name=name, owner=owner, changeset_revision=changeset_revision ) - pathspec = [ 'repository', 'updated_changeset_revisions' ] - text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) - return text - - def get_user( app, id ): """Get a user from the database by id.""" sa_session = app.model.context.current return sa_session.query( app.model.User ).get( app.security.decode_id( id ) ) -def get_user_by_username( app, username ): - """Get a user from the database by username.""" - sa_session = app.model.context.current - try: - user = sa_session.query( app.model.User ) \ - .filter( app.model.User.table.c.username == username ) \ - .one() - return user - except Exception: - return None - - def handle_email_alerts( app, host, repository, content_alert_str='', new_repo_alert=False, admin_only=False ): """ There are 2 complementary features that enable a tool shed user to receive email notification: @@ -1077,7 +403,7 @@ def handle_email_alerts( app, host, repository, content_alert_str='', new_repo_a """ sa_session = app.model.context.current repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False ) - sharable_link = generate_sharable_link_for_repository_in_tool_shed( repository, changeset_revision=None ) + sharable_link = tool_shed.util.repository_util.generate_sharable_link_for_repository_in_tool_shed( repository, changeset_revision=None ) smtp_server = app.config.smtp_server if smtp_server and ( new_repo_alert or repository.email_alerts ): # Send email alert to users that want them. @@ -1149,14 +475,39 @@ def have_shed_tool_conf_for_install( app ): return bool( app.toolbox.dynamic_confs( include_migrated_tool_conf=False ) ) -def is_tool_shed_client( app ): +def is_path_browsable( app, path, repository_id, is_admin=False ): """ - The tool shed and clients to the tool (i.e. Galaxy) require a lot - of similar functionality in this file but with small differences. This - method should determine if the app performing the action is the tool shed - or a client of the tool shed. + Detects whether the given path is browsable i.e. is within the + allowed repository folders. Admins can additionaly browse folders + with tool dependencies. """ - return hasattr( app, "install_model" ) + if is_admin and is_path_within_dependency_dir( app, path ): + return True + return is_path_within_repo( app, path, repository_id) + + +def is_path_within_dependency_dir( app, path ): + """ + Detect whether the given path is within the tool_dependency_dir folder on the disk. + (Specified by the config option). Use to filter malicious symlinks targeting outside paths. + """ + allowed = False + resolved_path = os.path.realpath( path ) + tool_dependency_dir = app.config.get( 'tool_dependency_dir', None ) + if tool_dependency_dir: + dependency_path = os.path.abspath( tool_dependency_dir ) + allowed = os.path.commonprefix( [ dependency_path, resolved_path ] ) == dependency_path + return allowed + + +def is_path_within_repo( app, path, repository_id ): + """ + Detect whether the given path is within the repository folder on the disk. + Use to filter malicious symlinks targeting outside paths. + """ + repo_path = os.path.abspath( tool_shed.util.repository_util.get_repository_by_id( app, repository_id ).repo_path( app ) ) + resolved_path = os.path.realpath( path ) + return os.path.commonprefix( [ repo_path, resolved_path ] ) == repo_path def open_repository_files_folder( app, folder_path, repository_id, is_admin=False ): @@ -1196,84 +547,6 @@ def open_repository_files_folder( app, folder_path, repository_id, is_admin=Fals return folder_contents -def is_path_browsable( app, path, repository_id, is_admin=False ): - """ - Detects whether the given path is browsable i.e. is within the - allowed repository folders. Admins can additionaly browse folders - with tool dependencies. - """ - if is_admin and is_path_within_dependency_dir( app, path ): - return True - return is_path_within_repo( app, path, repository_id) - - -def is_path_within_repo( app, path, repository_id ): - """ - Detect whether the given path is within the repository folder on the disk. - Use to filter malicious symlinks targeting outside paths. - """ - repo_path = os.path.abspath( get_repository_by_id( app, repository_id ).repo_path( app ) ) - resolved_path = os.path.realpath( path ) - return os.path.commonprefix( [ repo_path, resolved_path ] ) == repo_path - - -def is_path_within_dependency_dir( app, path ): - """ - Detect whether the given path is within the tool_dependency_dir folder on the disk. - (Specified by the config option). Use to filter malicious symlinks targeting outside paths. - """ - allowed = False - resolved_path = os.path.realpath( path ) - tool_dependency_dir = app.config.get( 'tool_dependency_dir', None ) - if tool_dependency_dir: - dependency_path = os.path.abspath( tool_dependency_dir ) - allowed = os.path.commonprefix( [ dependency_path, resolved_path ] ) == dependency_path - return allowed - - -def repository_was_previously_installed( app, tool_shed_url, repository_name, repo_info_tuple, from_tip=False ): - """ - Find out if a repository is already installed into Galaxy - there are several scenarios where this - is necessary. For example, this method will handle the case where the repository was previously - installed using an older changeset_revsion, but later the repository was updated in the tool shed - and now we're trying to install the latest changeset revision of the same repository instead of - updating the one that was previously installed. We'll look in the database instead of on disk since - the repository may be currently uninstalled. - """ - tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url ) - description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \ - get_repo_info_tuple_contents( repo_info_tuple ) - tool_shed = get_tool_shed_from_clone_url( repository_clone_url ) - # See if we can locate the repository using the value of changeset_revision. - tool_shed_repository = get_installed_repository( app, - tool_shed=tool_shed, - name=repository_name, - owner=repository_owner, - installed_changeset_revision=changeset_revision ) - if tool_shed_repository: - return tool_shed_repository, changeset_revision - # Get all previous changeset revisions from the tool shed for the repository back to, but excluding, - # the previous valid changeset revision to see if it was previously installed using one of them. - params = dict( galaxy_url=url_for( '/', qualified=True ), - name=repository_name, - owner=repository_owner, - changeset_revision=changeset_revision, - from_tip=str( from_tip ) ) - pathspec = [ 'repository', 'previous_changeset_revisions' ] - text = util.url_get( tool_shed_url, password_mgr=app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params ) - if text: - changeset_revisions = util.listify( text ) - for previous_changeset_revision in changeset_revisions: - tool_shed_repository = get_installed_repository( app, - tool_shed=tool_shed, - name=repository_name, - owner=repository_owner, - installed_changeset_revision=previous_changeset_revision ) - if tool_shed_repository: - return tool_shed_repository, previous_changeset_revision - return None, None - - def set_image_paths( app, encoded_repository_id, text ): """ Handle tool help image display for tools that are contained in repositories in @@ -1282,7 +555,7 @@ def set_image_paths( app, encoded_repository_id, text ): return the path to it that will enable the caller to open the file. """ if text: - if is_tool_shed_client( app ): + if tool_shed.util.repository_util.is_tool_shed_client( app ): route_to_images = 'admin_toolshed/static/images/%s' % encoded_repository_id else: # We're in the tool shed. @@ -1300,24 +573,6 @@ def set_image_paths( app, encoded_repository_id, text ): return text -def set_repository_attributes( app, repository, status, error_message, deleted, uninstalled, remove_from_disk=False ): - if remove_from_disk: - relative_install_dir = repository.repo_path( app ) - if relative_install_dir: - clone_dir = os.path.abspath( relative_install_dir ) - try: - shutil.rmtree( clone_dir ) - log.debug( "Removed repository installation directory: %s" % str( clone_dir ) ) - except Exception as e: - log.debug( "Error removing repository installation directory %s: %s" % ( str( clone_dir ), str( e ) ) ) - repository.error_message = error_message - repository.status = status - repository.deleted = deleted - repository.uninstalled = uninstalled - app.install_model.context.add( repository ) - app.install_model.context.flush() - - def tool_shed_is_this_tool_shed( toolshed_base_url ): """Determine if a tool shed is the current tool shed.""" cleaned_toolshed_base_url = common_util.remove_protocol_from_tool_shed_url( toolshed_base_url ) diff --git a/lib/tool_shed/util/workflow_util.py b/lib/tool_shed/util/workflow_util.py index 60a24949277..a1b3adb6682 100644 --- a/lib/tool_shed/util/workflow_util.py +++ b/lib/tool_shed/util/workflow_util.py @@ -18,7 +18,7 @@ from tool_shed.tools import tool_validator from tool_shed.util import encoding_util from tool_shed.util import metadata_util -from tool_shed.util import shed_util_common as suc +from tool_shed.util import repository_util log = logging.getLogger( __name__ ) @@ -164,7 +164,7 @@ def generate_workflow_image( trans, workflow_name, repository_metadata_id=None, metadata = repository_metadata.metadata else: # We're in Galaxy. - repository = suc.get_tool_shed_repository_by_id( trans.app, repository_id ) + repository = repository_util.get_tool_shed_repository_by_id( trans.app, repository_id ) changeset_revision = repository.changeset_revision metadata = repository.metadata # metadata[ 'workflows' ] is a list of tuples where each contained tuple is diff --git a/lib/tool_shed/utility_containers/utility_container_manager.py b/lib/tool_shed/utility_containers/utility_container_manager.py index d7427139df3..97918d5f877 100644 --- a/lib/tool_shed/utility_containers/utility_container_manager.py +++ b/lib/tool_shed/utility_containers/utility_container_manager.py @@ -3,7 +3,7 @@ import logging from galaxy import util from tool_shed.util import common_util from tool_shed.util import container_util -from tool_shed.util import shed_util_common as suc +from tool_shed.util import repository_util log = logging.getLogger( __name__ ) @@ -701,8 +701,8 @@ class UtilityContainerManager( object ): tool_shed_repository_id = None installation_status = 'unknown' if tool_shed_repository_id: - tool_shed_repository = suc.get_tool_shed_repository_by_id( self.app, - self.app.security.encode_id( tool_shed_repository_id ) ) + tool_shed_repository = repository_util.get_tool_shed_repository_by_id( self.app, + self.app.security.encode_id( tool_shed_repository_id ) ) if tool_shed_repository: if tool_shed_repository.missing_repository_dependencies: installation_status = '%s, missing repository dependencies' % installation_status @@ -720,7 +720,7 @@ class UtilityContainerManager( object ): def handle_repository_dependencies_container_entry( self, repository_dependencies_folder, rd_key, rd_value, folder_id, repository_dependency_id, folder_keys ): repository_components_tuple = container_util.get_components_from_key( rd_key ) - components_list = suc.extract_components_from_tuple( repository_components_tuple ) + components_list = repository_util.extract_components_from_tuple( repository_components_tuple ) toolshed, repository_name, repository_owner, changeset_revision = components_list[ 0:4 ] # For backward compatibility to the 12/20/12 Galaxy release. if len( components_list ) == 4: @@ -802,7 +802,7 @@ class UtilityContainerManager( object ): def key_is_current_repositorys_key( self, repository_name, repository_owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, key ): repository_components_tuple = container_util.get_components_from_key( key ) - components_list = suc.extract_components_from_tuple( repository_components_tuple ) + components_list = repository_util.extract_components_from_tuple( repository_components_tuple ) toolshed, key_name, key_owner, key_changeset_revision = components_list[ 0:4 ] # For backward compatibility to the 12/20/12 Galaxy release. if len( components_list ) == 4: diff --git a/templates/admin/tool_shed_repository/repair_repository.mako b/templates/admin/tool_shed_repository/repair_repository.mako index 8f1d643f983..db5c49008c4 100644 --- a/templates/admin/tool_shed_repository/repair_repository.mako +++ b/templates/admin/tool_shed_repository/repair_repository.mako @@ -41,7 +41,7 @@ ${render_galaxy_repository_actions( repository )}
<% - from tool_shed.util.shed_util_common import get_tool_shed_repository_status_label + from tool_shed.util.repository_util import get_tool_shed_repository_status_label ordered_repo_info_dicts = repair_dict.get( 'ordered_repo_info_dicts', [] ) %> diff --git a/templates/webapps/tool_shed/repository/common.mako b/templates/webapps/tool_shed/repository/common.mako index 6bd1dbd5fba..7470329b380 100644 --- a/templates/webapps/tool_shed/repository/common.mako +++ b/templates/webapps/tool_shed/repository/common.mako @@ -219,7 +219,7 @@ <%def name="render_sharable_str( repository, changeset_revision=None )"> <% - from tool_shed.util.shed_util_common import generate_sharable_link_for_repository_in_tool_shed + from tool_shed.util.repository_util import generate_sharable_link_for_repository_in_tool_shed sharable_link = generate_sharable_link_for_repository_in_tool_shed( repository, changeset_revision=changeset_revision ) %> ${ sharable_link } @@ -575,7 +575,7 @@ <%def name="render_repository_dependency( repository_dependency, pad, parent, row_counter, row_is_header=False, render_repository_actions_for='tool_shed' )"> <% from galaxy.util import asbool - from tool_shed.util.shed_util_common import get_repository_by_name_and_owner + from tool_shed.util.repository_util import get_repository_by_name_and_owner encoded_id = trans.security.encode_id( repository_dependency.id ) if trans.webapp.name == 'galaxy': if repository_dependency.tool_shed_repository_id: diff --git a/test/shed_functional/base/twilltestcase.py b/test/shed_functional/base/twilltestcase.py index 90f32c3970a..a8765c91512 100644 --- a/test/shed_functional/base/twilltestcase.py +++ b/test/shed_functional/base/twilltestcase.py @@ -700,6 +700,12 @@ class ShedTwillTestCase( TwillTestCase ): last_review = None return last_review + def get_repositories_category_api( self, categories, strings_displayed=None, strings_not_displayed=None ): + for category in categories: + url = '/api/categories/%s/repositories' % self.security.encode_id( category.id ) + self.visit_url( url ) + self.check_for_strings( strings_displayed, strings_not_displayed ) + def get_tool_dependency_path( self, tool_dependency_name, tool_dependency_version, repository ): '''Return the absolute path for an installed tool dependency.''' return os.path.join( self.galaxy_tool_dependency_dir, diff --git a/test/shed_functional/functional/test_0000_basic_repository_features.py b/test/shed_functional/functional/test_0000_basic_repository_features.py index f8b1b98f960..94fae6b3861 100644 --- a/test/shed_functional/functional/test_0000_basic_repository_features.py +++ b/test/shed_functional/functional/test_0000_basic_repository_features.py @@ -349,3 +349,10 @@ class TestBasicRepositoryFeatures( ShedTwillTestCase ): strings_displayed = [ '%d:%s' % ( revision_number, revision_hash ), 'filtering_0000', 'user1', 'repos', 'added:', '+These characters should not' ] self.load_changeset_in_tool_shed( repository_id, changeset_revision, strings_displayed=strings_displayed ) + + def test_0135_api_get_repositories_in_category( self ): + '''Load the api endpoint for repositories in a category.''' + categories = [] + categories.append( self.test_db_util.get_category_by_name( 'Test 0000 Basic Repository Features 1' ) ) + categories.append( self.test_db_util.get_category_by_name( 'Test 0000 Basic Repository Features 2' ) ) + self.get_repositories_category_api( categories ) diff --git a/test/shed_functional/functional/test_1000_install_basic_repository.py b/test/shed_functional/functional/test_1000_install_basic_repository.py index 5a65016035d..443a2932bc1 100644 --- a/test/shed_functional/functional/test_1000_install_basic_repository.py +++ b/test/shed_functional/functional/test_1000_install_basic_repository.py @@ -21,8 +21,9 @@ class BasicToolShedFeatures( ShedTwillTestCase ): def test_0005_ensure_repositories_and_categories_exist( self ): '''Create the 0000 category and upload the filtering repository to it, if necessary.''' - category = self.create_category( name='Test 0000 Basic Repository Features 1', description='Test 0000 Basic Repository Features 1' ) - self.create_category( name='Test 0000 Basic Repository Features 2', description='Test 0000 Basic Repository Features 2' ) + self.login( email=common.admin_email, username=common.admin_username ) + category = self.create_category( name='Test 0000 Basic Repository Features 2', description='Test Description 0000 Basic Repository Features 2' ) + category = self.create_category( name='Test 0000 Basic Repository Features 1', description='Test Description 0000 Basic Repository Features 1' ) self.login( email=common.test_user_1_email, username=common.test_user_1_name ) repository = self.get_or_create_repository( name='filtering_0000', description="Galaxy's filtering tool",