From 7bfe77fe03a7288f4d12a9a3f3c30caf46d059f2 Mon Sep 17 00:00:00 2001 From: Peter Cock Date: Fri, 19 Oct 2012 14:02:54 +0100 Subject: [PATCH 1/6] Error when attempt to merge zero files. Previously the code would try to call cat with no arguments, which would wait for stdin and so do nothing and stall. This error condition could be triggered with a cluster job when Galaxy didn't get any output back. --- lib/galaxy/datatypes/data.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index d2b5b564b3b..103c63fe795 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -536,7 +536,9 @@ class Data( object ): TODO: Do we need to merge gzip files using gzjoin? cat seems to work, but might be brittle. Need to revisit this. """ - if len(split_files) == 1: + if not split_files: + raise ValueError('Asked to merge zero files as %s' % output_file) + elif len(split_files) == 1: cmd = 'mv -f %s %s' % ( split_files[0], output_file ) else: cmd = 'cat %s > %s' % ( ' '.join(split_files), output_file ) From b1de0a83b6ace84c4c51c2b71ce2b0e2058b26e9 Mon Sep 17 00:00:00 2001 From: Peter Cock Date: Fri, 19 Oct 2012 14:51:37 +0100 Subject: [PATCH 2/6] Don't attempt to merge zero files --- lib/galaxy/jobs/splitters/multi.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/jobs/splitters/multi.py b/lib/galaxy/jobs/splitters/multi.py index 26ce9efe8b8..6f704b30cae 100644 --- a/lib/galaxy/jobs/splitters/multi.py +++ b/lib/galaxy/jobs/splitters/multi.py @@ -115,6 +115,7 @@ def do_merge( job_wrapper, task_wrappers): try: working_directory = job_wrapper.working_directory task_dirs = [os.path.join(working_directory, x) for x in os.listdir(working_directory) if x.startswith('task_')] + assert task_dirs, "Should be at least one sub-task!" # TODO: Output datasets can be very complex. This doesn't handle metadata files outputs = job_wrapper.get_output_hdas_and_fnames() pickone_done = [] @@ -129,10 +130,16 @@ def do_merge( job_wrapper, task_wrappers): # Just include those files f in the output list for which the # file f exists; some files may not exist if a task fails. output_files = [ f for f in output_files if os.path.exists(f) ] - log.debug('files %s ' % output_files) - output_type.merge(output_files, output_file_name) - log.debug('merge finished: %s' % output_file_name) - pass # TODO: merge all the files + if output_files: + log.debug('files %s ' % output_files) + if len(output_files) < len(task_dirs): + log.debug('merging only %i out of expected %i files for %s' + % (len(output_files), len(task_dirs), output_file_name)) + output_type.merge(output_files, output_file_name) + log.debug('merge finished: %s' % output_file_name) + else: + log.debug('nothing to merge for %s (expected %i files)' + % (output_file_name, len(task_dirs))) elif output in pickone_outputs: # just pick one of them if output not in pickone_done: From b7d245fee77d5356de19f853e3aaeee7c16011b1 Mon Sep 17 00:00:00 2001 From: Peter Cock Date: Fri, 19 Oct 2012 15:23:02 +0100 Subject: [PATCH 3/6] Log nothing to merge to stderr --- lib/galaxy/jobs/splitters/multi.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/jobs/splitters/multi.py b/lib/galaxy/jobs/splitters/multi.py index 6f704b30cae..95f30b7f3b7 100644 --- a/lib/galaxy/jobs/splitters/multi.py +++ b/lib/galaxy/jobs/splitters/multi.py @@ -138,8 +138,10 @@ def do_merge( job_wrapper, task_wrappers): output_type.merge(output_files, output_file_name) log.debug('merge finished: %s' % output_file_name) else: - log.debug('nothing to merge for %s (expected %i files)' - % (output_file_name, len(task_dirs))) + msg = 'nothing to merge for %s (expected %i files)' \ + % (output_file_name, len(task_dirs)) + log.debug(msg) + stderr += msg + "\n" elif output in pickone_outputs: # just pick one of them if output not in pickone_done: From a986bcde0e84211e341e781d1e481f38463aaddc Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Fri, 19 Oct 2012 15:26:51 -0400 Subject: [PATCH 4/6] Minor enhancements for GenomeSpace import naming of secondary files. --- tools/genomespace/genomespace_file_browser.py | 10 +++++++++- tools/genomespace/genomespace_importer.py | 11 +++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/tools/genomespace/genomespace_file_browser.py b/tools/genomespace/genomespace_file_browser.py index 788ab2bf69c..7eb6829709b 100644 --- a/tools/genomespace/genomespace_file_browser.py +++ b/tools/genomespace/genomespace_file_browser.py @@ -39,6 +39,7 @@ GENOMESPACE_EXT_TO_GALAXY_EXT = {'rifles': 'rifles', 'gmt': 'gmt', 'gct': 'gct'} +VALID_CHARS = '.-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ ' def chunk_write( source_stream, target_stream, source_method = "read", target_method="write" ): source_method = getattr( source_stream, source_method ) @@ -113,6 +114,7 @@ def download_from_genomespace_file_browser( json_parameter_file, genomespace_sit name = name[len( file_url_prefix ):] file_numbers.append( int( name ) ) file_numbers.sort() + used_filenames = [] for file_num in file_numbers: url_key = "%s%i" % ( file_url_prefix, file_num ) download_url = datasource_params.get( url_key, None ) @@ -135,8 +137,14 @@ def download_from_genomespace_file_browser( json_parameter_file, genomespace_sit parsed_url = urlparse.urlparse( download_url ) query_params = urlparse.parse_qs( parsed_url[4] ) filename = urllib.unquote_plus( parsed_url[2].split( '/' )[-1] ) + if not filename: + filename = download_url if output_filename is None: - output_filename = os.path.join( datasource_params['__new_file_path__'], 'primary_%i_output%i_visible_%s' % ( hda_id, file_num, galaxy_ext ) ) + filename = ''.join( c in VALID_CHARS and c or '-' for c in filename ) + while filename in used_filenames: + filename = "-%s" % filename + used_filenames.append( filename ) + output_filename = os.path.join( datasource_params['__new_file_path__'], 'primary_%i_%s_visible_%s' % ( hda_id, filename, galaxy_ext ) ) else: if dataset_id is not None: metadata_parameter_file.write( "%s\n" % simplejson.dumps( dict( type = 'dataset', diff --git a/tools/genomespace/genomespace_importer.py b/tools/genomespace/genomespace_importer.py index 604a4d83748..1ec66bbf82f 100644 --- a/tools/genomespace/genomespace_importer.py +++ b/tools/genomespace/genomespace_importer.py @@ -43,7 +43,7 @@ GENOMESPACE_EXT_TO_GALAXY_EXT = {'rifles': 'rifles', 'gmt': 'gmt', 'gct': 'gct'} -VALID_CHARS = '.-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' +VALID_CHARS = '.-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ ' def chunk_write( source_stream, target_stream, source_method = "read", target_method="write" ): source_method = getattr( source_stream, source_method ) @@ -112,6 +112,7 @@ def download_from_genomespace_importer( username, token, json_parameter_file, ge datatypes_registry = Registry() datatypes_registry.load_datatypes( root_dir = json_params[ 'job_config' ][ 'GALAXY_ROOT_DIR' ], config = json_params[ 'job_config' ][ 'GALAXY_DATATYPES_CONF_FILE' ] ) url_param = datasource_params.get( file_url_name, None ) + used_filenames = [] for download_url in url_param.split( ',' ): using_temp_file = False parsed_url = urlparse.urlparse( download_url ) @@ -129,6 +130,8 @@ def download_from_genomespace_importer( username, token, json_parameter_file, ge parsed_url = urlparse.urlparse( download_url ) query_params = urlparse.parse_qs( parsed_url[4] ) filename = urllib.unquote_plus( parsed_url[2].split( '/' )[-1] ) + if not filename: + filename = download_url if output_filename is None: #need to use a temp file here, because we do not know the ext yet using_temp_file = True @@ -179,7 +182,11 @@ def download_from_genomespace_importer( username, token, json_parameter_file, ge name = "GenomeSpace importer on %s" % ( filename ) ) ) ) #if using tmp file, move the file to the new file path dir to get scooped up later if using_temp_file: - shutil.move( output_filename, os.path.join( datasource_params['__new_file_path__'], 'primary_%i_output%s_visible_%s' % ( hda_id, ''.join( c in VALID_CHARS and c or '-' for c in filename ), file_type ) ) ) + filename = ''.join( c in VALID_CHARS and c or '-' for c in filename ) + while filename in used_filenames: + filename = "-%s" % filename + used_filenames.append( filename ) + shutil.move( output_filename, os.path.join( datasource_params['__new_file_path__'], 'primary_%i_%s_visible_%s' % ( hda_id, filename, file_type ) ) ) dataset_id = None #only one primary dataset available output_filename = None #only have one filename available From fe8d5766df5a479b19262391e534a3c39dad6328 Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Fri, 19 Oct 2012 16:25:08 -0400 Subject: [PATCH 5/6] Some tool she drepository grid cleanup, and add the ability to mark a repository as deprecated / undeprecated. Repositories in the tool shed that are marked as deprecated have restricted tool shed features and will not be included in several queries. --- .../webapps/community/controllers/admin.py | 99 ++--- .../community/controllers/repository.py | 339 +++++++++++++----- .../controllers/repository_review.py | 128 +++++-- .../webapps/community/model/__init__.py | 4 +- lib/galaxy/webapps/community/model/mapping.py | 8 +- .../versions/0014_add_deprecated_column.py | 53 +++ .../webapps/community/category/grid.mako | 4 +- .../community/category/valid_grid.mako | 4 +- templates/webapps/community/index.mako | 7 +- .../repository/manage_repository.mako | 29 +- .../community/repository/view_repository.mako | 17 +- .../repository/view_tool_metadata.mako | 2 +- 12 files changed, 494 insertions(+), 200 deletions(-) create mode 100644 lib/galaxy/webapps/community/model/migrate/versions/0014_add_deprecated_column.py diff --git a/lib/galaxy/webapps/community/controllers/admin.py b/lib/galaxy/webapps/community/controllers/admin.py index 7aef41e610f..e0b25568737 100644 --- a/lib/galaxy/webapps/community/controllers/admin.py +++ b/lib/galaxy/webapps/community/controllers/admin.py @@ -7,7 +7,7 @@ from galaxy.web.form_builder import SelectField from galaxy.util import inflector from galaxy.util.shed_util import get_changectx_for_changeset, get_configured_ui from common import * -from repository import RepositoryListGrid, CategoryListGrid +from repository import RepositoryGrid, CategoryGrid from galaxy import eggs eggs.require( 'mercurial' ) @@ -17,7 +17,7 @@ import logging log = logging.getLogger( __name__ ) -class UserListGrid( grids.Grid ): +class UserGrid( grids.Grid ): # TODO: move this to an admin_common controller since it is virtually the same in the galaxy webapp. class UserLoginColumn( grids.TextColumn ): def get_value( self, trans, grid, user ): @@ -66,10 +66,10 @@ class UserListGrid( grids.Grid ): default_sort_key = "email" columns = [ UserLoginColumn( "Email", - key="email", - link=( lambda item: dict( operation="information", id=item.id ) ), - attach_popup=True, - filterable="advanced" ), + key="email", + link=( lambda item: dict( operation="information", id=item.id ) ), + attach_popup=True, + filterable="advanced" ), UserNameColumn( "User Name", key="username", attach_popup=False, @@ -116,7 +116,7 @@ class UserListGrid( grids.Grid ): def get_current_item( self, trans, **kwargs ): return trans.user -class RoleListGrid( grids.Grid ): +class RoleGrid( grids.Grid ): # TODO: move this to an admin_common controller since it is virtually the same in the galaxy webapp. class NameColumn( grids.TextColumn ): def get_value( self, trans, grid, role ): @@ -207,7 +207,7 @@ class RoleListGrid( grids.Grid ): def apply_query_filter( self, trans, query, **kwd ): return query.filter( model.Role.type != model.Role.types.PRIVATE ) -class GroupListGrid( grids.Grid ): +class GroupGrid( grids.Grid ): # TODO: move this to an admin_common controller since it is virtually the same in the galaxy webapp. class NameColumn( grids.TextColumn ): def get_value( self, trans, grid, group ): @@ -278,34 +278,35 @@ class GroupListGrid( grids.Grid ): preserve_state = False use_paging = True -class ManageCategoryListGrid( CategoryListGrid ): - columns = [ col for col in CategoryListGrid.columns ] +class ManageCategoryGrid( CategoryGrid ): + columns = [ col for col in CategoryGrid.columns ] # Override the NameColumn to include an Edit link - columns[ 0 ] = CategoryListGrid.NameColumn( "Name", - key="Category.name", - link=( lambda item: dict( operation="Edit", id=item.id ) ), - model_class=model.Category, - attach_popup=False ) + columns[ 0 ] = CategoryGrid.NameColumn( "Name", + key="Category.name", + link=( lambda item: dict( operation="Edit", id=item.id ) ), + model_class=model.Category, + attach_popup=False ) global_actions = [ grids.GridAction( "Add new category", dict( controller='admin', action='manage_categories', operation='create' ) ) ] -class AdminRepositoryListGrid( RepositoryListGrid ): - columns = [ RepositoryListGrid.NameColumn( "Name", - key="name", - link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), - attach_popup=True ), - RepositoryListGrid.DescriptionColumn( "Synopsis", - key="description", - attach_popup=False ), - RepositoryListGrid.MetadataRevisionColumn( "Metadata Revisions" ), - RepositoryListGrid.UserColumn( "Owner", - model_class=model.User, - link=( lambda item: dict( operation="repositories_by_user", id=item.id ) ), - attach_popup=False, - key="User.username" ), - RepositoryListGrid.EmailAlertsColumn( "Alert", attach_popup=False ), +class AdminRepositoryGrid( RepositoryGrid ): + columns = [ RepositoryGrid.NameColumn( "Name", + key="name", + link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), + attach_popup=True ), + RepositoryGrid.DescriptionColumn( "Synopsis", + key="description", + attach_popup=False ), + RepositoryGrid.MetadataRevisionColumn( "Metadata Revisions" ), + RepositoryGrid.UserColumn( "Owner", + model_class=model.User, + link=( lambda item: dict( operation="repositories_by_user", id=item.id ) ), + attach_popup=False, + key="User.username" ), + RepositoryGrid.EmailAlertsColumn( "Alert", attach_popup=False ), + RepositoryGrid.DeprecatedColumn( "Deprecated", attach_popup=False ), # Columns that are valid for filtering but are not visible. grids.DeletedColumn( "Deleted", key="deleted", @@ -316,7 +317,7 @@ class AdminRepositoryListGrid( RepositoryListGrid ): key="free-text-search", visible=False, filterable="standard" ) ) - operations = [ operation for operation in RepositoryListGrid.operations ] + operations = [ operation for operation in RepositoryGrid.operations ] operations.append( grids.GridOperation( "Delete", allow_multiple=False, condition=( lambda item: not item.deleted ), @@ -327,7 +328,7 @@ class AdminRepositoryListGrid( RepositoryListGrid ): async_compatible=False ) ) standard_filters = [] -class RepositoryMetadataListGrid( grids.Grid ): +class RepositoryMetadataGrid( grids.Grid ): class IdColumn( grids.IntegerColumn ): def get_value( self, trans, grid, repository_metadata ): return repository_metadata.id @@ -414,17 +415,18 @@ class RepositoryMetadataListGrid( grids.Grid ): preserve_state = False use_paging = True def build_initial_query( self, trans, **kwd ): - return trans.sa_session.query( self.model_class ) \ - .join( model.Repository.table ) + return trans.sa_session.query( model.RepositoryMetadata ) \ + .join( model.Repository.table ) \ + .filter( model.Repository.table.c.deprecated == False ) class AdminController( BaseUIController, Admin ): - user_list_grid = UserListGrid() - role_list_grid = RoleListGrid() - group_list_grid = GroupListGrid() - manage_category_list_grid = ManageCategoryListGrid() - repository_list_grid = AdminRepositoryListGrid() - repository_metadata_list_grid = RepositoryMetadataListGrid() + user_list_grid = UserGrid() + role_list_grid = RoleGrid() + group_list_grid = GroupGrid() + manage_category_grid = ManageCategoryGrid() + repository_grid = AdminRepositoryGrid() + repository_metadata_grid = RepositoryMetadataGrid() @web.expose @web.require_admin @@ -477,7 +479,7 @@ class AdminController( BaseUIController, Admin ): return self.delete_repository( trans, **kwd ) elif operation == "undelete": return self.undelete_repository( trans, **kwd ) - # The changeset_revision_select_field in the RepositoryListGrid performs a refresh_on_change + # The changeset_revision_select_field in the RepositoryGrid performs a refresh_on_change # which sends in request parameters like changeset_revison_1, changeset_revision_2, etc. One # of the many select fields on the grid performed the refresh_on_change, so we loop through # all of the received values to see which value is not the repository tip. If we find it, we @@ -495,7 +497,7 @@ class AdminController( BaseUIController, Admin ): id=trans.security.encode_id( repository.id ), changeset_revision=v ) ) # Render the list view - return self.repository_list_grid( trans, **kwd ) + return self.repository_grid( trans, **kwd ) @web.expose @web.require_admin def browse_repository_metadata( self, trans, **kwd ): @@ -515,7 +517,7 @@ class AdminController( BaseUIController, Admin ): return trans.response.send_redirect( web.url_for( controller='repository', action='browse_repositories', **kwd ) ) - return self.repository_metadata_list_grid( trans, **kwd ) + return self.repository_metadata_grid( trans, **kwd ) @web.expose @web.require_admin def create_category( self, trans, **kwd ): @@ -645,9 +647,9 @@ class AdminController( BaseUIController, Admin ): @web.require_admin def manage_categories( self, trans, **kwd ): if 'f-free-text-search' in kwd: - # Trick to enable searching repository name, description from the CategoryListGrid. - # What we've done is rendered the search box for the RepositoryListGrid on the grid.mako - # template for the CategoryListGrid. See ~/templates/webapps/community/category/grid.mako. + # Trick to enable searching repository name, description from the CategoryGrid. + # What we've done is rendered the search box for the RepositoryGrid on the grid.mako + # template for the CategoryGrid. See ~/templates/webapps/community/category/grid.mako. # Since we are searching repositories and not categories, redirect to browse_repositories(). return trans.response.send_redirect( web.url_for( controller='admin', action='browse_repositories', @@ -674,7 +676,7 @@ class AdminController( BaseUIController, Admin ): return trans.response.send_redirect( web.url_for( controller='admin', action='edit_category', **kwd ) ) - return self.manage_category_list_grid( trans, **kwd ) + return self.manage_category_grid( trans, **kwd ) @web.expose @web.require_admin def regenerate_statistics( self, trans, **kwd ): @@ -732,7 +734,8 @@ class AdminController( BaseUIController, Admin ): multiple=True, display='checkboxes' ) for repository in trans.sa_session.query( trans.model.Repository ) \ - .filter( trans.model.Repository.table.c.deleted == False ) \ + .filter( and_( trans.model.Repository.table.c.deleted == False, + trans.model.Repository.table.c.deprecated == False ) ) \ .order_by( trans.model.Repository.table.c.name, trans.model.Repository.table.c.user_id ): owner = repository.user.username diff --git a/lib/galaxy/webapps/community/controllers/repository.py b/lib/galaxy/webapps/community/controllers/repository.py index fa485d3bc5b..bc40742bcf1 100644 --- a/lib/galaxy/webapps/community/controllers/repository.py +++ b/lib/galaxy/webapps/community/controllers/repository.py @@ -25,7 +25,7 @@ log = logging.getLogger( __name__ ) VALID_REPOSITORYNAME_RE = re.compile( "^[a-z0-9\_]+$" ) -class CategoryListGrid( grids.Grid ): +class CategoryGrid( grids.Grid ): class NameColumn( grids.TextColumn ): def get_value( self, trans, grid, category ): return category.name @@ -65,14 +65,14 @@ class CategoryListGrid( grids.Grid ): preserve_state = False use_paging = True -class ValidCategoryListGrid( CategoryListGrid ): +class ValidCategoryGrid( CategoryGrid ): class RepositoriesColumn( grids.TextColumn ): def get_value( self, trans, grid, category ): if category.repositories: viewable_repositories = 0 for rca in category.repositories: repository = rca.repository - if repository.downloadable_revisions: + if not repository.deprecated and repository.downloadable_revisions: viewable_repositories += 1 return viewable_repositories return 0 @@ -81,13 +81,13 @@ class ValidCategoryListGrid( CategoryListGrid ): template='/webapps/community/category/valid_grid.mako' default_sort_key = "name" columns = [ - CategoryListGrid.NameColumn( "Name", - key="Category.name", - link=( lambda item: dict( operation="valid_repositories_by_category", id=item.id ) ), - attach_popup=False ), - CategoryListGrid.DescriptionColumn( "Description", - key="Category.description", - attach_popup=False ), + CategoryGrid.NameColumn( "Name", + key="Category.name", + link=( lambda item: dict( operation="valid_repositories_by_category", id=item.id ) ), + attach_popup=False ), + CategoryGrid.DescriptionColumn( "Description", + key="Category.description", + attach_popup=False ), # Columns that are valid for filtering but are not visible. RepositoriesColumn( "Valid repositories", model_class=model.Repository, @@ -102,7 +102,7 @@ class ValidCategoryListGrid( CategoryListGrid ): preserve_state = False use_paging = True -class RepositoryListGrid( grids.Grid ): +class RepositoryGrid( grids.Grid ): class NameColumn( grids.TextColumn ): def get_value( self, trans, grid, repository ): return repository.name @@ -188,6 +188,11 @@ class RepositoryListGrid( grids.Grid ): if trans.user and repository.email_alerts and trans.user.email in from_json_string( repository.email_alerts ): return 'yes' return '' + class DeprecatedColumn( grids.TextColumn ): + def get_value( self, trans, grid, repository ): + if repository.deprecated: + return 'yes' + return '' # Grid definition title = "Repositories" model_class = model.Repository @@ -248,21 +253,83 @@ class RepositoryListGrid( grids.Grid ): .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) -class EmailAlertsRepositoryListGrid( RepositoryListGrid ): +class RepositoriesIOwnGrid( RepositoryGrid ): + title = "Repositories I own" columns = [ - RepositoryListGrid.NameColumn( "Name", - key="name", - link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), + RepositoryGrid.NameColumn( "Name", + key="name", + link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), + attach_popup=True ), + RepositoryGrid.MetadataRevisionColumn( "Metadata Revisions" ), + RepositoryGrid.TipRevisionColumn( "Tip Revision" ), + RepositoryGrid.CategoryColumn( "Category", + model_class=model.Category, + key="Category.name", attach_popup=False ), - RepositoryListGrid.DescriptionColumn( "Synopsis", - key="description", - attach_popup=False ), - RepositoryListGrid.UserColumn( "Owner", - model_class=model.User, - link=( lambda item: dict( operation="repositories_by_user", id=item.id ) ), - attach_popup=False, - key="User.username" ), - RepositoryListGrid.EmailAlertsColumn( "Alert", attach_popup=False ), + RepositoryGrid.DeprecatedColumn( "Deprecated" ) + ] + columns.append( grids.MulticolFilterColumn( "Search repository name", + cols_to_filter=[ columns[0] ], + key="free-text-search", + visible=False, + filterable="standard" ) ) + operations = [ grids.GridOperation( "Mark as deprecated", + allow_multiple=False, + condition=( lambda item: not item.deleted and not item.deprecated ), + async_compatible=False ), + grids.GridOperation( "Mark as not deprecated", + allow_multiple=False, + condition=( lambda item: not item.deleted and item.deprecated ), + async_compatible=False ) ] + def build_initial_query( self, trans, **kwd ): + return trans.sa_session.query( model.Repository ) \ + .filter( model.Repository.table.c.user_id == trans.user.id ) \ + .join( model.User.table ) \ + .outerjoin( model.RepositoryCategoryAssociation.table ) \ + .outerjoin( model.Category.table ) + +class DeprecatedRepositoriesIOwnGrid( RepositoriesIOwnGrid ): + title = "Deprecated repositories I own" + columns = [ + RepositoriesIOwnGrid.NameColumn( "Name", + key="name", + link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), + attach_popup=True ), + RepositoriesIOwnGrid.MetadataRevisionColumn( "Metadata Revisions" ), + RepositoriesIOwnGrid.TipRevisionColumn( "Tip Revision" ), + RepositoriesIOwnGrid.CategoryColumn( "Category", + model_class=model.Category, + key="Category.name", + attach_popup=False ), + ] + columns.append( grids.MulticolFilterColumn( "Search repository name", + cols_to_filter=[ columns[0] ], + key="free-text-search", + visible=False, + filterable="standard" ) ) + def build_initial_query( self, trans, **kwd ): + return trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.user_id == trans.user.id, + model.Repository.table.c.deprecated == True ) ) \ + .join( model.User.table ) \ + .outerjoin( model.RepositoryCategoryAssociation.table ) \ + .outerjoin( model.Category.table ) + +class EmailAlertsRepositoryGrid( RepositoryGrid ): + columns = [ + RepositoryGrid.NameColumn( "Name", + key="name", + link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), + attach_popup=False ), + RepositoryGrid.DescriptionColumn( "Synopsis", + key="description", + attach_popup=False ), + RepositoryGrid.UserColumn( "Owner", + model_class=model.User, + link=( lambda item: dict( operation="repositories_by_user", id=item.id ) ), + attach_popup=False, + key="User.username" ), + RepositoryGrid.EmailAlertsColumn( "Alert", attach_popup=False ), # Columns that are valid for filtering but are not visible. grids.DeletedColumn( "Deleted", key="deleted", @@ -277,29 +344,67 @@ class EmailAlertsRepositoryListGrid( RepositoryListGrid ): grids.GridAction( "User preferences", dict( controller='user', action='index', cntrller='repository' ) ) ] -class WritableRepositoryListGrid( RepositoryListGrid ): +class MyWritableRepositoriesGrid( RepositoryGrid ): + # This grid filters out repositories that have been marked as deprecated. + columns = [ + RepositoryGrid.NameColumn( "Name", + key="name", + link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), + attach_popup=True ), + RepositoryGrid.MetadataRevisionColumn( "Metadata Revisions" ), + RepositoryGrid.TipRevisionColumn( "Tip Revision" ), + RepositoryGrid.UserColumn( "Owner", + model_class=model.User, + link=( lambda item: dict( operation="repositories_by_user", id=item.id ) ), + attach_popup=False, + key="User.username" ), + RepositoryGrid.EmailAlertsColumn( "Alert", attach_popup=False ), + # Columns that are valid for filtering but are not visible. + RepositoryGrid.EmailColumn( "Email", + model_class=model.User, + key="email", + visible=False ), + RepositoryGrid.RepositoryCategoryColumn( "Category", + model_class=model.Category, + key="Category.name", + visible=False ), + grids.DeletedColumn( "Deleted", + key="deleted", + visible=False, + filterable="advanced" ) + ] + columns.append( grids.MulticolFilterColumn( "Search repository name", + cols_to_filter=[ columns[0] ], + key="free-text-search", + visible=False, + filterable="standard" ) ) + operations = [ grids.GridOperation( "Receive email alerts", + allow_multiple=False, + condition=( lambda item: not item.deleted ), + async_compatible=False ) ] def build_initial_query( self, trans, **kwd ): # TODO: improve performance by adding a db table associating users with repositories for which they have write access. - username = kwd[ 'username' ] + username = trans.user.username clause_list = [] - for repository in trans.sa_session.query( self.model_class ) \ - .filter( self.model_class.table.c.deleted == False ): + for repository in trans.sa_session.query( model.Repository ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.table.c.deleted == False ) ): allow_push = repository.allow_push if allow_push: allow_push_usernames = allow_push.split( ',' ) if username in allow_push_usernames: - clause_list.append( self.model_class.table.c.id == repository.id ) + clause_list.append( model.Repository.table.c.id == repository.id ) if clause_list: - return trans.sa_session.query( self.model_class ) \ + return trans.sa_session.query( model.Repository ) \ .filter( or_( *clause_list ) ) \ .join( model.User.table ) \ .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) # Return an empty query. - return trans.sa_session.query( self.model_class ) \ - .filter( self.model_class.table.c.id < 0 ) + return [] -class ValidRepositoryListGrid( RepositoryListGrid ): +class ValidRepositoryGrid( RepositoryGrid ): + # This grid filters out repositories that have been marked as deprecated. class CategoryColumn( grids.TextColumn ): def get_value( self, trans, grid, repository ): rval = '
    ' @@ -330,16 +435,16 @@ class ValidRepositoryListGrid( RepositoryListGrid ): return '' title = "Valid repositories" columns = [ - RepositoryListGrid.NameColumn( "Name", - key="name", - attach_popup=True ), - RepositoryListGrid.DescriptionColumn( "Synopsis", - key="description", - attach_popup=False ), + RepositoryGrid.NameColumn( "Name", + key="name", + attach_popup=True ), + RepositoryGrid.DescriptionColumn( "Synopsis", + key="description", + attach_popup=False ), RevisionColumn( "Installable Revisions" ), - RepositoryListGrid.UserColumn( "Owner", - model_class=model.User, - attach_popup=False ), + RepositoryGrid.UserColumn( "Owner", + model_class=model.User, + attach_popup=False ), # Columns that are valid for filtering but are not visible. RepositoryCategoryColumn( "Category", model_class=model.Category, @@ -355,22 +460,25 @@ class ValidRepositoryListGrid( RepositoryListGrid ): def build_initial_query( self, trans, **kwd ): if 'id' in kwd: # The user is browsing categories of valid repositories, so filter the request by the received id, which is a category id. - return trans.sa_session.query( self.model_class ) \ + return trans.sa_session.query( model.Repository ) \ + .filter( model.Repository.table.c.deprecated == False ) \ .join( model.RepositoryMetadata.table ) \ .join( model.User.table ) \ .join( model.RepositoryCategoryAssociation.table ) \ .join( model.Category.table ) \ .filter( and_( model.Category.table.c.id == trans.security.decode_id( kwd[ 'id' ] ), model.RepositoryMetadata.table.c.downloadable == True ) ) - # The user performed a free text search on the ValidCategoryListGrid. - return trans.sa_session.query( self.model_class ) \ + # The user performed a free text search on the ValidCategoryGrid. + return trans.sa_session.query( model.Repository ) \ + .filter( model.Repository.table.c.deprecated == False ) \ .join( model.RepositoryMetadata.table ) \ .join( model.User.table ) \ .outerjoin( model.RepositoryCategoryAssociation.table ) \ .outerjoin( model.Category.table ) \ .filter( model.RepositoryMetadata.table.c.downloadable == True ) -class MatchedRepositoryListGrid( grids.Grid ): +class MatchedRepositoryGrid( grids.Grid ): + # This grid filters out repositories that have been marked as deprecated. class NameColumn( grids.TextColumn ): def get_value( self, trans, grid, repository_metadata ): return repository_metadata.repository.name @@ -413,38 +521,38 @@ class MatchedRepositoryListGrid( grids.Grid ): if match_tuples: for match_tuple in match_tuples: repository_id, changeset_revision = match_tuple - clause_list.append( "%s=%d and %s='%s'" % ( self.model_class.table.c.repository_id, + clause_list.append( "%s=%d and %s='%s'" % ( model.RepositoryMetadata.table.c.repository_id, int( repository_id ), - self.model_class.table.c.changeset_revision, + model.RepositoryMetadata.table.c.changeset_revision, changeset_revision ) ) - return trans.sa_session.query( self.model_class ) \ + return trans.sa_session.query( model.RepositoryMetadata ) \ .join( model.Repository ) \ + .filter( model.Repository.table.c.deprecated == False ) \ .join( model.User.table ) \ .filter( or_( *clause_list ) ) \ .order_by( model.Repository.name ) # Return an empty query - return trans.sa_session.query( self.model_class ) \ - .join( model.Repository ) \ - .join( model.User.table ) \ - .filter( self.model_class.table.c.repository_id == 0 ) + return [] -class InstallMatchedRepositoryListGrid( MatchedRepositoryListGrid ): - columns = [ col for col in MatchedRepositoryListGrid.columns ] +class InstallMatchedRepositoryGrid( MatchedRepositoryGrid ): + columns = [ col for col in MatchedRepositoryGrid.columns ] # Override the NameColumn - columns[ 0 ] = MatchedRepositoryListGrid.NameColumn( "Name", - link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), - attach_popup=False ) + columns[ 0 ] = MatchedRepositoryGrid.NameColumn( "Name", + link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), + attach_popup=False ) class RepositoryController( BaseUIController, ItemRatings ): - install_matched_repository_list_grid = InstallMatchedRepositoryListGrid() - matched_repository_list_grid = MatchedRepositoryListGrid() - valid_repository_list_grid = ValidRepositoryListGrid() - repository_list_grid = RepositoryListGrid() - email_alerts_repository_list_grid = EmailAlertsRepositoryListGrid() - category_list_grid = CategoryListGrid() - valid_category_list_grid = ValidCategoryListGrid() - writable_repository_list_grid = WritableRepositoryListGrid() + install_matched_repository_grid = InstallMatchedRepositoryGrid() + matched_repository_grid = MatchedRepositoryGrid() + valid_repository_grid = ValidRepositoryGrid() + repository_grid = RepositoryGrid() + email_alerts_repository_grid = EmailAlertsRepositoryGrid() + category_grid = CategoryGrid() + valid_category_grid = ValidCategoryGrid() + my_writable_repositories_grid = MyWritableRepositoriesGrid() + repositories_i_own_grid = RepositoriesIOwnGrid() + deprecated_repositories_i_own_grid = DeprecatedRepositoriesIOwnGrid() def __add_hgweb_config_entry( self, trans, repository, repository_path ): # Add an entry in the hgweb.config file for a new repository. An entry looks something like: @@ -470,8 +578,8 @@ class RepositoryController( BaseUIController, ItemRatings ): def browse_categories( self, trans, **kwd ): # The request came from the tool shed. if 'f-free-text-search' in kwd: - # Trick to enable searching repository name, description from the CategoryListGrid. What we've done is rendered the search box for the - # RepositoryListGrid on the grid.mako template for the CategoryListGrid. See ~/templates/webapps/community/category/grid.mako. Since we + # Trick to enable searching repository name, description from the CategoryGrid. What we've done is rendered the search box for the + # RepositoryGrid on the grid.mako template for the CategoryGrid. See ~/templates/webapps/community/category/grid.mako. Since we # are searching repositories and not categories, redirect to browse_repositories(). if 'id' in kwd and 'f-free-text-search' in kwd and kwd[ 'id' ] == kwd[ 'f-free-text-search' ]: # The value of 'id' has been set to the search string, which is a repository name. We'll try to get the desired encoded repository id to pass on. @@ -491,7 +599,7 @@ class RepositoryController( BaseUIController, ItemRatings ): return trans.response.send_redirect( web.url_for( controller='repository', action='browse_repositories', **kwd ) ) - return self.category_list_grid( trans, **kwd ) + return self.category_grid( trans, **kwd ) @web.expose def browse_invalid_tools( self, trans, **kwd ): params = util.Params( kwd ) @@ -564,13 +672,27 @@ class RepositoryController( BaseUIController, ItemRatings ): for k, v in kwd.items(): if k.startswith( 'f-' ): del kwd[ k ] - kwd[ 'f-email' ] = trans.user.email + return self.repositories_i_own_grid( trans, **kwd ) + elif operation == "deprecated_repositories_i_own": + # Eliminate the current filters if any exist. + for k, v in kwd.items(): + if k.startswith( 'f-' ): + del kwd[ k ] + return self.deprecated_repositories_i_own_grid( trans, **kwd ) + elif operation in [ 'mark as deprecated', 'mark as not deprecated' ]: + # Eliminate the current filters if any exist. + for k, v in kwd.items(): + if k.startswith( 'f-' ): + del kwd[ k ] + kwd[ 'mark_deprecated' ] = operation == 'mark as deprecated' + return trans.response.send_redirect( web.url_for( controller='repository', + action='deprecate', + **kwd ) ) elif operation == "reviewed_repositories_i_own": return trans.response.send_redirect( web.url_for( controller='repository_review', action='reviewed_repositories_i_own' ) ) - elif operation == "writable_repositories": - kwd[ 'username' ] = trans.user.username - return self.writable_repository_list_grid( trans, **kwd ) + elif operation == "my_writable_repositories": + return self.my_writable_repositories_grid( trans, **kwd ) elif operation == "repositories_by_category": # Eliminate the current filters if any exist. for k, v in kwd.items(): @@ -590,7 +712,7 @@ class RepositoryController( BaseUIController, ItemRatings ): kwd[ 'message' ] = 'You must be logged in to set email alerts.' kwd[ 'status' ] = 'error' del kwd[ 'operation' ] - # The changeset_revision_select_field in the RepositoryListGrid performs a refresh_on_change + # The changeset_revision_select_field in the RepositoryGrid performs a refresh_on_change # which sends in request parameters like changeset_revison_1, changeset_revision_2, etc. One # of the many select fields on the grid performed the refresh_on_change, so we loop through # all of the received values to see which value is not the repository tip. If we find it, we @@ -607,7 +729,7 @@ class RepositoryController( BaseUIController, ItemRatings ): operation='view_or_manage_repository', id=trans.security.encode_id( repository.id ), changeset_revision=v ) ) - return self.repository_list_grid( trans, **kwd ) + return self.repository_grid( trans, **kwd ) @web.expose def browse_repository( self, trans, id, **kwd ): params = util.Params( kwd ) @@ -637,7 +759,7 @@ class RepositoryController( BaseUIController, ItemRatings ): if kwd[ 'f-free-text-search' ] == 'All': # The user performed a search, then clicked the "x" to eliminate the search criteria. new_kwd = {} - return self.valid_category_list_grid( trans, **new_kwd ) + return self.valid_category_grid( trans, **new_kwd ) # Since we are searching valid repositories and not categories, redirect to browse_valid_repositories(). if 'id' in kwd and 'f-free-text-search' in kwd and kwd[ 'id' ] == kwd[ 'f-free-text-search' ]: # The value of 'id' has been set to the search string, which is a repository name. @@ -658,7 +780,7 @@ class RepositoryController( BaseUIController, ItemRatings ): return trans.response.send_redirect( web.url_for( controller='repository', action='browse_valid_repositories', **kwd ) ) - return self.valid_category_list_grid( trans, **kwd ) + return self.valid_category_grid( trans, **kwd ) @web.expose def browse_valid_repositories( self, trans, **kwd ): galaxy_url = kwd.get( 'galaxy_url', None ) @@ -667,7 +789,7 @@ class RepositoryController( BaseUIController, ItemRatings ): # The user browsed to a category and then entered a search string, so get the category associated with it's value. category_name = kwd[ 'f-Category.name' ] category = get_category_by_name( trans, category_name ) - # Set the id value in kwd since it is required by the ValidRepositoryListGrid.build_initial_query method. + # Set the id value in kwd since it is required by the ValidRepositoryGrid.build_initial_query method. kwd[ 'id' ] = trans.security.encode_id( category.id ) if galaxy_url: trans.set_cookie( galaxy_url, name='toolshedgalaxyurl' ) @@ -690,7 +812,7 @@ class RepositoryController( BaseUIController, ItemRatings ): category_id = kwd.get( 'id', None ) category = get_category( trans, category_id ) kwd[ 'f-Category.name' ] = category.name - # The changeset_revision_select_field in the ValidRepositoryListGrid performs a refresh_on_change which sends in request parameters like + # The changeset_revision_select_field in the ValidRepositoryGrid performs a refresh_on_change which sends in request parameters like # changeset_revison_1, changeset_revision_2, etc. One of the many select fields on the grid performed the refresh_on_change, so we loop # through all of the received values to see which value is not the repository tip. If we find it, we know the refresh_on_change occurred # and we have the necessary repository id and change set revision to pass on. @@ -708,11 +830,11 @@ class RepositoryController( BaseUIController, ItemRatings ): url_args = dict( action='browse_valid_repositories', operation='preview_tools_in_changeset', repository_id=repository_id ) - self.valid_repository_list_grid.operations = [ grids.GridOperation( "Preview and install", + self.valid_repository_grid.operations = [ grids.GridOperation( "Preview and install", url_args=url_args, allow_multiple=False, async_compatible=False ) ] - return self.valid_repository_list_grid( trans, **kwd ) + return self.valid_repository_grid( trans, **kwd ) def __build_allow_push_select_field( self, trans, current_push_list, selected_value='none' ): options = [] for user in trans.sa_session.query( trans.model.User ): @@ -931,6 +1053,27 @@ class RepositoryController( BaseUIController, ItemRatings ): message=message, status=status ) @web.expose + @web.require_login( "deprecate repository" ) + def deprecate( self, trans, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + repository_id = params.get( 'id', None ) + repository = get_repository( trans, repository_id ) + mark_deprecated = util.string_as_bool( params.get( 'mark_deprecated', False ) ) + repository.deprecated = mark_deprecated + trans.sa_session.add( repository ) + trans.sa_session.flush() + if mark_deprecated: + message = 'The repository %s has been marked as deprecated.' % repository.name + else: + message = 'The repository %s has been marked as not deprecated.' % repository.name + trans.response.send_redirect( web.url_for( controller='repository', + action='browse_repositories', + operation='repositories_i_own', + message=message, + status=status ) ) + @web.expose def display_tool( self, trans, repository_id, tool_config, changeset_revision, **kwd ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) @@ -1049,16 +1192,16 @@ class RepositoryController( BaseUIController, ItemRatings ): dict( controller='repository', action='find_tools' ) ), grids.GridAction( "Search for workflows", dict( controller='repository', action='find_workflows' ) ) ] - self.install_matched_repository_list_grid.global_actions = global_actions + self.install_matched_repository_grid.global_actions = global_actions install_url_args = dict( controller='repository', action='find_tools' ) operations = [ grids.GridOperation( "Install", url_args=install_url_args, allow_multiple=True, async_compatible=False ) ] - self.install_matched_repository_list_grid.operations = operations - return self.install_matched_repository_list_grid( trans, **kwd ) + self.install_matched_repository_grid.operations = operations + return self.install_matched_repository_grid( trans, **kwd ) else: kwd[ 'message' ] = "tool id: %s
    tool name: %s
    tool version: %s
    exact matches only: %s" % \ ( self.__stringify( tool_ids ), self.__stringify( tool_names ), self.__stringify( tool_versions ), str( exact_matches_checked ) ) - self.matched_repository_list_grid.title = "Repositories with matching tools" - return self.matched_repository_list_grid( trans, **kwd ) + self.matched_repository_grid.title = "Repositories with matching tools" + return self.matched_repository_grid( trans, **kwd ) else: message = "No search performed - each field must contain the same number of comma-separated items." status = "error" @@ -1135,16 +1278,16 @@ class RepositoryController( BaseUIController, ItemRatings ): dict( controller='repository', action='find_tools' ) ), grids.GridAction( "Search for workflows", dict( controller='repository', action='find_workflows' ) ) ] - self.install_matched_repository_list_grid.global_actions = global_actions + self.install_matched_repository_grid.global_actions = global_actions install_url_args = dict( controller='repository', action='find_workflows' ) operations = [ grids.GridOperation( "Install", url_args=install_url_args, allow_multiple=True, async_compatible=False ) ] - self.install_matched_repository_list_grid.operations = operations - return self.install_matched_repository_list_grid( trans, **kwd ) + self.install_matched_repository_grid.operations = operations + return self.install_matched_repository_grid( trans, **kwd ) else: kwd[ 'message' ] = "workflow name: %s
    exact matches only: %s" % \ ( self.__stringify( workflow_names ), str( exact_matches_checked ) ) - self.matched_repository_list_grid.title = "Repositories with matching workflows" - return self.matched_repository_list_grid( trans, **kwd ) + self.matched_repository_grid.title = "Repositories with matching workflows" + return self.matched_repository_grid( trans, **kwd ) else: message = "No search performed - each field must contain the same number of comma-separated items." status = "error" @@ -1407,17 +1550,24 @@ class RepositoryController( BaseUIController, ItemRatings ): status = params.get( 'status', 'done' ) # See if there are any RepositoryMetadata records since menu items require them. repository_metadata = trans.sa_session.query( model.RepositoryMetadata ).first() - # See if the current user owns any repositories that have been reviewed. - has_reviewed_repositories = False current_user = trans.user + has_reviewed_repositories = False + has_deprecated_repositories = False if current_user: + # See if the current user owns any repositories that have been reviewed. for repository in current_user.active_repositories: if repository.reviewed_revisions: has_reviewed_repositories = True break + # See if the current user has any repositories that have been marked as deprecated. + for repository in current_user.active_repositories: + if repository.deprecated: + has_deprecated_repositories = True + break return trans.fill_template( '/webapps/community/index.mako', repository_metadata=repository_metadata, has_reviewed_repositories=has_reviewed_repositories, + has_deprecated_repositories=has_deprecated_repositories, message=message, status=status ) @web.expose @@ -1738,7 +1888,8 @@ class RepositoryController( BaseUIController, ItemRatings ): kwd[ 'message' ] = 'You must be logged in to set email alerts.' kwd[ 'status' ] = 'error' del kwd[ 'operation' ] - return self.email_alerts_repository_list_grid( trans, **kwd ) + self.email_alerts_repository_grid.title = "Set email alerts for repository changes" + return self.email_alerts_repository_grid( trans, **kwd ) def __new_state( self, trans, all_pages=False ): """ Create a new `DefaultToolState` for this tool. It will not be initialized diff --git a/lib/galaxy/webapps/community/controllers/repository_review.py b/lib/galaxy/webapps/community/controllers/repository_review.py index cfcc3f6136b..4f82a33358c 100644 --- a/lib/galaxy/webapps/community/controllers/repository_review.py +++ b/lib/galaxy/webapps/community/controllers/repository_review.py @@ -7,7 +7,7 @@ from galaxy.web.framework.helpers import time_ago, iff, grids from galaxy.model.orm import * from sqlalchemy.sql.expression import func from common import * -from repository import RepositoryListGrid +from repository import RepositoryGrid from galaxy.util.shed_util import get_configured_ui from galaxy.util.odict import odict @@ -48,36 +48,33 @@ class ComponentGrid( grids.Grid ): preserve_state = False use_paging = True -class RepositoriesWithReviewsGrid( RepositoryListGrid ): +class RepositoriesWithReviewsGrid( RepositoryGrid ): + # This grid filters out repositories that have been marked as deprecated. class ReviewersColumn( grids.TextColumn ): def get_value( self, trans, grid, repository ): + rval = '' if repository.reviewers: - rval = '' for user in repository.reviewers: - rval += '%s
    ' % user.username - return rval - return '' - title = "All reviewed Repositories" + rval += '' % trans.security.encode_id( user.id ) + rval += '%s | ' % user.username + rval = rval.rstrip( ' | ' ) + return rval + title = "All reviewed repositories" model_class = model.Repository template='/webapps/community/repository_review/grid.mako' default_sort_key = "Repository.name" columns = [ - RepositoryListGrid.NameColumn( "Repository name", - key="name", - link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), - attach_popup=True ), - RepositoryListGrid.DescriptionColumn( "Synopsis", - key="description", - attach_popup=False ), - RepositoryListGrid.WithReviewsRevisionColumn( "Reviewed revisions" ), - RepositoryListGrid.WithoutReviewsRevisionColumn( "Revisions for review" ), - RepositoryListGrid.UserColumn( "Owner", - attach_popup=False ), - ReviewersColumn( "Reviewers", - attach_popup=False ) + RepositoryGrid.NameColumn( "Repository name", + key="name", + link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), + attach_popup=True ), + RepositoryGrid.WithReviewsRevisionColumn( "Reviewed revisions" ), + RepositoryGrid.WithoutReviewsRevisionColumn( "Revisions for review" ), + RepositoryGrid.UserColumn( "Owner", attach_popup=False ), + ReviewersColumn( "Reviewers", attach_popup=False ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name, description", - cols_to_filter=[ columns[0], columns[1] ], + columns.append( grids.MulticolFilterColumn( "Search repository name", + cols_to_filter=[ columns[0] ], key="free-text-search", visible=False, filterable="standard" ) ) @@ -89,12 +86,14 @@ class RepositoriesWithReviewsGrid( RepositoryListGrid ): ] def build_initial_query( self, trans, **kwd ): return trans.sa_session.query( model.Repository ) \ + .filter( model.Repository.table.c.deprecated == False ) \ .join( ( model.RepositoryReview.table, model.RepositoryReview.table.c.repository_id == model.Repository.table.c.id ) ) \ .join( ( model.User.table, model.User.table.c.id == model.Repository.table.c.user_id ) ) \ .outerjoin( ( model.ComponentReview.table, model.ComponentReview.table.c.repository_review_id == model.RepositoryReview.table.c.id ) ) \ .outerjoin( ( model.Component.table, model.Component.table.c.id == model.ComponentReview.table.c.component_id ) ) class RepositoriesWithoutReviewsGrid( RepositoriesWithReviewsGrid ): + # This grid filters out repositories that have been marked as deprecated. title = "Repositories with no reviews" columns = [ RepositoriesWithReviewsGrid.NameColumn( "Repository name", @@ -119,12 +118,15 @@ class RepositoriesWithoutReviewsGrid( RepositoriesWithReviewsGrid ): async_compatible=False ) ] def build_initial_query( self, trans, **kwd ): return trans.sa_session.query( model.Repository ) \ - .filter( model.Repository.reviews == None ) \ + .filter( and_( model.Repository.table.c.deprecated == False, + model.Repository.reviews == None ) ) \ .join( model.User.table ) class RepositoriesReviewedByMeGrid( RepositoriesWithReviewsGrid ): + # This grid filters out repositories that have been marked as deprecated. def build_initial_query( self, trans, **kwd ): return trans.sa_session.query( model.Repository ) \ + .filter( model.Repository.table.c.deprecated == False ) \ .join( ( model.RepositoryReview.table, model.RepositoryReview.table.c.repository_id == model.Repository.table.c.id ) ) \ .filter( model.RepositoryReview.table.c.user_id == trans.user.id ) \ .join( ( model.User.table, model.User.table.c.id == model.RepositoryReview.table.c.user_id ) ) \ @@ -132,6 +134,7 @@ class RepositoriesReviewedByMeGrid( RepositoriesWithReviewsGrid ): .outerjoin( ( model.Component.table, model.Component.table.c.id == model.ComponentReview.table.c.component_id ) ) class RepositoryReviewsByUserGrid( grids.Grid ): + # This grid filters out repositories that have been marked as deprecated. class RepositoryNameColumn( grids.TextColumn ): def get_value( self, trans, grid, review ): return review.repository.name @@ -166,33 +169,61 @@ class RepositoryReviewsByUserGrid( grids.Grid ): default_sort_key = 'repository_id' columns = [ RepositoryNameColumn( "Repository Name", - model_class=model.Repository, - key="Repository.name", - attach_popup=False ), + model_class=model.Repository, + key="Repository.name", + link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), + attach_popup=True ), RepositoryDescriptionColumn( "Description", - model_class=model.Repository, - key="Repository.description", - attach_popup=False ), - RevisionColumn( "Revision", - attach_popup=False ), - RatingColumn( "Rating", - attach_popup=False ) + model_class=model.Repository, + key="Repository.description", + attach_popup=False ), + RevisionColumn( "Revision", attach_popup=False ), + RatingColumn( "Rating", attach_popup=False ), ] # Override these default_filter = {} global_actions = [] - operations = [] + operations = [ + grids.GridOperation( "Inspect repository revisions", + allow_multiple=False, + condition=( lambda item: not item.deleted ), + async_compatible=False ) + ] standard_filters = [] num_rows_per_page = 50 preserve_state = False use_paging = True def build_initial_query( self, trans, **kwd ): user_id = trans.security.decode_id( kwd[ 'id' ] ) - return trans.sa_session.query( self.model_class ) \ + return trans.sa_session.query( model.RepositoryReview ) \ .filter( and_( model.RepositoryReview.table.c.deleted == False, \ - model.RepositoryReview.table.c.user_id == user_id ) ) + model.RepositoryReview.table.c.user_id == user_id ) ) \ + .join( ( model.Repository.table, model.RepositoryReview.table.c.repository_id == model.Repository.table.c.id ) ) \ + .filter( model.Repository.table.c.deprecated == False ) class ReviewedRepositoriesIOwnGrid( RepositoriesWithReviewsGrid ): + title = "Reviewed repositories I own" + columns = [ + RepositoriesWithReviewsGrid.NameColumn( "Repository name", + key="name", + link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ), + attach_popup=True ), + RepositoriesWithReviewsGrid.WithReviewsRevisionColumn( "Reviewed revisions" ), + RepositoriesWithReviewsGrid.WithoutReviewsRevisionColumn( "Revisions for review" ), + RepositoriesWithReviewsGrid.ReviewersColumn( "Reviewers", attach_popup=False ), + RepositoryGrid.DeprecatedColumn( "Deprecated" ) + ] + columns.append( grids.MulticolFilterColumn( "Search repository name", + cols_to_filter=[ columns[0] ], + key="free-text-search", + visible=False, + filterable="standard" ) ) + operations = [ + grids.GridOperation( "Inspect repository revisions", + allow_multiple=False, + condition=( lambda item: not item.deleted ), + async_compatible=False ) + ] def build_initial_query( self, trans, **kwd ): return trans.sa_session.query( model.Repository ) \ .join( ( model.RepositoryReview.table, model.RepositoryReview.table.c.repository_id == model.Repository.table.c.id ) ) \ @@ -655,10 +686,26 @@ class RepositoryReviewController( BaseUIController, ItemRatings ): @web.expose @web.require_login( "repository reviews by user" ) def repository_reviews_by_user( self, trans, **kwd ): - # The user may not be the current user. The value of the received id is the encoded user id. params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) + + if 'operation' in kwd: + operation = kwd['operation'].lower() + # The value of the received id is the encoded review id. + review = get_review( trans, kwd[ 'id' ] ) + repository = review.repository + kwd[ 'id' ] = trans.security.encode_id( repository.id ) + if operation == "inspect repository revisions": + return trans.response.send_redirect( web.url_for( controller='repository_review', + action='manage_repository_reviews', + **kwd ) ) + if operation == "view_or_manage_repository": + kwd[ 'changeset_revision' ] = review.changeset_revision + return trans.response.send_redirect( web.url_for( controller='repository_review', + action='view_or_manage_repository', + **kwd ) ) + # The user may not be the current user. The value of the received id is the encoded user id. user = get_user( trans, kwd[ 'id' ] ) self.repository_reviews_by_user_grid.title = "All repository revision reviews for user '%s'" % user.username return self.repository_reviews_by_user_grid( trans, **kwd ) @@ -668,6 +715,13 @@ class RepositoryReviewController( BaseUIController, ItemRatings ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) + # The value of the received id is the encoded repository id. + if 'operation' in kwd: + operation = kwd['operation'].lower() + if operation == "view_or_manage_repository": + return trans.response.send_redirect( web.url_for( controller='repository_review', + action='view_or_manage_repository', + **kwd ) ) return self.reviewed_repositories_i_own_grid( trans, **kwd ) @web.expose @web.require_login( "select previous review" ) diff --git a/lib/galaxy/webapps/community/model/__init__.py b/lib/galaxy/webapps/community/model/__init__.py index 7d8c2dd8905..c2ec7828f1a 100644 --- a/lib/galaxy/webapps/community/model/__init__.py +++ b/lib/galaxy/webapps/community/model/__init__.py @@ -109,7 +109,8 @@ class Repository( object ): MARKED_FOR_REMOVAL = 'r', MARKED_FOR_ADDITION = 'a', NOT_TRACKED = '?' ) - def __init__( self, name=None, description=None, long_description=None, user_id=None, private=False, email_alerts=None, times_downloaded=0 ): + def __init__( self, name=None, description=None, long_description=None, user_id=None, private=False, email_alerts=None, times_downloaded=0, + deprecated=False ): self.name = name or "Unnamed repository" self.description = description self.long_description = long_description @@ -117,6 +118,7 @@ class Repository( object ): self.private = private self.email_alerts = email_alerts self.times_downloaded = times_downloaded + self.deprecated = deprecated @property def repo_path( self ): # Repository locations on disk are defined in the hgweb.config file diff --git a/lib/galaxy/webapps/community/model/mapping.py b/lib/galaxy/webapps/community/model/mapping.py index 184dfd222f2..7ad1203ea8b 100644 --- a/lib/galaxy/webapps/community/model/mapping.py +++ b/lib/galaxy/webapps/community/model/mapping.py @@ -111,7 +111,8 @@ Repository.table = Table( "repository", metadata, Column( "private", Boolean, default=False ), Column( "deleted", Boolean, index=True, default=False ), Column( "email_alerts", JSONType, nullable=True ), - Column( "times_downloaded", Integer ) ) + Column( "times_downloaded", Integer ), + Column( "deprecated", Boolean, default=False ) ) RepositoryMetadata.table = Table( "repository_metadata", metadata, Column( "id", Integer, primary_key=True ), @@ -218,7 +219,10 @@ assign_mapper( context, Tag, Tag.table, properties=dict( children=relation(Tag, backref=backref( 'parent', remote_side=[ Tag.table.c.id ] ) ) ) ) assign_mapper( context, Category, Category.table, - properties=dict( repositories=relation( RepositoryCategoryAssociation ) ) ) + properties=dict( repositories=relation( RepositoryCategoryAssociation, + secondary=Repository.table, + primaryjoin=( Category.table.c.id == RepositoryCategoryAssociation.table.c.category_id ), + secondaryjoin=( ( RepositoryCategoryAssociation.table.c.repository_id == Repository.table.c.id ) & ( Repository.table.c.deprecated == False ) ) ) ) ) assign_mapper( context, Repository, Repository.table, properties = dict( diff --git a/lib/galaxy/webapps/community/model/migrate/versions/0014_add_deprecated_column.py b/lib/galaxy/webapps/community/model/migrate/versions/0014_add_deprecated_column.py new file mode 100644 index 00000000000..e0e8de4ff53 --- /dev/null +++ b/lib/galaxy/webapps/community/model/migrate/versions/0014_add_deprecated_column.py @@ -0,0 +1,53 @@ +""" +Migration script to add the deprecated column to the repository table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +def upgrade(): + print __doc__ + metadata.reflect() + # Create and initialize imported column in job table. + Repository_table = Table( "repository", metadata, autoload=True ) + c = Column( "deprecated", Boolean, default=False ) + try: + # Create + c.create( Repository_table ) + assert c is Repository_table.c.deprecated + # Initialize. + if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + default_false = "0" + elif migrate_engine.name == 'postgres': + default_false = "false" + db_session.execute( "UPDATE repository SET deprecated=%s" % default_false ) + except Exception, e: + print "Adding deprecated column to the repository table failed: %s" % str( e ) + log.debug( "Adding deprecated column to the repository table failed: %s" % str( e ) ) + +def downgrade(): + metadata.reflect() + # Drop email_alerts column from repository table. + Repository_table = Table( "repository", metadata, autoload=True ) + try: + Repository_table.c.deprecated.drop() + except Exception, e: + print "Dropping column deprecated from the repository table failed: %s" % str( e ) + log.debug( "Dropping column deprecated from the repository table failed: %s" % str( e ) ) diff --git a/templates/webapps/community/category/grid.mako b/templates/webapps/community/category/grid.mako index 84833a4ce66..653c1dd4abe 100644 --- a/templates/webapps/community/category/grid.mako +++ b/templates/webapps/community/category/grid.mako @@ -49,8 +49,8 @@ <%def name="grid_body( grid )"> <% - from galaxy.webapps.community.controllers.repository import RepositoryListGrid - repo_grid = RepositoryListGrid() + from galaxy.webapps.community.controllers.repository import RepositoryGrid + repo_grid = RepositoryGrid() %> ${self.make_grid( grid, repo_grid )} diff --git a/templates/webapps/community/category/valid_grid.mako b/templates/webapps/community/category/valid_grid.mako index f3c8633918b..1ec8d367a2d 100644 --- a/templates/webapps/community/category/valid_grid.mako +++ b/templates/webapps/community/category/valid_grid.mako @@ -48,8 +48,8 @@ <%def name="grid_body( grid )"> <% - from galaxy.webapps.community.controllers.repository import ValidRepositoryListGrid - repo_grid = ValidRepositoryListGrid() + from galaxy.webapps.community.controllers.repository import ValidRepositoryGrid + repo_grid = ValidRepositoryGrid() %> ${self.make_grid( grid, repo_grid )} diff --git a/templates/webapps/community/index.mako b/templates/webapps/community/index.mako index 4c2d27531a9..708ba96359e 100644 --- a/templates/webapps/community/index.mako +++ b/templates/webapps/community/index.mako @@ -78,8 +78,13 @@ Reviewed repositories I own %endif + %if has_deprecated_repositories: + + %endif
    My invalid tools diff --git a/templates/webapps/community/repository/manage_repository.mako b/templates/webapps/community/repository/manage_repository.mako index 2d0a2288878..310805f5f3d 100644 --- a/templates/webapps/community/repository/manage_repository.mako +++ b/templates/webapps/community/repository/manage_repository.mako @@ -7,22 +7,25 @@ from galaxy.web.framework.helpers import time_ago is_admin = trans.user_is_admin() is_new = repository.is_new + is_deprecated = repository.deprecated can_contact_owner = trans.user and trans.user != repository.user - can_push = trans.app.security_agent.can_push( trans.user, repository ) + can_push = not is_deprecated and trans.app.security_agent.can_push( trans.user, repository ) can_upload = can_push - can_download = not is_new and ( not is_malicious or can_push ) + can_download = not is_deprecated and not is_new and ( not is_malicious or can_push ) can_browse_contents = not is_new - can_set_metadata = not is_new - can_rate = not is_new and trans.user and repository.user != trans.user + can_set_metadata = not is_new and not is_deprecated + can_rate = not is_new and not is_deprecated and trans.user and repository.user != trans.user can_view_change_log = not is_new if can_push: browse_label = 'Browse or delete repository tip files' else: browse_label = 'Browse repository tip files' can_set_malicious = metadata and can_set_metadata and is_admin and changeset_revision == repository.tip - can_reset_all_metadata = is_admin and len( repo ) > 0 + can_deprecate = not is_new and trans.user and ( is_admin or repository.user == trans.user ) and not is_deprecated + can_undeprecate = trans.user and ( is_admin or repository.user == trans.user ) and is_deprecated + can_reset_all_metadata = not is_deprecated and is_admin and len( repo ) > 0 has_readme = metadata and 'readme' in metadata - can_review_repository = trans.app.security_agent.user_can_review_repositories( trans.user ) + can_review_repository = not is_deprecated and trans.app.security_agent.user_can_review_repositories( trans.user ) reviewing_repository = cntrller and cntrller == 'repository_review' if changeset_revision == repository.tip: tip_str = 'repository tip' @@ -87,6 +90,12 @@ %if can_reset_all_metadata: Reset all repository metadata %endif + %if can_deprecate: + Mark repository as deprecated + %endif + %if can_undeprecate: + Mark repository as not deprecated + %endif %if can_download: Download as a .tar.gz file Download as a .tar.bz2 file @@ -101,6 +110,12 @@ ${render_msg( message, status )} %endif +%if repository.deprecated: +
    + This repository has been marked as deprecated, so some tool shed features may be restricted. +
    +%endif + %if len( changeset_revision_select_field.options ) > 1:
    Repository revision
    @@ -122,7 +137,7 @@

    %endif

    -
    ${repository.name}
    +
    Repository '${repository.name}'
    %if can_download: diff --git a/templates/webapps/community/repository/view_repository.mako b/templates/webapps/community/repository/view_repository.mako index 8909178938c..80872ac3e47 100644 --- a/templates/webapps/community/repository/view_repository.mako +++ b/templates/webapps/community/repository/view_repository.mako @@ -6,11 +6,12 @@ <% from galaxy.web.framework.helpers import time_ago is_new = repository.is_new + is_deprecated = repository.deprecated can_contact_owner = trans.user and trans.user != repository.user - can_push = trans.app.security_agent.can_push( trans.user, repository ) - can_rate = not is_new and trans.user and repository.user != trans.user + can_push = not is_deprecated and trans.app.security_agent.can_push( trans.user, repository ) + can_rate = not is_deprecated and not is_new and trans.user and repository.user != trans.user can_upload = can_push - can_download = not is_new and ( not is_malicious or can_push ) + can_download = not is_deprecated and not is_new and ( not is_malicious or can_push ) can_browse_contents = trans.webapp.name == 'community' and not is_new can_view_change_log = trans.webapp.name == 'community' and not is_new if can_push: @@ -19,7 +20,7 @@ browse_label = 'Browse repository tip files' has_readme = metadata and 'readme' in metadata reviewing_repository = cntrller and cntrller == 'repository_review' - can_review_repository = trans.app.security_agent.user_can_review_repositories( trans.user ) + can_review_repository = not is_deprecated and trans.app.security_agent.user_can_review_repositories( trans.user ) %> <%! @@ -100,6 +101,12 @@ ${render_msg( message, status )} %endif +%if repository.deprecated: +
    + This repository has been marked as deprecated, so some tool shed features may be restricted. +
    +%endif + %if len( changeset_revision_select_field.options ) > 1:
    Repository revision
    @@ -123,7 +130,7 @@

    %endif

    -
    ${repository.name}
    +
    Repository '${repository.name}'
    %if can_download:
    diff --git a/templates/webapps/community/repository/view_tool_metadata.mako b/templates/webapps/community/repository/view_tool_metadata.mako index c78fd4e670f..aa0893ca7b4 100644 --- a/templates/webapps/community/repository/view_tool_metadata.mako +++ b/templates/webapps/community/repository/view_tool_metadata.mako @@ -110,7 +110,7 @@

    %if can_download:

    -
    ${repository.name}
    +
    Repository '${repository.name}'
    From c37e445600b569d67f42d2a2c5b0e41c9fd5d0c0 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Fri, 19 Oct 2012 16:27:48 -0400 Subject: [PATCH 6/6] Closed branch zero-merge.