Add RepositoryMetadata attributes that enable improved query abilities for tool shed repository revisions that contain Galaxy utilities.

2026-09-24 16:30:27 +08:00 · 2013-03-01 11:13:15 -05:00
parent 9096f96b22
commit ee10eed481
6 changed files with 357 additions and 23 deletions
@@ -33,6 +33,7 @@ class CategoryGrid( grids.Grid ):


    class NameColumn( grids.TextColumn ):
+
        def get_value( self, trans, grid, category ):
            return category.name

@@ -43,6 +44,7 @@ class CategoryGrid( grids.Grid ):


    class RepositoriesColumn( grids.TextColumn ):
+
        def get_value( self, trans, grid, category ):
            if category.repositories:
                viewable_repositories = 0
@@ -82,6 +84,7 @@ class ValidCategoryGrid( CategoryGrid ):


    class RepositoriesColumn( grids.TextColumn ):
+
        def get_value( self, trans, grid, category ):
            if category.repositories:
                viewable_repositories = 0
@@ -123,6 +126,7 @@ class RepositoryGrid( grids.Grid ):


    class NameColumn( grids.TextColumn ):
+
        def get_value( self, trans, grid, repository ):
            return escape_html( repository.name )

@@ -468,6 +472,7 @@ class ValidRepositoryGrid( RepositoryGrid ):


    class CategoryColumn( grids.TextColumn ):
+
        def get_value( self, trans, grid, repository ):
            rval = '<ul>'
            if repository.categories:
@@ -481,6 +486,7 @@ class ValidRepositoryGrid( RepositoryGrid ):


    class RepositoryCategoryColumn( grids.GridColumn ):
+
        def filter( self, trans, user, query, column_filter ):
            """Modify query to filter by category."""
            if column_filter == "All":
@@ -501,6 +507,7 @@ class ValidRepositoryGrid( RepositoryGrid ):
            elif len( select_field.options ) == 1:
                return select_field.options[ 0 ][ 0 ]
            return ''
+
    title = "Valid repositories"
    columns = [
        RepositoryGrid.NameColumn( "Name",
@@ -633,19 +640,171 @@ class InstallMatchedRepositoryGrid( MatchedRepositoryGrid ):
                                                     attach_popup=False )


+class RepositoryMetadataGrid( grids.Grid ):
+
+
+    class RepositoryNameColumn( grids.TextColumn ):
+
+        def get_value( self, trans, grid, repository_metadata ):
+            return escape_html( repository_metadata.repository.name )
+
+
+    class RepositoryOwnerColumn( grids.TextColumn ):
+
+        def get_value( self, trans, grid, repository_metadata ):
+            return escape_html( repository_metadata.user.username )
+
+
+    class ChangesetRevisionColumn( grids.TextColumn ):
+
+        def get_value( self, trans, grid, repository_metadata ):
+            return escape_html( repository_metadata.changeset_revision )
+
+
+    class MaliciousColumn( grids.BooleanColumn ):
+        def get_value( self, trans, grid, repository_metadata ):
+            if repository_metadata.malicious:
+                return 'yes'
+            return ''
+
+
+    class DownloadableColumn( grids.BooleanColumn ):
+        def get_value( self, trans, grid, repository_metadata ):
+            if repository_metadata.downloadable:
+                return 'yes'
+            return ''
+
+
+    class ToolsFunctionallyCorrectColumn( grids.BooleanColumn ):
+        def get_value( self, trans, grid, repository_metadata ):
+            if repository_metadata.tools_functionally_correct:
+                return 'yes'
+            return ''
+
+
+    class DoNotTestColumn( grids.BooleanColumn ):
+        def get_value( self, trans, grid, repository_metadata ):
+            if repository_metadata.do_not_test:
+                return 'yes'
+            return ''
+
+
+    class TimeLastTestedColumn( grids.DateTimeColumn ):
+        def get_value( self, trans, grid, repository_metadata ):
+            return repository_metadata.time_last_tested
+
+
+    class HasRepositoryDependenciesColumn( grids.BooleanColumn ):
+        def get_value( self, trans, grid, repository_metadata ):
+            if repository_metadata.has_repository_dependencies:
+                return 'yes'
+            return ''
+
+
+    class IncludesDatatypesColumn( grids.BooleanColumn ):
+        def get_value( self, trans, grid, repository_metadata ):
+            if repository_metadata.includes_datatypes:
+                return 'yes'
+            return ''
+
+
+    class IncludesToolsColumn( grids.BooleanColumn ):
+        def get_value( self, trans, grid, repository_metadata ):
+            if repository_metadata.includes_tools:
+                return 'yes'
+            return ''
+
+
+    class IncludesToolDependenciesColumn( grids.BooleanColumn ):
+        def get_value( self, trans, grid, repository_metadata ):
+            if repository_metadata.includes_tool_dependencies:
+                return 'yes'
+            return ''
+
+
+    class IncludesWorkflowsColumn( grids.BooleanColumn ):
+        def get_value( self, trans, grid, repository_metadata ):
+            if repository_metadata.includes_workflows:
+                return 'yes'
+            return ''
+
+    title = "Repository metadata"
+    model_class = model.RepositoryMetadata
+    template='/webapps/tool_shed/repository/grid.mako'
+    default_sort_key = "Repository.name"
+    columns = [
+        RepositoryNameColumn( "Repository name",
+                              link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ),
+                              attach_popup=False ),
+        RepositoryOwnerColumn( "Owner",
+                               model_class=model.User,
+                               attach_popup=False,
+                               key="User.username" )
+    ]
+    columns.append( grids.MulticolFilterColumn( "Search repository name, description", 
+                                                cols_to_filter=[ columns[0], columns[1] ],
+                                                key="free-text-search",
+                                                visible=False,
+                                                filterable="standard" ) )
+    operations = []
+    standard_filters = []
+    default_filter = dict( malicious="False" )
+    num_rows_per_page = 50
+    preserve_state = False
+    use_paging = True
+
+    def build_initial_query( self, trans, **kwd ):
+        return trans.sa_session.query( model.RepositoryMetadata ) \
+                               .join( model.Repository ) \
+                               .filter( and_( model.Repository.table.c.deleted == False,
+                                              model.Repository.table.c.deprecated == False ) ) \
+                               .join( model.User.table ) \
+                               .order_by( model.Repository.name )
+
+
+class RepositoryDependenciesGrid( RepositoryMetadataGrid ):
+    
+    title = "Repository dependencies available in this Tool Shed"
+    columns = [
+        RepositoryMetadataGrid.RepositoryNameColumn( "Repository name",
+                                                     link=( lambda item: dict( operation="view_or_manage_repository", id=item.id ) ),
+                                                     attach_popup=False ),
+        RepositoryMetadataGrid.RepositoryOwnerColumn( "Owner",
+                                                      model_class=model.User,
+                                                      attach_popup=False,
+                                                      key="User.username" )
+    ]
+    columns.append( grids.MulticolFilterColumn( "Search repository name, description", 
+                                                cols_to_filter=[ columns[0], columns[1] ],
+                                                key="free-text-search",
+                                                visible=False,
+                                                filterable="standard" ) )
+
+    def build_initial_query( self, trans, **kwd ):
+        return trans.sa_session.query( model.RepositoryMetadata ) \
+                               .join( model.Repository ) \
+                               .filter( and_( model.RepositoryMetadata.table.c.has_repository_dependencies == True,
+                                              model.Repository.table.c.deleted == False,
+                                              model.Repository.table.c.deprecated == False ) ) \
+                               .join( model.User.table ) \
+                               .order_by( model.Repository.name )
+
+
 class RepositoryController( BaseUIController, common_util.ItemRatings ):

+    category_grid = CategoryGrid()
+    deprecated_repositories_i_own_grid = DeprecatedRepositoriesIOwnGrid()
+    email_alerts_repository_grid = EmailAlertsRepositoryGrid()
    install_matched_repository_grid = InstallMatchedRepositoryGrid()
    matched_repository_grid = MatchedRepositoryGrid()
-    valid_repository_grid = ValidRepositoryGrid()
-    repository_grid = RepositoryGrid()
-    email_alerts_repository_grid = EmailAlertsRepositoryGrid()
-    category_grid = CategoryGrid()
-    valid_category_grid = ValidCategoryGrid()
    my_writable_repositories_grid = MyWritableRepositoriesGrid()
-    repositories_i_own_grid = RepositoriesIOwnGrid()
-    deprecated_repositories_i_own_grid = DeprecatedRepositoriesIOwnGrid()
    repositories_by_user_grid = RepositoriesByUserGrid()
+    repositories_i_own_grid = RepositoriesIOwnGrid()
+    repository_grid = RepositoryGrid()
+    # The repository_metadata_grid is not currently displayed, but is sub-classed by several grids.
+    repository_metadata_grid = RepositoryMetadataGrid()
+    valid_category_grid = ValidCategoryGrid()
+    valid_repository_grid = ValidRepositoryGrid()

    @web.expose
    def browse_categories( self, trans, **kwd ):
@@ -3036,7 +3195,11 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ):
        repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision )
        if repository_metadata:
            repository_metadata_id = trans.security.encode_id( repository_metadata.id )
-            tool_test_errors = json.from_json_string( repository_metadata.tool_test_errors )
+            # TODO: Fix this when the install and test framework is completed.
+            # if repository_metadata.tool_test_errors:
+            #    tool_test_errors = json.from_json_string( repository_metadata.tool_test_errors )
+            # else:
+            #    tool_test_errors = None
            metadata = repository_metadata.metadata
            if metadata:
                if 'tools' in metadata:
@@ -3070,7 +3233,7 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ):
        else:
            repository_metadata_id = None
            metadata = None
-            tool_test_errors = None
+            #tool_test_errors = None
        is_malicious = suc.changeset_is_malicious( trans, repository_id, repository.tip( trans.app ) )
        changeset_revision_select_field = build_changeset_revision_select_field( trans,
                                                                                 repository,
@@ -3094,7 +3257,7 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ):
                                    tool=tool,
                                    tool_metadata_dict=tool_metadata_dict,
                                    tool_lineage=tool_lineage,
-                                    tool_test_errors=tool_test_errors,
+                                    #tool_test_errors=tool_test_errors,
                                    changeset_revision=changeset_revision,
                                    revision_label=revision_label,
                                    changeset_revision_select_field=changeset_revision_select_field,
@@ -191,11 +191,14 @@ class Repository( object, APIItem ):
        fp.close()

 class RepositoryMetadata( object, APIItem ):
-    api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable' )
+    api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'has_repository_dependencies', 'includes_datatypes',
+                                    'includes_tools', 'includes_tool_dependencies', 'includes_workflows' )
    api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'tools_functionally_correct',
-                                 'do_not_test', 'time_last_tested', 'tool_test_errors' )
+                                 'do_not_test', 'time_last_tested', 'tool_test_errors', 'has_repository_dependencies', 'includes_datatypes', 'includes_tools',
+                                 'includes_tool_dependencies', 'includes_workflows' )
    def __init__( self, id=None, repository_id=None, changeset_revision=None, metadata=None, tool_versions=None, malicious=False, downloadable=False, 
-                  tools_functionally_correct=False, do_not_test=False, time_last_tested=None, tool_test_errors=None ):
+                  tools_functionally_correct=False, do_not_test=False, time_last_tested=None, tool_test_errors=None, has_repository_dependencies=False,
+                  includes_datatypes=False, includes_tools=False, includes_tool_dependencies=False, includes_workflows=False ):
        self.id = id
        self.repository_id = repository_id
        self.changeset_revision = changeset_revision
@@ -207,6 +210,11 @@ class RepositoryMetadata( object, APIItem ):
        self.do_not_test = do_not_test
        self.time_last_tested = time_last_tested
        self.tool_test_errors = tool_test_errors
+        self.has_repository_dependencies = has_repository_dependencies
+        self.includes_datatypes = includes_datatypes
+        self.includes_tools = includes_tools
+        self.includes_tool_dependencies = includes_tool_dependencies
+        self.includes_workflows = includes_workflows
    def as_dict( self, value_mapper=None ):
        return self.get_api_value( view='element', value_mapper=value_mapper )
    def get_api_value( self, view='collection', value_mapper=None ):
@@ -227,8 +235,8 @@ class RepositoryMetadata( object, APIItem ):
        return rval

 class RepositoryReview( object, APIItem ):
-    api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'user_id', 'rating' )
-    api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'user_id', 'rating' )
+    api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'user_id', 'rating', 'deleted' )
+    api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'user_id', 'rating', 'deleted' )
    approved_states = Bunch( NO='no', YES='yes' )
    def __init__( self, repository_id=None, changeset_revision=None, user_id=None, rating=None, deleted=False ):
        self.repository_id = repository_id
@@ -134,7 +134,12 @@ RepositoryMetadata.table = Table( "repository_metadata", metadata,
    Column( "tools_functionally_correct", Boolean, default=False, index=True ),
    Column( "do_not_test", Boolean, default=False, index=True ),
    Column( "time_last_tested", DateTime, default=None, nullable=True ),
-    Column( "tool_test_errors", JSONType, nullable=True ) )
+    Column( "tool_test_errors", JSONType, nullable=True ),
+    Column( "has_repository_dependencies", Boolean, default=False, index=True ),
+    Column( "includes_datatypes", Boolean, default=False, index=True ),
+    Column( "includes_tools", Boolean, default=False, index=True ),
+    Column( "includes_tool_dependencies", Boolean, default=False, index=True ),
+    Column( "includes_workflows", Boolean, default=False, index=True ) )

 RepositoryReview.table = Table( "repository_review", metadata,
    Column( "id", Integer, primary_key=True ),
@@ -0,0 +1,115 @@
+"""
+Migration script to add the includes_datatypes, has_repository_dependencies, includes_tools, includes_tool_dependencies and includes_workflows
+columns to the repository_metadata table.
+"""
+
+from sqlalchemy import *
+from sqlalchemy.orm import *
+from migrate import *
+from migrate.changeset import *
+
+# Need our custom types, but don't import anything else from model
+from galaxy.model.custom_types import *
+
+import sys, logging
+log = logging.getLogger( __name__ )
+log.setLevel(logging.DEBUG)
+handler = logging.StreamHandler( sys.stdout )
+format = "%(name)s %(levelname)s %(asctime)s %(message)s"
+formatter = logging.Formatter( format )
+handler.setFormatter( formatter )
+log.addHandler( handler )
+
+metadata = MetaData( migrate_engine )
+db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
+
+def upgrade():
+    print __doc__
+    metadata.reflect()
+    # Initialize.
+    if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': 
+        default_false = "0"
+    elif migrate_engine.name == 'postgres':
+        default_false = "false"
+    # Create and initialize tools_functionally_correct, do_not_test, time_last_tested, and tool_test_errors columns in repository_metadata table.
+    RepositoryMetadata_table = Table( "repository_metadata", metadata, autoload=True )
+    
+    # Create tools_functionally_correct column
+    c = Column( "includes_datatypes", Boolean, default=False, index=True )
+    try:
+        c.create( RepositoryMetadata_table )
+        assert c is RepositoryMetadata_table.c.includes_datatypes
+        db_session.execute( "UPDATE repository_metadata SET includes_datatypes=%s" % default_false )
+    except Exception, e:
+        print "Adding includes_datatypes column to the repository_metadata table failed: %s" % str( e )
+    
+    # Create includes_datatypes column
+    c = Column( "has_repository_dependencies", Boolean, default=False, index=True )
+    try:
+        c.create( RepositoryMetadata_table )
+        assert c is RepositoryMetadata_table.c.has_repository_dependencies
+        db_session.execute( "UPDATE repository_metadata SET has_repository_dependencies=%s" % default_false )
+    except Exception, e:
+        print "Adding has_repository_dependencies column to the repository_metadata table failed: %s" % str( e )
+    
+    # Create includes_tools column
+    c = Column( "includes_tools", Boolean, default=False, index=True )
+    try:
+        c.create( RepositoryMetadata_table )
+        assert c is RepositoryMetadata_table.c.includes_tools
+        db_session.execute( "UPDATE repository_metadata SET includes_tools=%s" % default_false )
+    except Exception, e:
+        print "Adding includes_tools column to the repository_metadata table failed: %s" % str( e )
+    
+    # Create includes_tool_dependencies column
+    c = Column( "includes_tool_dependencies", Boolean, default=False, index=True )
+    try:
+        c.create( RepositoryMetadata_table )
+        assert c is RepositoryMetadata_table.c.includes_tool_dependencies
+        db_session.execute( "UPDATE repository_metadata SET includes_tool_dependencies=%s" % default_false )
+    except Exception, e:
+        print "Adding includes_tool_dependencies column to the repository_metadata table failed: %s" % str( e )
+    
+    # Create includes_workflows column
+    c = Column( "includes_workflows", Boolean, default=False, index=True )
+    try:
+        c.create( RepositoryMetadata_table )
+        assert c is RepositoryMetadata_table.c.includes_workflows
+        db_session.execute( "UPDATE repository_metadata SET includes_workflows=%s" % default_false )
+    except Exception, e:
+        print "Adding includes_workflows column to the repository_metadata table failed: %s" % str( e )
+
+def downgrade():
+    metadata.reflect()
+    # Drop tool_test_errors, time_last_tested, do_not_test, and tools_functionally_correct columns from repository_metadata table.
+    RepositoryMetadata_table = Table( "repository_metadata", metadata, autoload=True )
+    
+    # Drop the includes_workflows column.
+    try:
+        RepositoryMetadata_table.c.includes_workflows.drop()
+    except Exception, e:
+        print "Dropping column includes_workflows from the repository_metadata table failed: %s" % str( e )
+    
+    # Drop the includes_tool_dependencies column.
+    try:
+        RepositoryMetadata_table.c.includes_tool_dependencies.drop()
+    except Exception, e:
+        print "Dropping column includes_tool_dependencies from the repository_metadata table failed: %s" % str( e )
+
+    # Drop the includes_tools column.
+    try:
+        RepositoryMetadata_table.c.includes_tools.drop()
+    except Exception, e:
+        print "Dropping column includes_tools from the repository_metadata table failed: %s" % str( e )
+
+    # Drop the has_repository_dependencies column.
+    try:
+        RepositoryMetadata_table.c.has_repository_dependencies.drop()
+    except Exception, e:
+        print "Dropping column has_repository_dependencies from the repository_metadata table failed: %s" % str( e )
+
+    # Drop the includes_datatypes column.
+    try:
+        RepositoryMetadata_table.c.includes_datatypes.drop()
+    except Exception, e:
+        print "Dropping column includes_datatypes from the repository_metadata table failed: %s" % str( e )
@@ -900,16 +900,42 @@ def copy_sample_file( app, filename, dest_path=None ):
        shutil.copy( full_source_path, os.path.join( dest_path, copied_file ) )
 def create_or_update_repository_metadata( trans, id, repository, changeset_revision, metadata_dict ):
    """Create or update a repository_metadatqa record in the tool shed."""
-    downloadable = is_downloadable( metadata_dict )
+    has_repository_dependencies = False
+    includes_datatypes = False
+    includes_tools = False
+    includes_tool_dependencies = False
+    includes_workflows = False
+    if metadata_dict:
+        if 'repository_dependencies' in metadata_dict:
+            has_repository_dependencies = True
+        if 'datatypes' in metadata_dict:
+            includes_datatypes = True
+        if 'tools' in metadata_dict:
+            includes_tools = True
+        if 'tool_dependencies' in metadata_dict:
+            includes_tool_dependencies = True
+        if 'workflows' in metadata_dict:
+            includes_workflows = True
+    downloadable = has_repository_dependencies or includes_datatypes or includes_tools or includes_tool_dependencies or includes_workflows
    repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision )
    if repository_metadata:
        repository_metadata.metadata = metadata_dict
        repository_metadata.downloadable = downloadable
+        repository_metadata.has_repository_dependencies = has_repository_dependencies
+        repository_metadata.includes_datatypes = includes_datatypes
+        repository_metadata.includes_tools = includes_tools
+        repository_metadata.includes_tool_dependencies = includes_tool_dependencies
+        repository_metadata.includes_workflows = includes_workflows
    else:
        repository_metadata = trans.model.RepositoryMetadata( repository_id=repository.id,
                                                              changeset_revision=changeset_revision,
                                                              metadata=metadata_dict,
-                                                              downloadable=downloadable )
+                                                              downloadable=downloadable,
+                                                              has_repository_dependencies=has_repository_dependencies,
+                                                              includes_datatypes=includes_datatypes,
+                                                              includes_tools=includes_tools,
+                                                              includes_tool_dependencies=includes_tool_dependencies,
+                                                              includes_workflows=includes_workflows )
    # Always set the default values for the following columns.  When resetting all metadata on a repository, this will reset the values.
    repository_metadata.tools_functionally_correct = False
    repository_metadata.do_not_test = False
@@ -22,6 +22,8 @@
    can_review_repository = has_metadata and not is_deprecated and trans.app.security_agent.user_can_review_repositories( trans.user )
    can_upload = can_push
    can_view_change_log = trans.webapp.name == 'tool_shed' and not is_new
+    # TODO: fix the following when the install and test buildbot is functional.
+    #can_view_tool_test_errors = tool_test_errors is not None

    if can_push:
        browse_label = 'Browse or delete repository tip files'
@@ -265,11 +267,26 @@
                    ${tool.force_history_refresh | h}
                    <div style="clear: both"></div>
                </div>
-                <div class="form-row">
-                    <label>Parallelism:</label>
-                    ${tool.parallelism | h}
-                    <div style="clear: both"></div>
-                </div>
+                <% parallelism_info = tool.parallelism %>
+                %if parallelism_info:
+                    <div class="form-row">
+                        <table width="100%">
+                            <tr bgcolor="#D8D8D8" width="100%"><td><b>Parallelism</td></tr>
+                        </table>
+                    </div>
+                    <div class="form-row">
+                        <label>Method:</label>
+                        ${parallelism_info.method | h}
+                        <div style="clear: both"></div>
+                    </div>
+                    %for key, val in parallelism_info.attributes.items():
+                        <div class="form-row">
+                            <label>${key}:</label>
+                            ${val | h}
+                            <div style="clear: both"></div>
+                        </div>
+                    %endfor
+                %endif
            %endif
            <div class="form-row">
                <table width="100%">