Add basic UI for managng tool shed repositories installed into a local Galaxy instance from a tool shed. Add the ability to view the list of tools and workflows included in installed tool shed repositories. Ad the ability to import a workflow from an installed tool shed repository into the local Galaxy instance.

This commit is contained in:
Greg Von Kuster
2011-10-28 14:05:14 -04:00
parent ef6da827cb
commit ac7d488cb3
7 changed files with 351 additions and 42 deletions
-7
View File
@@ -2443,13 +2443,6 @@ def get_quota( trans, id ):
id = trans.security.decode_id( id )
quota = trans.sa_session.query( trans.model.Quota ).get( id )
return quota
def get_tool_shed_repository( trans, tool_shed, name, owner, changeset_revision ):
return trans.sa_session.query( trans.model.ToolShedRepository ) \
.filter( and_( trans.model.ToolShedRepository.table.c.tool_shed == tool_shed,
trans.model.ToolShedRepository.table.c.name == name,
trans.model.ToolShedRepository.table.c.owner == owner,
trans.model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) ) \
.first()
def handle_sample_tool_data_table_conf_file( trans, filename ):
"""
Parse the incoming filename and add new entries to the in-memory
+139 -27
View File
@@ -410,7 +410,8 @@ class RepositoryListGrid( grids.Grid ):
columns = [
NameColumn( "Name",
key="name",
attach_popup=True ),
link=( lambda item: dict( operation="manage_repository", id=item.id, webapp="galaxy" ) ),
attach_popup=False ),
DescriptionColumn( "Description" ),
OwnerColumn( "Owner" ),
RevisionColumn( "Revision" ),
@@ -426,17 +427,14 @@ class RepositoryListGrid( grids.Grid ):
key="free-text-search",
visible=False,
filterable="standard" ) )
operations = [ grids.GridOperation( "Get updates",
allow_multiple=False,
condition=( lambda item: not item.deleted ),
async_compatible=False ) ]
standard_filters = []
default_filter = dict( deleted="False" )
num_rows_per_page = 50
preserve_state = False
use_paging = True
def build_initial_query( self, trans, **kwd ):
return trans.sa_session.query( self.model_class )
return trans.sa_session.query( self.model_class ) \
.filter( self.model_class.table.c.deleted == False )
class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuota, QuotaParamParser ):
@@ -678,11 +676,88 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuota, QuotaParamP
return quota, params
@web.expose
@web.require_admin
def browse_repositories( self, trans, **kwd ):
def browse_tool_shed_repository( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
repository = get_repository( trans, kwd[ 'id' ] )
relative_install_dir = self.__get_relative_install_dir( trans, repository )
repo_files_dir = os.path.abspath( os.path.join( relative_install_dir, repository.name ) )
tool_dicts = []
workflow_dicts = []
for root, dirs, files in os.walk( repo_files_dir ):
if not root.find( '.hg' ) >= 0 and not root.find( 'hgrc' ) >= 0:
if '.hg' in dirs:
# Don't visit .hg directories.
dirs.remove( '.hg' )
if 'hgrc' in files:
# Don't include hgrc files.
files.remove( 'hgrc' )
for name in files:
# Find all tool configs.
if name.endswith( '.xml' ):
try:
full_path = os.path.abspath( os.path.join( root, name ) )
tool = trans.app.toolbox.load_tool( full_path )
if tool is not None:
tool_config = os.path.join( root, name )
# Handle tool.requirements.
tool_requirements = []
for tr in tool.requirements:
name=tr.name
type=tr.type
if type == 'fabfile':
version = None
fabfile = tr.fabfile
method = tr.method
else:
version = tr.version
fabfile = None
method = None
requirement_dict = dict( name=name,
type=type,
version=version,
fabfile=fabfile,
method=method )
tool_requirements.append( requirement_dict )
tool_dict = dict( id=tool.id,
old_id=tool.old_id,
name=tool.name,
version=tool.version,
description=tool.description,
requirements=tool_requirements,
tool_config=tool_config )
tool_dicts.append( tool_dict )
except Exception, e:
# The file is not a Galaxy tool config.
pass
# Find all exported workflows
elif name.endswith( '.ga' ):
try:
full_path = os.path.abspath( os.path.join( root, name ) )
# Convert workflow data from json
fp = open( full_path, 'rb' )
workflow_text = fp.read()
fp.close()
workflow_dict = from_json_string( workflow_text )
if workflow_dict[ 'a_galaxy_workflow' ] == 'true':
workflow_dicts.append( dict( full_path=full_path, workflow_dict=workflow_dict ) )
except Exception, e:
# The file is not a Galaxy workflow.
pass
return trans.fill_template( '/admin/tool_shed_repository/browse_repository.mako',
repository=repository,
tool_dicts=tool_dicts,
workflow_dicts=workflow_dicts,
message=message,
status=status )
@web.expose
@web.require_admin
def browse_tool_shed_repositories( self, trans, **kwd ):
if 'operation' in kwd:
operation = kwd.pop('operation').lower()
if operation == "get updates":
return self.check_for_updates( trans, **kwd )
operation = kwd.pop( 'operation' ).lower()
if operation == "manage_repository":
return self.manage_tool_shed_repository( trans, **kwd )
# Render the list view
return self.repository_list_grid( trans, **kwd )
@web.expose
@@ -850,10 +925,35 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuota, QuotaParamP
status=status )
@web.expose
@web.require_admin
def check_for_updates( self, trans, **kwd ):
def manage_tool_shed_repository( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
repository_id = params.get( 'id', None )
repository = get_repository( trans, repository_id )
description = util.restore_text( params.get( 'description', repository.description ) )
if params.get( 'edit_repository_button', False ):
if description != repository.description:
repository.description = description
trans.sa_session.add( repository )
trans.sa_session.flush()
message = "The repository information has been updated."
relative_install_dir = self.__get_relative_install_dir( trans, repository )
if relative_install_dir:
repo_files_dir = os.path.abspath( os.path.join( relative_install_dir, repository.name ) )
else:
repo_files_dir = 'unknown'
return trans.fill_template( '/admin/tool_shed_repository/manage_repository.mako',
repository=repository,
description=description,
repo_files_dir=repo_files_dir,
message=message,
status=status )
@web.expose
@web.require_admin
def check_for_updates( self, trans, **kwd ):
params = util.Params( kwd )
repository = get_repository( trans, kwd[ 'id' ] )
galaxy_url = trans.request.host
# Send a request to the relevant tool shed to see if there are any updates.
# TODO: support https in the following url.
@@ -872,26 +972,16 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuota, QuotaParamP
owner = params.get( 'owner', None )
changeset_revision = params.get( 'changeset_revision', None )
latest_changeset_revision = params.get( 'latest_changeset_revision', None )
repository = get_repository_by_shed_name_owner_changeset_revision( trans, tool_shed_url, name, owner, changeset_revision )
if changeset_revision and latest_changeset_revision:
if changeset_revision == latest_changeset_revision:
message = "The cloned tool shed repository named '%s' is current (there are no updates available)." % name
else:
repository = get_repository_by_name_owner_changeset_revision( trans, name, owner, changeset_revision )
current_working_dir = os.getcwd()
# Get the directory where the repository is cloned.
cleaned_tool_shed_url = self.__clean_tool_shed_url( tool_shed_url )
partial_cloned_dir = '%s/repos/%s/%s/%s' % ( cleaned_tool_shed_url, owner, name, changeset_revision )
# Get the relative tool installation paths from each of the shed tool configs.
shed_tool_confs = trans.app.toolbox.shed_tool_confs
relative_cloned_dir = None
# The shed_tool_confs dictionary contains shed_conf_filename : tool_path pairs.
for shed_conf_filename, tool_path in shed_tool_confs.items():
relative_cloned_dir = os.path.join( tool_path, partial_cloned_dir )
if os.path.isdir( relative_cloned_dir ):
break
if relative_cloned_dir:
relative_install_dir = self.__get_relative_install_dir( trans, repository )
if relative_install_dir:
# Update the cloned repository to changeset_revision.
repo_files_dir = os.path.join( relative_cloned_dir, name )
repo_files_dir = os.path.join( relative_install_dir, name )
log.debug( "Updating cloned repository named '%s' from revision '%s' to revision '%s'..." % \
( name, changeset_revision, latest_changeset_revision ) )
cmd = 'hg pull'
@@ -935,9 +1025,23 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuota, QuotaParamP
message = "The latest changeset revision could not be retrieved for the repository named '%s'." % name
status = 'error'
return trans.response.send_redirect( web.url_for( controller='admin',
action='browse_repositories',
action='manage_tool_shed_repository',
id=trans.security.encode_id( repository.id ),
message=message,
status=status ) )
def __get_relative_install_dir( self, trans, repository ):
# Get the directory where the repository is install.
tool_shed = self.__clean_tool_shed_url( repository.tool_shed )
partial_install_dir = '%s/repos/%s/%s/%s' % ( tool_shed, repository.owner, repository.name, repository.changeset_revision )
# Get the relative tool installation paths from each of the shed tool configs.
shed_tool_confs = trans.app.toolbox.shed_tool_confs
relative_install_dir = None
# The shed_tool_confs dictionary contains { shed_conf_filename : tool_path } pairs.
for shed_conf_filename, tool_path in shed_tool_confs.items():
relative_install_dir = os.path.join( tool_path, partial_install_dir )
if os.path.isdir( relative_install_dir ):
break
return relative_install_dir
def __handle_missing_data_table_entry( self, trans, tool_path, sample_files, repository_tools_tups ):
# Inspect each tool to see if any have input parameters that are dynamically
# generated select lists that require entries in the tool_data_table_conf.xml file.
@@ -1057,7 +1161,7 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuota, QuotaParamP
tool_shed = tmp_url.split( 'repos' )[ 0 ].rstrip( '/' )
owner = self.__get_repository_owner( tmp_url )
flush_needed = False
tool_shed_repository = get_tool_shed_repository( trans, tool_shed, name, owner, changeset_revision )
tool_shed_repository = get_repository_by_shed_name_owner_changeset_revision( trans, tool_shed, name, owner, changeset_revision )
if tool_shed_repository:
if tool_shed_repository.deleted:
tool_shed_repository.deleted = False
@@ -1200,3 +1304,11 @@ def get_repository_by_name_owner_changeset_revision( trans, name, owner, changes
trans.model.ToolShedRepository.table.c.owner == owner,
trans.model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) ) \
.first()
def get_repository_by_shed_name_owner_changeset_revision( trans, tool_shed, name, owner, changeset_revision ):
return trans.sa_session.query( trans.model.ToolShedRepository ) \
.filter( and_( trans.model.ToolShedRepository.table.c.tool_shed == tool_shed,
trans.model.ToolShedRepository.table.c.name == name,
trans.model.ToolShedRepository.table.c.owner == owner,
trans.model.ToolShedRepository.table.c.changeset_revision == changeset_revision ) ) \
.first()
+24 -5
View File
@@ -1095,8 +1095,8 @@ class WorkflowController( BaseUIController, Sharable, UsesStoredWorkflow, UsesAn
@web.expose
def import_workflow( self, trans, **kwd ):
"""
Import a workflow by reading an url, uploading a file, or receiving the textual
representation of a workflow.
Import a workflow by reading an url, uploading a file, opening and reading the contents
of a local file, or receiving the textual representation of a workflow via http.
"""
url = kwd.get( 'url', '' )
workflow_text = kwd.get( 'workflow_text', '' )
@@ -1104,6 +1104,8 @@ class WorkflowController( BaseUIController, Sharable, UsesStoredWorkflow, UsesAn
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
import_button = kwd.get( 'import_button', False )
# The following parameters will have values only if the workflow
# id being imported from a Galaxy tool shed repository.
tool_shed_url = kwd.get( 'tool_shed_url', '' )
repository_metadata_id = kwd.get( 'repository_metadata_id', '' )
# The workflow_name parameter is in the request only if the import originated
@@ -1111,6 +1113,15 @@ class WorkflowController( BaseUIController, Sharable, UsesStoredWorkflow, UsesAn
workflow_name = kwd.get( 'workflow_name', '' )
if workflow_name:
workflow_name = tool_shed_decode( workflow_name )
# The following parameters will have a value only if the import originated
# from a tool shed repository installed locally.
local_file = kwd.get( 'local_file', '' )
repository_id = kwd.get( 'repository_id', '' )
if local_file and not import_button:
workflow_file = open( local_file, 'rb' )
workflow_text = workflow_file.read()
workflow_file.close()
import_button = True
if tool_shed_url and not import_button:
# Use urllib (send another request to the tool shed) to retrieve the workflow.
workflow_url = 'http://%s/workflow/import_workflow?repository_metadata_id=%s&workflow_name=%s&webapp=%s&open_for_url=true' % \
@@ -1118,7 +1129,6 @@ class WorkflowController( BaseUIController, Sharable, UsesStoredWorkflow, UsesAn
response = urllib2.urlopen( workflow_url )
workflow_text = response.read()
response.close()
workflow_text = workflow_text
import_button = True
if import_button:
workflow_data = None
@@ -1131,6 +1141,7 @@ class WorkflowController( BaseUIController, Sharable, UsesStoredWorkflow, UsesAn
message = "Failed to open URL: <b>%s</b><br>Exception: %s" % ( url, str( e ) )
status = 'error'
elif workflow_text:
# This case occurs when the workflow_text was sent via http from the tool shed.
workflow_data = workflow_text
else:
# Load workflow from browsed file.
@@ -1170,7 +1181,7 @@ class WorkflowController( BaseUIController, Sharable, UsesStoredWorkflow, UsesAn
message += "Imported, but this workflow contains cycles. "
status = "error"
else:
message += "Workflow '%s' imported successfully. " % workflow.name
message += "Workflow <b>%s</b> imported successfully. " % workflow.name
if missing_tool_tups:
if trans.user_is_admin():
# A required tool is not available in the local Galaxy instance.
@@ -1201,11 +1212,19 @@ class WorkflowController( BaseUIController, Sharable, UsesStoredWorkflow, UsesAn
pass
if tool_shed_url:
# We've received the textual representation of a workflow from a Galaxy tool shed.
message = "This workflow has been successfully imported into your local Galaxy instance."
message = "Workflow <b>%s</b> imported successfully." % workflow.name
# TODO: support https in the following url.
url = 'http://%s/workflow/view_workflow?repository_metadata_id=%s&workflow_name=%s&webapp=%s&message=%s' % \
( tool_shed_url, repository_metadata_id, tool_shed_encode( workflow_name ), webapp, message )
return trans.response.send_redirect( url )
elif local_file:
# The workflow was read from a file included with an installed tool shed repository.
message = "Workflow <b>%s</b> imported successfully." % workflow.name
return trans.response.send_redirect( web.url_for( controller='admin',
action='browse_tool_shed_repository',
id=repository_id,
message=message,
status=status ) )
return self.list( trans )
return trans.fill_template( "workflow/import.mako",
url=url,
@@ -361,8 +361,9 @@ def set_repository_metadata( trans, id, changeset_revision, **kwd ):
workflow_text = fp.read()
fp.close()
exported_workflow_dict = from_json_string( workflow_text )
# Update the list of metadata dictionaries for workflows in metadata_dict.
metadata_dict = generate_workflow_metadata( trans, id, changeset_revision, exported_workflow_dict, metadata_dict )
if exported_workflow_dict[ 'a_galaxy_workflow' ] == 'true':
# Update the list of metadata dictionaries for workflows in metadata_dict.
metadata_dict = generate_workflow_metadata( trans, id, changeset_revision, exported_workflow_dict, metadata_dict )
except Exception, e:
invalid_files.append( ( name, str( e ) ) )
else:
@@ -0,0 +1,127 @@
<%inherit file="/base.mako"/>
<%namespace file="/message.mako" import="render_msg" />
<% from galaxy.web.base.controller import tool_shed_encode, tool_shed_decode %>
<br/><br/>
<ul class="manage-table-actions">
<li><a class="action-button" id="repository-${repository.id}-popup" class="menubutton">Repository Actions</a></li>
<div popupmenu="repository-${repository.id}-popup">
<a class="action-button" href="${h.url_for( controller='admin', action='manage_tool_shed_repository', id=trans.security.encode_id( repository.id ) )}">Manage repository</a>
<a class="action-button" href="${h.url_for( controller='admin', action='check_for_updates', id=trans.security.encode_id( repository.id ) )}">Get updates</a>
</div>
</ul>
%if message:
${render_msg( message, status )}
%endif
<div class="toolForm">
<div class="toolFormTitle">Installed tool shed repository '${repository.name}'</div>
<div class="toolFormBody">
%if tool_dicts:
<div class="form-row">
<table width="100%">
<tr bgcolor="#D8D8D8" width="100%">
<td><b>Tools</b></td>
</tr>
</table>
</div>
<div class="form-row">
<table class="grid">
<tr>
<td><b>name</b></td>
<td><b>description</b></td>
<td><b>version</b></td>
<td><b>requirements</b></td>
</tr>
%for tool_dict in tool_dicts:
<tr>
<td>${tool_dict[ 'name' ]}</div>
</td>
<td>${tool_dict[ 'description' ]}</td>
<td>${tool_dict[ 'version' ]}</td>
<td>
<%
if 'requirements' in tool_dict:
requirements = tool_dict[ 'requirements' ]
else:
requirements = None
%>
%if requirements:
<%
requirements_str = ''
for requirement_dict in tool_dict[ 'requirements' ]:
requirements_str += '%s (%s), ' % ( requirement_dict[ 'name' ], requirement_dict[ 'type' ] )
requirements_str = requirements_str.rstrip( ', ' )
%>
${requirements_str}
%else:
none
%endif
</td>
</tr>
%endfor
</table>
</div>
<div style="clear: both"></div>
%endif
%if workflow_dicts:
<div class="form-row">
<table width="100%">
<tr bgcolor="#D8D8D8" width="100%">
<td><b>Workflows</b></td>
</tr>
</table>
</div>
<div style="clear: both"></div>
<div class="form-row">
<table class="grid">
<tr>
<td><b>name</b></td>
<td><b>steps</b></td>
<td><b>format-version</b></td>
<td><b>annotation</b></td>
</tr>
<% index = 0 %>
%for wf_dict in workflow_dicts:
<%
full_path = wf_dict[ 'full_path' ]
workflow_dict = wf_dict[ 'workflow_dict' ]
workflow_name = workflow_dict[ 'name' ]
if 'steps' in workflow_dict:
## Initially steps were not stored in the metadata record.
steps = workflow_dict[ 'steps' ]
else:
steps = []
format_version = workflow_dict[ 'format-version' ]
annotation = workflow_dict[ 'annotation' ]
%>
<tr>
<td>
<div class="menubutton" style="float: left;" id="workflow-${index}-popup">
${workflow_name}
<div popupmenu="workflow-${index}-popup">
<a class="action-button" href="${h.url_for( controller='workflow', action='import_workflow', local_file=full_path, repository_id=trans.security.encode_id( repository.id ) )}">Import to Galaxy</a>
</div>
</div>
</td>
<td>
%if 'steps' in workflow_dict:
${len( steps )}
%else:
unknown
%endif
</td>
<td>${format_version}</td>
<td>${annotation}</td>
</tr>
<% index += 1 %>
%endfor
</table>
</div>
<div style="clear: both"></div>
%endif
</div>
</div>
@@ -0,0 +1,57 @@
<%inherit file="/base.mako"/>
<%namespace file="/message.mako" import="render_msg" />
<br/><br/>
<ul class="manage-table-actions">
<li><a class="action-button" id="repository-${repository.id}-popup" class="menubutton">Repository Actions</a></li>
<div popupmenu="repository-${repository.id}-popup">
<a class="action-button" href="${h.url_for( controller='admin', action='browse_tool_shed_repository', id=trans.security.encode_id( repository.id ) )}">Browse repository</a>
<a class="action-button" href="${h.url_for( controller='admin', action='check_for_updates', id=trans.security.encode_id( repository.id ) )}">Get updates</a>
</div>
</ul>
%if message:
${render_msg( message, status )}
%endif
<div class="toolForm">
<div class="toolFormTitle">${repository.name}</div>
<div class="toolFormBody">
<form name="edit_repository" id="edit_repository" action="${h.url_for( controller='admin', action='manage_tool_shed_repository', id=trans.security.encode_id( repository.id ) )}" method="post" >
<div class="form-row">
<label>Tool shed:</label>
${repository.tool_shed}
<div style="clear: both"></div>
</div>
<div class="form-row">
<label>Name:</label>
${repository.name}
<div style="clear: both"></div>
</div>
<div class="form-row">
<label>Description:</label>
<input name="description" type="textfield" value="${description}" size="80"/>
<div style="clear: both"></div>
</div>
<div class="form-row">
<label>Revision:</label>
${repository.changeset_revision}
</div>
<div class="form-row">
<label>Owner:</label>
${repository.owner}
</div>
<div class="form-row">
<label>Location:</label>
${repo_files_dir}
</div>
<div class="form-row">
<label>Deleted:</label>
${repository.deleted}
</div>
<div class="form-row">
<input type="submit" name="edit_repository_button" value="Save"/>
</div>
</form>
</div>
</div>
+1 -1
View File
@@ -64,7 +64,7 @@
<div class="toolTitle"><a href="${h.url_for( controller='admin', action='memdump' )}" target="galaxy_main">Profile memory usage</a></div>
<div class="toolTitle"><a href="${h.url_for( controller='admin', action='jobs' )}" target="galaxy_main">Manage jobs</a></div>
%if cloned_repositories:
<div class="toolTitle"><a href="${h.url_for( controller='admin', action='browse_repositories' )}" target="galaxy_main">Manage installed tool shed repositories</a></div>
<div class="toolTitle"><a href="${h.url_for( controller='admin', action='browse_tool_shed_repositories' )}" target="galaxy_main">Manage installed tool shed repositories</a></div>
%endif
</div>
</div>