opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add baseline support for using Fabric scripts for automatically installing tool dependencies on a local Galaxy instance when repositories are installed from a Galaxy tool shed, and persist the url to the local Galaxy instance in a cookie named toolshedgalaxyurl when installing a tool shed repository to a local Galaxy instance.

Browse Source
This commit is contained in:
Greg Von Kuster
2011-09-20 10:38:30 -04:00
parent 2380d990e2
commit ad85c3d9fb
12 changed files with 111 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/config.py
+4 -1
View File
@@ -119,9 +119,12 @@ class Configuration( object ):
self.ftp_upload_site = kwargs.get( 'ftp_upload_site', None )
self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False )
self.disable_library_comptypes = kwargs.get( 'disable_library_comptypes', '' ).lower().split( ',' )
# Location for dependencies
# Location for tool dependencies.
if 'tool_dependency_dir' in kwargs:
self.tool_dependency_dir = resolve_path( kwargs.get( "tool_dependency_dir" ), self.root )
# Setting the following flag to true will ultimately cause tool dependencies
# to be located in the shell environment and used by the job that is executing
# the tool.
self.use_tool_dependencies = True
else:
self.tool_dependency_dir = None
lib/galaxy/datatypes/assembly.py
-1
View File
@@ -143,7 +143,6 @@ class Velvet( Html ):
def __init__( self, **kwd ):
Html.__init__( self, **kwd )
log.debug( "Velvet log info %s" % 'JJ __init__')
self.add_composite_file( 'Sequences', mimetype = 'text/html', description = 'Sequences', substitute_name_with_metadata = None, is_binary = False )
self.add_composite_file( 'Roadmaps', mimetype = 'text/html', description = 'Roadmaps', substitute_name_with_metadata = None, is_binary = False )
self.add_composite_file( 'Log', mimetype = 'text/html', description = 'Log', optional = 'True', substitute_name_with_metadata = None, is_binary = False )
lib/galaxy/tools/__init__.py
+27 -12
View File
@@ -227,12 +227,11 @@ class ToolBox( object ):
id = self.app.security.decode_id( workflow_id )
stored = self.app.model.context.query( self.app.model.StoredWorkflow ).get( id )
return stored.latest_workflow
def init_dependency_manager( self ):
self.dependency_manager = None
if self.app.config.use_tool_dependencies:
self.dependency_manager = DependencyManager( [ self.app.config.tool_dependency_dir ] )
else:
self.dependency_manager = None
@property
def sa_session( self ):
"""
@@ -341,12 +340,17 @@ class ToolRequirement( object ):
"""
Represents an external requirement that must be available for the tool to
run (for example, a program, package, or library). Requirements can
optionally assert a specific version
optionally assert a specific version, or reference a command to execute a
fabric script. If fabric is used, the type is 'fabfile' and the version
attribute is not used since the fabric script includes all necessary
information for automatic dependency installation.
"""
def __init__( self ):
self.name = None
self.type = None
self.version = None
def __init__( self, name=None, type=None, version=None, fabfile=None, method=None ):
self.name = name
self.type = type
self.version = version
self.fabfile = fabfile
self.method = method
class Tool:
"""
@@ -882,10 +886,21 @@ class Tool:
self.requirements
"""
for requirement_elem in requirements_elem.findall( 'requirement' ):
requirement = ToolRequirement()
requirement.name = util.xml_text( requirement_elem )
requirement.type = requirement_elem.get( "type", "package" )
requirement.version = requirement_elem.get( "version" )
name = util.xml_text( requirement_elem )
type = requirement_elem.get( "type", "package" )
if type == 'fabfile':
# The fabric script will include all necessary information for
# automatically installing the tool dependencies.
fabfile = requirement_elem.get( "fabfile" )
method = requirement_elem.get( "method" )
version = None
else:
# For backward compatibility, requirements tag sets should not require the
# use of a fabric script.
version = requirement_elem.get( "version" )
fabfile = None
method = None
requirement = ToolRequirement( name=name, type=type, version=version, fabfile=fabfile, method=method )
self.requirements.append( requirement )
def check_workflow_compatible( self ):
lib/galaxy/tools/deps/__init__.py
+3 -9
View File
@@ -18,11 +18,10 @@ class DependencyManager( object ):
and should each contain a file 'env.sh' which can be sourced to make the
dependency available in the current shell environment.
"""
def __init__( self, base_paths=[] ):
"""
Create a new dependency manager looking for packages under the
paths listed in `base_paths`.
Create a new dependency manager looking for packages under the paths listed
in `base_paths`. The default base path is app.config.tool_dependency_dir.
"""
self.base_paths = []
for base_path in base_paths:
@@ -31,19 +30,17 @@ class DependencyManager( object ):
if not os.path.isdir( base_path ):
log.warn( "Path '%s' is not directory, ignoring", base_path )
self.base_paths.append( os.path.abspath( base_path ) )
def find_dep( self, name, version=None ):
"""
Attempt to find a dependency named `name` at version `version`. If
version is None, return the "default" version as determined using a
symbolic link (if found). Returns a triple of:
env_script, base_path, real_version
env_script, base_path, real_version
"""
if version is None:
return self._find_dep_default( name )
else:
return self._find_dep_versioned( name, version )
def _find_dep_versioned( self, name, version ):
for base_path in self.base_paths:
path = os.path.join( base_path, name, version )
@@ -52,7 +49,6 @@ class DependencyManager( object ):
return script, path, version
else:
return None, None, None
def _find_dep_default( self, name ):
version = None
for base_path in self.base_paths:
@@ -65,5 +61,3 @@ class DependencyManager( object ):
return script, real_path, real_version
else:
return None, None, None
lib/galaxy/web/base/controller.py
+38 -4
View File
@@ -2749,10 +2749,11 @@ class Admin( object ):
@web.expose
@web.require_admin
def browse_tool_shed( self, trans, **kwd ):
tool_shed_name = kwd[ 'tool_shed_name' ]
tool_shed_url = kwd[ 'tool_shed_url' ]
galaxy_url = trans.request.host
url = '%s/repository/browse_downloadable_repositories?tool_shed_name=%s&galaxy_url=%s&webapp=community' % ( tool_shed_url, tool_shed_name, galaxy_url )
# Set the galayurl cookie so we can get back here from the remote tool shed.
trans.set_cookie( galaxy_url, name='toolshedgalaxyurl' )
url = '%s/repository/browse_downloadable_repositories?webapp=community' % ( tool_shed_url )
return trans.response.send_redirect( url )
@web.expose
@web.require_admin
@@ -2784,8 +2785,8 @@ class Admin( object ):
status = 'error'
else:
os.makedirs( clone_dir )
log.debug( 'Cloning %s...' % repository_clone_url )
cmd = 'hg clone %s' % repository_clone_url
log.debug( 'Cloning: %s' % repository_clone_url )
tmp_name = tempfile.NamedTemporaryFile().name
tmp_stderr = open( tmp_name, 'wb' )
os.chdir( clone_dir )
@@ -2795,7 +2796,7 @@ class Admin( object ):
tmp_stderr.close()
if returncode == 0:
repo_files_dir = os.path.join( clone_dir, repository_name )
log.debug( 'Updating cloned repository to revision: %s' % changeset_revision )
log.debug( 'Updating cloned repository to revision "%s"...' % changeset_revision )
cmd = 'hg update -r %s' % changeset_revision
tmp_name = tempfile.NamedTemporaryFile().name
tmp_stderr = open( tmp_name, 'wb' )
@@ -2831,6 +2832,39 @@ class Admin( object ):
# We have an inavlid .xml file, so not a tool config.
log.debug( "Ignoring invalid tool config (%s). Error: %s" % ( str( relative_path ), str( e ) ) )
if repository_tools_tups:
# Look at each tool to see if it includes a "requirement" that refers to
# a fabric script. For those that do, execute the fabric script to install
# tool dependencies.
for repository_tools_tup in repository_tools_tups:
tup_path, repository_tool = repository_tools_tup
for requirement in repository_tool.requirements:
if requirement.type == 'fabfile':
log.debug( 'Executing fabric script to install dependencies for tool "%s"...' % repository_tool.name )
fabfile = requirement.fabfile
method = requirement.method
# Find the relative path to the fabfile.
relative_fabfile_path = None
for root, dirs, files in os.walk( repo_files_dir ):
for name in files:
if name == fabfile:
relative_fabfile_path = os.path.join( root, name )
break
if relative_fabfile_path:
# cmd will look something like: fab -f fabfile.py install_bowtie
cmd = 'fab -f %s %s' % ( relative_fabfile_path, method )
tmp_name = tempfile.NamedTemporaryFile().name
tmp_stderr = open( tmp_name, 'wb' )
os.chdir( repo_files_dir )
proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() )
returncode = proc.wait()
os.chdir( current_working_dir )
tmp_stderr.close()
if returncode != 0:
# TODO: do something more here than logging the problem.
tmp_stderr = open( tmp_name, 'rb' )
error = tmp_stderr.read()
tmp_stderr.close()
log.debug( 'Problem installing dependencies for tool "%s"\n%s' % ( repository_tool.name, error ) )
# Generate an in-memory tool conf section that includes the new tools.
new_tool_section = self.__generate_tool_panel_section( repository_name,
repository_clone_url,
lib/galaxy/webapps/community/controllers/common.py
+24 -14
View File
@@ -171,9 +171,21 @@ def generate_tool_metadata( trans, id, changeset_revision, tool_config, tool, me
repository = get_repository( trans, id )
tool_requirements = []
for tr in tool.requirements:
requirement_dict = dict( name=tr.name,
type=tr.type,
version=tr.version )
name=tr.name
type=tr.type
if type == 'fabfile':
version = None
fabfile = tr.fabfile
method = tr.method
else:
version = tr.version
fabfile = None
method = None
requirement_dict = dict( name=name,
type=type,
version=version,
fabfile=fabfile,
method=method )
tool_requirements.append( requirement_dict )
tool_tests = []
if tool.tests:
@@ -335,13 +347,17 @@ def set_repository_metadata( trans, id, changeset_revision, **kwd ):
trans.sa_session.flush()
else:
message = "Changeset revision '%s' includes no tools or exported workflows for which metadata can be set." % str( changeset_revision )
status = "done"
status = "error"
else:
# change_set is None
message = "Repository does not include changeset revision '%s'." % str( changeset_revision )
status = 'error'
if invalid_files:
message = "Metadata cannot be defined for change set revision '%s'. Correct the following problems and reset metadata.<br/>" % str( changeset_revision )
if metadata_dict:
message = "Metadata was defined for some items in change set revision '%s'. " % str( changeset_revision )
message += "If the following files are Galaxy tool configs or exported Galaxy workflows, correct the problems and reset metadata.<br/>"
else:
message = "Metadata cannot be defined for change set revision '%s'. Correct the following problems and reset metadata.<br/>" % str( changeset_revision )
for itc_tup in invalid_files:
tool_file = itc_tup[0]
exception_msg = itc_tup[1]
@@ -527,7 +543,7 @@ def load_tool( trans, config_file ):
ToolClass = Tool
return ToolClass( config_file, root, trans.app )
return None
def build_changeset_revision_select_field( trans, repository, selected_value=None, add_id_to_name=True, galaxy_url=None ):
def build_changeset_revision_select_field( trans, repository, selected_value=None, add_id_to_name=True ):
"""
Build a SelectField whose options are the changeset_revision
strings of all downloadable_revisions of the received repository.
@@ -541,15 +557,9 @@ def build_changeset_revision_select_field( trans, repository, selected_value=Non
options.append( ( revision_label, changeset_revision ) )
refresh_on_change_values.append( changeset_revision )
if add_id_to_name:
if galaxy_url:
name = '%s_changeset_revision_%d' % ( galaxy_url, repository.id )
else:
name = 'changeset_revision_%d' % repository.id
name = 'changeset_revision_%d' % repository.id
else:
if galaxy_url:
name = '%s_changeset_revision' % galaxy_url
else:
name = 'changeset_revision'
name = 'changeset_revision'
select_field = SelectField( name=name,
refresh_on_change=True,
refresh_on_change_values=refresh_on_change_values )
lib/galaxy/webapps/community/controllers/repository.py
+6 -36
View File
@@ -122,7 +122,6 @@ class RepositoryListGrid( grids.Grid ):
return 'yes'
return ''
# Grid definition
galaxy_url = None
title = "Repositories"
model_class = model.Repository
template='/webapps/community/repository/grid.mako'
@@ -192,12 +191,11 @@ class DownloadableRepositoryListGrid( RepositoryListGrid ):
Display a SelectField whose options are the changeset_revision
strings of all downloadable_revisions of this repository.
"""
select_field = build_changeset_revision_select_field( trans,
repository,
galaxy_url=grid.galaxy_url )
select_field = build_changeset_revision_select_field( trans, repository )
if len( select_field.options ) > 1:
return select_field.get_html()
return repository.revision
title = "Downloadable repositories"
columns = [
RepositoryListGrid.NameColumn( "Name",
key="name",
@@ -264,9 +262,7 @@ class RepositoryController( BaseController, ItemRatings ):
return self.category_list_grid( trans, **kwd )
@web.expose
def browse_downloadable_repositories( self, trans, **kwd ):
tool_shed_name = kwd.get( 'tool_shed_name', None )
repository_id = kwd.get( 'id', None )
galaxy_url = kwd.get( 'galaxy_url', None )
if 'operation' in kwd:
operation = kwd[ 'operation' ].lower()
if operation == "preview_tools_in_changeset":
@@ -274,8 +270,7 @@ class RepositoryController( BaseController, ItemRatings ):
return trans.response.send_redirect( web.url_for( controller='repository',
action='preview_tools_in_changeset',
repository_id=repository_id,
changeset_revision=repository.tip,
galaxy_url=galaxy_url ) )
changeset_revision=repository.tip ) )
# The changeset_revision_select_field in the RepositoryListGrid performs a refresh_on_change
# which sends in request parameters like changeset_revison_1, changeset_revision_2, etc. One
@@ -292,31 +287,10 @@ class RepositoryController( BaseController, ItemRatings ):
return trans.response.send_redirect( web.url_for( controller='repository',
action='preview_tools_in_changeset',
repository_id=trans.security.encode_id( repository.id ),
changeset_revision=v,
galaxy_url=galaxy_url ) )
elif k.find( changset_revision_str ) > 0:
# Keys look like: 'localhost:8763_changeset_revision_3' and values: '4ef2cf631604'.
items = k.split( '_%s' % changset_revision_str )
galaxy_url = items[0]
repository_id = trans.security.encode_id( int( items[1] ) )
repository = get_repository( trans, repository_id )
if repository.tip != v:
return trans.response.send_redirect( web.url_for( controller='repository',
action='preview_tools_in_changeset',
repository_id=trans.security.encode_id( repository.id ),
changeset_revision=v,
galaxy_url=galaxy_url ) )
if tool_shed_name:
title = "%s downloadable repositories" % tool_shed_name
else:
title = "Downloadable repositories"
self.downloadable_repository_list_grid.title = title
self.downloadable_repository_list_grid.galaxy_url = galaxy_url
changeset_revision=v ) )
url_args = dict( action='browse_downloadable_repositories',
operation='preview_tools_in_changeset',
repository_id=repository_id,
galaxy_url=galaxy_url )
repository_id=repository_id )
self.downloadable_repository_list_grid.operations = [ grids.GridOperation( "Preview and install tools",
url_args=url_args,
allow_multiple=False,
@@ -329,7 +303,6 @@ class RepositoryController( BaseController, ItemRatings ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
galaxy_url = util.restore_text( params.get( 'galaxy_url', '' ) )
repository = get_repository( trans, repository_id )
changeset_revision = util.restore_text( params.get( 'changeset_revision', repository.tip ) )
repository_metadata = get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision )
@@ -348,7 +321,6 @@ class RepositoryController( BaseController, ItemRatings ):
revision_label=revision_label,
changeset_revision_select_field=changeset_revision_select_field,
metadata=metadata,
galaxy_url=galaxy_url,
display_for_install=True,
message=message,
status=status )
@@ -357,7 +329,7 @@ class RepositoryController( BaseController, ItemRatings ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
galaxy_url = util.restore_text( params.get( 'galaxy_url', '' ) )
galaxy_url = trans.get_cookie( name='toolshedgalaxyurl' )
repository = get_repository( trans, repository_id )
changeset_revision = util.restore_text( params.get( 'changeset_revision', repository.tip ) )
# Redirect back to local Galaxy to perform install.
@@ -1232,7 +1204,6 @@ class RepositoryController( BaseController, ItemRatings ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
galaxy_url = util.restore_text( params.get( 'galaxy_url', '' ) )
display_for_install = util.string_as_bool( params.get( 'display_for_install', False ) )
repository = get_repository( trans, repository_id )
repo = hg.repository( get_configured_ui(), repository.repo_path )
@@ -1274,7 +1245,6 @@ class RepositoryController( BaseController, ItemRatings ):
tool_state=tool_state,
is_malicious=is_malicious,
display_for_install=display_for_install,
galaxy_url=galaxy_url,
message=message,
status=status )
except Exception, e:
templates/admin/select_tool_panel_section.mako
+1 -1
View File
@@ -16,7 +16,7 @@
or concerns.
</div>
<br/>
<div class="errormessage">
<div class="warningmessage">
Installation may take a while, depending upon the size of the repository contents. Wait until a message is displayed in your
browser after clicking the <b>Install</b> button below.
</div>
templates/webapps/community/repository/common.mako
+3 -3
View File
@@ -83,7 +83,7 @@
hg clone <a href="${clone_str}">${clone_str}</a>
</%def>
<%def name="render_repository_tools_and_workflows( metadata, can_set_metadata=False, display_for_install=False, galaxy_url=None )">
<%def name="render_repository_tools_and_workflows( metadata, can_set_metadata=False, display_for_install=False )">
%if metadata or can_set_metadata:
<p/>
<div class="toolForm">
@@ -111,12 +111,12 @@
<tr>
<td>
<div style="float: left; margin-left: 1px;" class="menubutton split popup" id="tool-${tool_dict[ 'id' ].replace( ' ', '_' )}-popup">
<a class="view-info" href="${h.url_for( controller='repository', action='display_tool', repository_id=trans.security.encode_id( repository.id ), tool_config=tool_dict[ 'tool_config' ], changeset_revision=changeset_revision, display_for_install=display_for_install, galaxy_url=galaxy_url )}">
<a class="view-info" href="${h.url_for( controller='repository', action='display_tool', repository_id=trans.security.encode_id( repository.id ), tool_config=tool_dict[ 'tool_config' ], changeset_revision=changeset_revision, display_for_install=display_for_install )}">
${tool_dict[ 'name' ]}
</a>
</div>
<div popupmenu="tool-${tool_dict[ 'id' ].replace( ' ', '_' )}-popup">
<a class="action-button" href="${h.url_for( controller='repository', action='view_tool_metadata', repository_id=trans.security.encode_id( repository.id ), changeset_revision=changeset_revision, tool_id=tool_dict[ 'id' ], display_for_install=display_for_install, galaxy_url=galaxy_url )}">View tool metadata</a>
<a class="action-button" href="${h.url_for( controller='repository', action='view_tool_metadata', repository_id=trans.security.encode_id( repository.id ), changeset_revision=changeset_revision, tool_id=tool_dict[ 'id' ], display_for_install=display_for_install )}">View tool metadata</a>
</div>
</td>
<td>${tool_dict[ 'description' ]}</td>
templates/webapps/community/repository/preview_tools_in_changeset.mako
+3 -3
View File
@@ -64,7 +64,7 @@
<br/><br/>
<ul class="manage-table-actions">
<a class="action-button" href="${h.url_for( controller='repository', action='install_repository_revision', repository_id=trans.security.encode_id( repository.id ), webapp='community', changeset_revision=changeset_revision, galaxy_url=galaxy_url )}">Install to local Galaxy</a>
<a class="action-button" href="${h.url_for( controller='repository', action='install_repository_revision', repository_id=trans.security.encode_id( repository.id ), webapp='community', changeset_revision=changeset_revision )}">Install to local Galaxy</a>
</ul>
%if message:
@@ -75,7 +75,7 @@
<div class="toolFormTitle">Repository ${repository.name}</div>
<div class="toolFormBody">
%if len( changeset_revision_select_field.options ) > 1:
<form name="change_revision" id="change_revision" action="${h.url_for( controller='repository', action='preview_tools_in_changeset', repository_id=trans.security.encode_id( repository.id ), galaxy_url=galaxy_url )}" method="post" >
<form name="change_revision" id="change_revision" action="${h.url_for( controller='repository', action='preview_tools_in_changeset', repository_id=trans.security.encode_id( repository.id ) )}" method="post" >
<div class="form-row">
<%
if changeset_revision == repository.tip:
@@ -98,4 +98,4 @@
</div>
</div>
<p/>
${render_repository_tools_and_workflows( metadata, display_for_install=display_for_install, galaxy_url=galaxy_url )}
${render_repository_tools_and_workflows( metadata, display_for_install=display_for_install )}
templates/webapps/community/repository/tool_form.mako
+1 -1
View File
@@ -111,7 +111,7 @@
<br/><br/>
<ul class="manage-table-actions">
%if display_for_install:
<a class="action-button" href="${h.url_for( controller='repository', action='install_repository_revision', repository_id=trans.security.encode_id( repository.id ), webapp='community', changeset_revision=changeset_revision, galaxy_url=galaxy_url )}">Install to local Galaxy</a>
<a class="action-button" href="${h.url_for( controller='repository', action='install_repository_revision', repository_id=trans.security.encode_id( repository.id ), webapp='community', changeset_revision=changeset_revision )}">Install to local Galaxy</a>
%else:
%if is_new:
<a class="action-button" href="${h.url_for( controller='upload', action='upload', repository_id=trans.security.encode_id( repository.id ), webapp='community' )}">Upload files to repository</a>
templates/webapps/galaxy/admin/index.mako
+1 -1
View File
@@ -70,7 +70,7 @@
<div class="toolSectionBody">
<div class="toolSectionBg">
%for name, url in trans.app.tool_shed_registry.tool_sheds.items():
<div class="toolTitle"><a href="${h.url_for( controller='admin', action='browse_tool_shed', tool_shed_name=name, tool_shed_url=url )}" target="galaxy_main">${name}</a></div>
<div class="toolTitle"><a href="${h.url_for( controller='admin', action='browse_tool_shed', tool_shed_url=url )}" target="galaxy_main">${name}</a></div>
%endfor
</div>
</div>