opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Ehancements for automatic installation of tool dependencies when installing tool shed repositories; the enhancements support moving files and changing subdirectories during tool dependency installation.

Browse Source
This commit is contained in:
Greg Von Kuster
2012-06-08 11:46:29 -04:00
parent 15ef69b7d2
commit ab9afd1af2
5 changed files with 72 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/tool_shed/tool_dependencies/common_util.py
+6 -4
View File
@@ -18,14 +18,16 @@ def istar( file_path ):
def iszip( file_path ):
return check_zip( file_path )
def tar_extraction_directory( file_path, file_name ):
file_name = file_name.strip()
extensions = [ '.tar.gz', '.tgz', '.tar.bz2', '.zip' ]
for extension in extensions:
if file_name.endswith( extension ):
if file_name.find( extension ) > 0:
dir_name = file_name[ :-len( extension ) ]
full_path = os.path.abspath( os.path.join( file_path, dir_name ) )
if os.path.exists( full_path ):
if os.path.exists( os.path.abspath( os.path.join( file_path, dir_name ) ) ):
return dir_name
raise ValueError( 'Could not find directory %s' % full_path )
if os.path.exists( os.path.abspath( os.path.join( file_path, file_name ) ) ):
return os.path.abspath( os.path.join( file_path, file_name ) )
raise ValueError( 'Could not find directory %s' % os.path.abspath( os.path.join( file_path, file_name[ :-len( extension ) ] ) ) )
def url_download( install_dir, downloaded_file_name, download_url ):
file_path = os.path.join( install_dir, downloaded_file_name )
src = None
lib/galaxy/tool_shed/tool_dependencies/fabric_util.py
+43 -11
View File
@@ -1,7 +1,7 @@
# For Python 2.5
from __future__ import with_statement
import os
import os, shutil
from contextlib import contextmanager
import common_util
@@ -11,7 +11,11 @@ import pkg_resources
pkg_resources.require('ssh' )
pkg_resources.require( 'Fabric' )
from fabric.api import env, lcd, local
from fabric.api import env, lcd, local, settings
DIRECTORY_BUILD_COMMAND_NAMES = [ 'change_directory' ]
MOVE_BUILD_COMMAND_NAMES = [ 'move_directory_files', 'move_file' ]
ALL_BUILD_COMMAND_NAMES = DIRECTORY_BUILD_COMMAND_NAMES + MOVE_BUILD_COMMAND_NAMES
def check_fabric_version():
version = env.version
@@ -83,11 +87,39 @@ def install_and_build_package( params_dict ):
dir = package_name
if build_commands:
with lcd( dir ):
for build_command in build_commands:
output = local( build_command, capture=True )
log_results( build_command, output, os.path.join( install_dir, 'build_commands.log' ) )
if output.return_code:
return '%s. ' % str( output.stderr )
current_dir = os.path.abspath( os.path.join( work_dir, dir ) )
for build_command_tup in build_commands:
build_command_key, build_command_dict = build_command_tup
if build_command_key.find( 'v^v^v' ) >= 0:
build_command_items = build_command_key.split( 'v^v^v' )
build_command_name = build_command_items[ 0 ]
build_command = build_command_items[ 1 ]
elif build_command_key in ALL_BUILD_COMMAND_NAMES:
build_command_name = build_command_key
else:
build_command_name = None
if build_command_name:
if build_command_name == 'change_directory':
current_dir = os.path.join( current_dir, build_command )
lcd( current_dir )
elif build_command_name == 'move_directory_files':
source_directory = os.path.abspath( os.path.join( current_dir, build_command_dict[ 'source_directory' ] ) )
destination_directory = build_command_dict[ 'destination_directory' ]
for file_name in os.listdir( source_directory ):
source_file = os.path.join( source_directory, file_name )
destination_file = os.path.join( destination_directory, file_name )
shutil.move( source_file, destination_file )
elif build_command_name == 'move_file':
source_file = os.path.abspath( os.path.join( current_dir, build_command_dict[ 'source' ] ) )
destination = build_command_dict[ 'destination' ]
shutil.move( source_file, destination )
else:
build_command = build_command_key
with settings( warn_only=True ):
output = local( build_command, capture=True )
log_results( build_command, output, os.path.join( install_dir, 'build_commands.log' ) )
if output.return_code:
return '%s. ' % str( output.stderr )
return ''
def log_results( command, fabric_AttributeString, file_path ):
"""
@@ -98,12 +130,12 @@ def log_results( command, fabric_AttributeString, file_path ):
logfile = open( file_path, 'ab' )
else:
logfile = open( file_path, 'wb' )
logfile.write( "#############################################" )
logfile.write( "\n#############################################" )
logfile.write( '\n%s\nSTDOUT\n' % command )
logfile.write( "#############################################" )
logfile.write( "#############################################\n" )
logfile.write( str( fabric_AttributeString.stdout ) )
logfile.write( "#############################################" )
logfile.write( "\n#############################################" )
logfile.write( '\n%s\nSTDERR\n' % command )
logfile.write( "#############################################" )
logfile.write( "#############################################\n" )
logfile.write( str( fabric_AttributeString.stderr ) )
logfile.close()
lib/galaxy/tool_shed/tool_dependencies/install_util.py
+18 -3
View File
@@ -96,7 +96,24 @@ def run_fabric_method( app, elem, fabfile_path, tool_dependency_dir, install_dir
if param_name:
if param_name == 'build_commands':
for build_command_elem in param_elem:
build_commands.append( build_command_elem.text.replace( '$INSTALL_DIR', install_dir ) )
build_command_dict = {}
build_command_name = build_command_elem.get( 'name' )
if build_command_name:
if build_command_name in MOVE_BUILD_COMMAND_NAMES:
build_command_key = build_command_name
for move_elem in build_command_elem:
move_elem_text = move_elem.text.replace( '$INSTALL_DIR', install_dir )
if move_elem_text:
build_command_dict[ move_elem.tag ] = move_elem_text
elif build_command_elem.text:
build_command_key = '%sv^v^v%s' % ( build_command_name, build_command_elem.text )
else:
continue
else:
build_command_key = build_command_elem.text.replace( '$INSTALL_DIR', install_dir )
if not build_command_key:
continue
build_commands.append( ( build_command_key, build_command_dict ) )
if build_commands:
params_dict[ 'build_commands' ] = build_commands
else:
@@ -122,7 +139,6 @@ def run_fabric_method( app, elem, fabfile_path, tool_dependency_dir, install_dir
return message
except:
return '%s. ' % str( e )
print package_name, 'installed to', install_dir
return ''
def run_proprietary_fabric_method( app, elem, fabfile_path, tool_dependency_dir, install_dir, package_name=None, **kwd ):
"""
@@ -161,7 +177,6 @@ def run_proprietary_fabric_method( app, elem, fabfile_path, tool_dependency_dir,
if message:
return message
else:
print package_name, 'installed to', install_dir
return ''
def run_subprocess( app, cmd ):
env = os.environ
lib/galaxy/util/shed_util.py
+4 -9
View File
@@ -485,7 +485,7 @@ def generate_fabfile_metadata( elem, requirements_dict, proprietary=False ):
requirements_dict[ key ] = fabfiles_dict
return requirements_dict
def generate_metadata_using_disk_files( toolbox, relative_install_dir, repository_clone_url ):
"""generate metadata using only the repository files on disk - files are not retrieved from the repository manifest."""
"""Generate metadata using only the repository files on disk - files are not retrieved from the repository manifest."""
metadata_dict = {}
tool_dependencies_config = None
datatypes_config = get_config_from_disk( 'datatypes_conf.xml', relative_install_dir )
@@ -1081,9 +1081,8 @@ def handle_tool_dependencies( app, tool_shed_repository, installed_changeset_rev
"""
Install and build tool dependencies defined in the tool_dependencies_config. This config's tag sets can refer to installation
methods in Galaxy's tool_dependencies module or to proprietary fabric scripts contained in the repository. Future enhancements
to handling tool dependencies may provide installation processes in addition to fabric based processes.
The dependencies will be installed in:
to handling tool dependencies may provide installation processes in addition to fabric based processes. The dependencies will be
installed in:
~/<app.config.tool_dependency_dir>/<package_name>/<package_version>/<repository_owner>/<repository_name>/<installed_changeset_revision>
"""
status = 'ok'
@@ -1192,10 +1191,6 @@ def load_repository_contents( trans, repository_name, description, owner, change
tool_path,
repository_tools_tups,
work_dir )
try:
shutil.rmtree( work_dir )
except:
pass
# Handle missing index files for tool parameters that are dynamically generated select lists.
sample_files = metadata_dict.get( 'sample_files', [] )
repository_tools_tups, sample_files_copied = handle_missing_index_file( trans.app, tool_path, sample_files, repository_tools_tups )
@@ -1287,7 +1282,7 @@ def pull_repository( repo, repository_clone_url, ctx_rev ):
commands.pull( get_configured_ui(),
repo,
source=repository_clone_url,
rev=ctx_rev )
rev=[ ctx_rev ] )
def remove_from_shed_tool_config( trans, shed_tool_conf_dict, guids_to_remove ):
# A tool shed repository is being uninstalled so change the shed_tool_conf file. Parse the config file to generate the entire list
# of config_elems instead of using the in-memory list since it will be a subset of the entire list if one or more repositories have
lib/galaxy/webapps/community/controllers/common.py
+1 -3
View File
@@ -1002,13 +1002,11 @@ def reset_all_metadata_on_repository( trans, id, **kwd ):
metadata_changeset_revision = ancestor_changeset_revision
metadata_dict = ancestor_metadata_dict
create_or_update_repository_metadata( trans, id, repository, metadata_changeset_revision, metadata_dict )
# Keep track of the changeset_revisions that we've persisted.
changeset_revisions.append( metadata_changeset_revision )
ancestor_changeset_revision = current_changeset_revision
ancestor_metadata_dict = current_metadata_dict
else:
# We're either at the first change set in the change log or we have just created or updated a repository_metadata record. At
# this point we set the ancestor changeset to the current changeset for comparison in the next iteration.
# We're at the beginning of the change log.
ancestor_changeset_revision = current_changeset_revision
ancestor_metadata_dict = current_metadata_dict
if not ctx.children():