mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
Merge pull request #1263 from nsoranzo/dev_tool_shed_fixes
Extend the logging of tool dependency status changes.
This commit is contained in:
Martin Cech
@@ -503,7 +503,7 @@ class AdminToolshed( AdminGalaxy ):
|
||||
if err_msg:
|
||||
message += err_msg
|
||||
status = 'error'
|
||||
message += "Installed tool dependencies: %s" % ','.join( td.name for td in installed_tool_dependencies )
|
||||
message += "Installed tool dependencies: %s" % ', '.join( td.name for td in installed_tool_dependencies )
|
||||
td_ids = [ trans.security.encode_id( td.id ) for td in tool_shed_repository.tool_dependencies ]
|
||||
return trans.response.send_redirect( web.url_for( controller='admin_toolshed',
|
||||
action='manage_tool_dependencies',
|
||||
@@ -1919,8 +1919,8 @@ class AdminToolshed( AdminGalaxy ):
|
||||
if new_repository_dependencies == original_repository_dependencies:
|
||||
for new_repository_tup in new_repository_dependencies:
|
||||
# Make sure all dependencies are installed.
|
||||
# TODO: Repository dependencies that are not installed should be displayed to to the user,
|
||||
# giving them the option to install them or now. This is the same behavior as when initially
|
||||
# TODO: Repository dependencies that are not installed should be displayed to the user,
|
||||
# giving them the option to install them or not. This is the same behavior as when initially
|
||||
# installing and when re-installing.
|
||||
new_tool_shed, new_name, new_owner, new_changeset_revision, new_pir, new_oicct = \
|
||||
common_util.parse_repository_dependency_tuple( new_repository_tup )
|
||||
|
||||
@@ -126,10 +126,10 @@ class InstallToolDependencyManager( object ):
|
||||
# Since there was an installation error, update the tool dependency status to Error. The remove_installation_path option must
|
||||
# be left False here.
|
||||
error_message = '%s\n%s' % ( self.format_traceback(), str( e ) )
|
||||
tool_dependency = tool_dependency_util.handle_tool_dependency_installation_error( self.app,
|
||||
tool_dependency,
|
||||
error_message,
|
||||
remove_installation_path=False )
|
||||
tool_dependency = tool_dependency_util.set_tool_dependency_attributes(self.app,
|
||||
tool_dependency=tool_dependency,
|
||||
status=self.app.install_model.ToolDependency.installation_status.ERROR,
|
||||
error_message=error_message)
|
||||
tool_dependency = self.mark_tool_dependency_installed( tool_dependency )
|
||||
return tool_dependency
|
||||
|
||||
@@ -204,10 +204,10 @@ class InstallToolDependencyManager( object ):
|
||||
# Since there was an installation error, update the tool dependency status to Error. The
|
||||
# remove_installation_path option must be left False here.
|
||||
tool_dependency = \
|
||||
tool_dependency_util.handle_tool_dependency_installation_error( self.app,
|
||||
tool_dependency,
|
||||
error_message,
|
||||
remove_installation_path=False )
|
||||
tool_dependency_util.set_tool_dependency_attributes(self.app,
|
||||
tool_dependency=tool_dependency,
|
||||
status=self.app.install_model.ToolDependency.installation_status.ERROR,
|
||||
error_message=error_message)
|
||||
if tool_dependency and tool_dependency.status in [ self.install_model.ToolDependency.installation_status.INSTALLED,
|
||||
self.install_model.ToolDependency.installation_status.ERROR ]:
|
||||
installed_packages.append( tool_dependency )
|
||||
@@ -422,9 +422,7 @@ class InstallToolDependencyManager( object ):
|
||||
status = self.install_model.ToolDependency.installation_status.INSTALLED
|
||||
tool_dependency = tool_dependency_util.set_tool_dependency_attributes( self.app,
|
||||
tool_dependency=tool_dependency,
|
||||
status=status,
|
||||
error_message=None,
|
||||
remove_from_disk=False )
|
||||
status=status )
|
||||
return tool_dependency
|
||||
|
||||
|
||||
|
||||
@@ -204,9 +204,7 @@ class RepairRepositoryManager():
|
||||
tool_dependency = \
|
||||
tool_dependency_util.set_tool_dependency_attributes( self.app,
|
||||
tool_dependency=tool_dependency,
|
||||
status=self.app.install_model.ToolDependency.installation_status.UNINSTALLED,
|
||||
error_message=None,
|
||||
remove_from_disk=True )
|
||||
status=self.app.install_model.ToolDependency.installation_status.UNINSTALLED )
|
||||
# Install tool dependencies.
|
||||
irm.update_tool_shed_repository_status( repository,
|
||||
self.app.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES )
|
||||
|
||||
@@ -19,7 +19,7 @@ from galaxy.util import unicodify
|
||||
|
||||
from tool_shed.galaxy_install.tool_dependencies.recipe import asynchronous_reader
|
||||
|
||||
from tool_shed.util import basic_util
|
||||
from tool_shed.util import basic_util, tool_dependency_util
|
||||
|
||||
log = logging.getLogger( __name__ )
|
||||
|
||||
@@ -113,7 +113,6 @@ class InstallEnvironment( object ):
|
||||
|
||||
def handle_command( self, tool_dependency, cmd, return_output=False, job_name="" ):
|
||||
"""Handle a command and log the results."""
|
||||
context = self.app.install_model.context
|
||||
command = str( cmd )
|
||||
output = self.handle_complex_command( command, job_name=job_name )
|
||||
self.log_results( cmd, output, os.path.join( self.install_dir, basic_util.INSTALLATION_LOG ) )
|
||||
@@ -126,17 +125,19 @@ class InstallEnvironment( object ):
|
||||
log.warn( "Length of stderr > %s, so only a portion will be saved in the database." % str( DATABASE_MAX_STRING_SIZE_PRETTY ) )
|
||||
stderr = shrink_string_by_size( stderr, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
|
||||
if output.return_code not in [ 0 ]:
|
||||
tool_dependency.status = self.app.install_model.ToolDependency.installation_status.ERROR
|
||||
status = self.app.install_model.ToolDependency.installation_status.ERROR
|
||||
if stderr:
|
||||
tool_dependency.error_message = unicodify( stderr )
|
||||
error_message = unicodify( stderr )
|
||||
elif stdout:
|
||||
tool_dependency.error_message = unicodify( stdout )
|
||||
error_message = unicodify( stdout )
|
||||
else:
|
||||
# We have a problem if there was no stdout and no stderr.
|
||||
tool_dependency.error_message = "Unknown error occurred executing shell command %s, return_code: %s" % \
|
||||
error_message = "Unknown error occurred executing shell command %s, return_code: %s" % \
|
||||
( str( cmd ), str( output.return_code ) )
|
||||
context.add( tool_dependency )
|
||||
context.flush()
|
||||
tool_dependency_util.set_tool_dependency_attributes(self.app,
|
||||
tool_dependency=tool_dependency,
|
||||
status=status,
|
||||
error_message=error_message)
|
||||
if return_output:
|
||||
return output
|
||||
return output.return_code
|
||||
|
||||
@@ -284,8 +284,7 @@ class AssertDirectoryExecutable( RecipeStep ):
|
||||
tool_dependency = tool_dependency_util.set_tool_dependency_attributes( self.app,
|
||||
tool_dependency,
|
||||
status=status,
|
||||
error_message=error_message,
|
||||
remove_from_disk=False )
|
||||
error_message=error_message )
|
||||
return tool_dependency, None, None
|
||||
|
||||
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
|
||||
@@ -330,8 +329,7 @@ class AssertDirectoryExists( RecipeStep ):
|
||||
tool_dependency = tool_dependency_util.set_tool_dependency_attributes( self.app,
|
||||
tool_dependency,
|
||||
status=status,
|
||||
error_message=error_message,
|
||||
remove_from_disk=False )
|
||||
error_message=error_message )
|
||||
return tool_dependency, None, None
|
||||
|
||||
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
|
||||
@@ -380,8 +378,7 @@ class AssertFileExecutable( RecipeStep ):
|
||||
tool_dependency = tool_dependency_util.set_tool_dependency_attributes( self.app,
|
||||
tool_dependency,
|
||||
status=status,
|
||||
error_message=error_message,
|
||||
remove_from_disk=False )
|
||||
error_message=error_message )
|
||||
return tool_dependency, None, None
|
||||
|
||||
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
|
||||
@@ -427,8 +424,7 @@ class AssertFileExists( RecipeStep ):
|
||||
tool_dependency = tool_dependency_util.set_tool_dependency_attributes( self.app,
|
||||
tool_dependency,
|
||||
status=status,
|
||||
error_message=error_message,
|
||||
remove_from_disk=False )
|
||||
error_message=error_message )
|
||||
return tool_dependency, None, None
|
||||
|
||||
def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ):
|
||||
|
||||
@@ -180,10 +180,10 @@ class Install( RecipeTag, SyncDatabase ):
|
||||
error_message += 'tag set.'
|
||||
# Since there was an installation error, update the tool dependency status to Error.
|
||||
# The remove_installation_path option must be left False here.
|
||||
tool_dependency = tool_dependency_util.handle_tool_dependency_installation_error( self.app,
|
||||
tool_dependency,
|
||||
error_message,
|
||||
remove_installation_path=False )
|
||||
tool_dependency = tool_dependency_util.set_tool_dependency_attributes(self.app,
|
||||
tool_dependency=tool_dependency,
|
||||
status=self.app.install_model.ToolDependency.installation_status.ERROR,
|
||||
error_message=error_message)
|
||||
else:
|
||||
raise NotImplementedError( 'Only install version 1.0 is currently supported (i.e., change your tag to be <install version="1.0">).' )
|
||||
return tool_dependency, proceed_with_install, actions_elem_tuples
|
||||
@@ -213,9 +213,7 @@ class Package( RecipeTag ):
|
||||
tool_dependency = \
|
||||
tool_dependency_util.set_tool_dependency_attributes( self.app,
|
||||
tool_dependency=tool_dependency,
|
||||
status=self.app.install_model.ToolDependency.installation_status.ERROR,
|
||||
error_message=None,
|
||||
remove_from_disk=False )
|
||||
status=self.app.install_model.ToolDependency.installation_status.ERROR )
|
||||
else:
|
||||
proceed_with_install = True
|
||||
return tool_dependency, proceed_with_install, action_elem_tuples
|
||||
@@ -329,18 +327,18 @@ class Repository( RecipeTag, SyncDatabase ):
|
||||
error_message = 'Error defining env.sh file for package %s, return_code: %s' % \
|
||||
( str( package_name ), str( return_code ) )
|
||||
tool_dependency = \
|
||||
tool_dependency_util.handle_tool_dependency_installation_error( self.app,
|
||||
tool_dependency,
|
||||
error_message,
|
||||
remove_installation_path=False )
|
||||
tool_dependency_util.set_tool_dependency_attributes(self.app,
|
||||
tool_dependency=tool_dependency,
|
||||
status=self.app.install_model.ToolDependency.installation_status.ERROR,
|
||||
error_message=error_message)
|
||||
elif required_tool_dependency is not None and required_tool_dependency.in_error_state:
|
||||
error_message = "This tool dependency's required tool dependency %s version %s has status %s." % \
|
||||
( str( required_tool_dependency.name ), str( required_tool_dependency.version ), str( required_tool_dependency.status ) )
|
||||
tool_dependency = \
|
||||
tool_dependency_util.handle_tool_dependency_installation_error( self.app,
|
||||
tool_dependency,
|
||||
error_message,
|
||||
remove_installation_path=False )
|
||||
tool_dependency_util.set_tool_dependency_attributes(self.app,
|
||||
tool_dependency=tool_dependency,
|
||||
status=self.app.install_model.ToolDependency.installation_status.ERROR,
|
||||
error_message=error_message)
|
||||
else:
|
||||
tool_dependency = \
|
||||
tool_dependency_util.set_tool_dependency_attributes( self.app,
|
||||
@@ -556,18 +554,10 @@ class SetEnvironment( RecipeTag ):
|
||||
else:
|
||||
attr_tups_of_dependencies_for_install = [ ( td.name, td.version, td.type ) for td in tool_dependency_db_records ]
|
||||
try:
|
||||
tool_dependencies = self.set_environment( package_elem, tool_shed_repository, attr_tups_of_dependencies_for_install )
|
||||
self.set_environment( package_elem, tool_shed_repository, attr_tups_of_dependencies_for_install )
|
||||
except Exception, e:
|
||||
error_message = "Error setting environment for tool dependency: %s" % str( e )
|
||||
log.debug( error_message )
|
||||
for tool_dependency in tool_dependencies:
|
||||
if tool_dependency and tool_dependency.status == self.app.install_model.ToolDependency.installation_status.ERROR:
|
||||
# Since there was an installation error, update the tool dependency status to Error. The
|
||||
# remove_installation_path option must be left False here.
|
||||
tool_dependency = tool_dependency_util.handle_tool_dependency_installation_error( self.app,
|
||||
tool_dependency,
|
||||
error_message,
|
||||
remove_installation_path=False )
|
||||
return tool_dependency, proceed_with_install, action_elem_tuples
|
||||
|
||||
def set_environment( self, elem, tool_shed_repository, attr_tups_of_dependencies_for_install ):
|
||||
@@ -613,7 +603,7 @@ class SetEnvironment( RecipeTag ):
|
||||
# <requirement type="set_environment">R_SCRIPT_PATH</requirement>).
|
||||
env_var_action = env_var_elem.get( 'action', None )
|
||||
if env_var_name and env_var_action:
|
||||
# Tool dependencies of type "set_environmnet" always have the version attribute set to None.
|
||||
# Tool dependencies of type "set_environment" always have the version attribute set to None.
|
||||
attr_tup = ( env_var_name, None, 'set_environment' )
|
||||
if attr_tup in attr_tups_of_dependencies_for_install:
|
||||
install_dir = \
|
||||
@@ -654,8 +644,7 @@ class SetEnvironment( RecipeTag ):
|
||||
tool_dependency_util.set_tool_dependency_attributes( self.app,
|
||||
tool_dependency=tool_dependency,
|
||||
status=status,
|
||||
error_message=error_message,
|
||||
remove_from_disk=False )
|
||||
error_message=error_message )
|
||||
else:
|
||||
if tool_dependency.status not in [ self.app.install_model.ToolDependency.installation_status.ERROR,
|
||||
self.app.install_model.ToolDependency.installation_status.INSTALLED ]:
|
||||
@@ -663,9 +652,7 @@ class SetEnvironment( RecipeTag ):
|
||||
tool_dependency = \
|
||||
tool_dependency_util.set_tool_dependency_attributes( self.app,
|
||||
tool_dependency=tool_dependency,
|
||||
status=status,
|
||||
error_message=None,
|
||||
remove_from_disk=False )
|
||||
status=status )
|
||||
log.debug( 'Environment variable %s set in %s for tool dependency %s.' %
|
||||
( str( env_var_name ), str( install_dir ), str( tool_dependency.name ) ) )
|
||||
else:
|
||||
@@ -675,7 +662,6 @@ class SetEnvironment( RecipeTag ):
|
||||
tool_dependency_util.set_tool_dependency_attributes( self.app,
|
||||
tool_dependency=tool_dependency,
|
||||
status=status,
|
||||
error_message=error_message,
|
||||
remove_from_disk=False )
|
||||
error_message=error_message )
|
||||
tool_dependencies.append( tool_dependency )
|
||||
return tool_dependencies
|
||||
|
||||
@@ -48,14 +48,7 @@ def create_or_update_tool_dependency( app, tool_shed_repository, name, version,
|
||||
# In some cases we should not override the current status of an existing tool_dependency, so do so only
|
||||
# if set_status is True.
|
||||
if set_status:
|
||||
if str( tool_dependency.status ) != str( status ):
|
||||
debug_msg = 'Updating an existing record for version %s of tool dependency %s for revision %s of repository %s ' % \
|
||||
( str( version ), str( name ), str( tool_shed_repository.changeset_revision ), str( tool_shed_repository.name ) )
|
||||
debug_msg += 'by updating the status from %s to %s.' % ( str( tool_dependency.status ), str( status ) )
|
||||
log.debug( debug_msg )
|
||||
tool_dependency.status = status
|
||||
context.add( tool_dependency )
|
||||
context.flush()
|
||||
set_tool_dependency_attributes(app, tool_dependency=tool_dependency, status=status)
|
||||
else:
|
||||
# Create a new tool_dependency record for the tool_shed_repository.
|
||||
debug_msg = 'Creating a new record for version %s of tool dependency %s for revision %s of repository %s. ' % \
|
||||
@@ -227,38 +220,6 @@ def get_tool_dependency_install_dir( app, repository_name, repository_owner, rep
|
||||
repository_changeset_revision ) )
|
||||
|
||||
|
||||
def handle_tool_dependency_installation_error( app, tool_dependency, error_message, remove_installation_path=False ):
|
||||
# Since there was an installation error, remove the installation directory because the install_package method uses
|
||||
# this: "if os.path.exists( install_dir ):". Setting remove_installation_path to True should rarely occur. It is
|
||||
# currently set to True only to handle issues with installing tool dependencies into an Amazon S3 bucket.
|
||||
sa_session = app.install_model.context
|
||||
tool_shed_repository = tool_dependency.tool_shed_repository
|
||||
install_dir = get_tool_dependency_install_dir( app=app,
|
||||
repository_name=tool_shed_repository.name,
|
||||
repository_owner=tool_shed_repository.owner,
|
||||
repository_changeset_revision=tool_shed_repository.installed_changeset_revision,
|
||||
tool_dependency_type=tool_dependency.type,
|
||||
tool_dependency_name=tool_dependency.name,
|
||||
tool_dependency_version=tool_dependency.version )
|
||||
if remove_installation_path:
|
||||
# This will be True only in the case where an exception was encountered during the installation process after
|
||||
# the installation path was created but before any information was written to the installation log and the
|
||||
# tool dependency status was not set to "Installed" or "Error".
|
||||
if os.path.exists( install_dir ):
|
||||
log.debug( 'Attempting to remove installation directory %s for version %s of tool dependency %s %s' %
|
||||
( str( install_dir ), str( tool_dependency.version ), str( tool_dependency.type ), str( tool_dependency.name ) ) )
|
||||
log.debug( 'due to the following installation error:\n%s' % str( error_message ) )
|
||||
try:
|
||||
shutil.rmtree( install_dir )
|
||||
except Exception, e:
|
||||
log.exception( 'Error removing existing installation directory %s: %s', (install_dir, str(e)) )
|
||||
tool_dependency.status = app.install_model.ToolDependency.installation_status.ERROR
|
||||
tool_dependency.error_message = error_message
|
||||
sa_session.add( tool_dependency )
|
||||
sa_session.flush()
|
||||
return tool_dependency
|
||||
|
||||
|
||||
def parse_package_elem( package_elem, platform_info_dict=None, include_after_install_actions=True ):
|
||||
"""
|
||||
Parse a <package> element within a tool dependency definition and return a list of action tuples.
|
||||
@@ -405,19 +366,25 @@ def remove_tool_dependency_installation_directory( dependency_install_dir ):
|
||||
except Exception, e:
|
||||
removed = False
|
||||
error_message = "Error removing tool dependency installation directory %s: %s" % ( str( dependency_install_dir ), str( e ) )
|
||||
log.debug( error_message )
|
||||
log.warn( error_message )
|
||||
else:
|
||||
removed = True
|
||||
error_message = ''
|
||||
return removed, error_message
|
||||
|
||||
|
||||
def set_tool_dependency_attributes( app, tool_dependency, status, error_message=None, remove_from_disk=False ):
|
||||
def set_tool_dependency_attributes( app, tool_dependency, status, error_message=None ):
|
||||
sa_session = app.install_model.context
|
||||
if remove_from_disk:
|
||||
if status == app.install_model.ToolDependency.installation_status.UNINSTALLED:
|
||||
installation_directory = tool_dependency.installation_directory( app )
|
||||
removed, err_msg = remove_tool_dependency_installation_directory( installation_directory )
|
||||
remove_tool_dependency_installation_directory( installation_directory )
|
||||
tool_dependency.error_message = error_message
|
||||
if str( tool_dependency.status ) != str( status ):
|
||||
tool_shed_repository = tool_dependency.tool_shed_repository
|
||||
debug_msg = 'Updating an existing record for version %s of tool dependency %s for revision %s of repository %s ' % \
|
||||
( str( tool_dependency.version ), str( tool_dependency.name ), str( tool_shed_repository.changeset_revision ), str( tool_shed_repository.name ) )
|
||||
debug_msg += 'by updating the status from %s to %s.' % ( str( tool_dependency.status ), str( status ) )
|
||||
log.debug( debug_msg )
|
||||
tool_dependency.status = status
|
||||
sa_session.add( tool_dependency )
|
||||
sa_session.flush()
|
||||
|
||||
@@ -59,22 +59,22 @@ RUN mkdir /etc/galaxy && cd /tmp/ansible && mkdir roles && \
|
||||
apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
|
||||
|
||||
RUN cd $GALAXY_ROOT && \
|
||||
echo "Prepopuating postgres database" && \
|
||||
echo "Prepopulating postgres database" && \
|
||||
su -c '/usr/lib/postgresql/9.3/bin/pg_ctl -o "-F" start -D /opt/galaxy/db' postgres && \
|
||||
sleep 3 && \
|
||||
GALAXY_CONFIG_DATABASE_CONNECTION="postgresql://root@localhost:5930/galaxy" sh create_db.sh && \
|
||||
echo "Prepopuating sqlite database" && \
|
||||
echo "Prepopulating sqlite database" && \
|
||||
GALAXY_CONFIG_DATABASE_CONNECTION="sqlite:////opt/galaxy/galaxy.sqlite" sh create_db.sh && \
|
||||
sh /opt/galaxy/start_mysql.sh && \
|
||||
echo "Prepopuating mysql database" && \
|
||||
echo "Prepopulating mysql database" && \
|
||||
GALAXY_CONFIG_DATABASE_CONNECTION="mysql://galaxy:galaxy@localhost/galaxy?unix_socket=/var/run/mysqld/mysqld.sock" sh create_db.sh && \
|
||||
echo "Prepopulating toolshed postgres database" && \
|
||||
su -c '/usr/lib/postgresql/9.3/bin/pg_ctl -o "-F" start -D /opt/galaxy/db' postgres && \
|
||||
GALAXY_CONFIG_DATABASE_CONNECTION="postgresql://root@localhost:5930/toolshed" sh create_db.sh tool_shed && \
|
||||
echo "Prepopuating toolshed sqlite database" && \
|
||||
echo "Prepopulating toolshed sqlite database" && \
|
||||
GALAXY_CONFIG_DATABASE_CONNECTION="sqlite:////opt/galaxy/toolshed.sqlite" sh create_db.sh tool_shed && \
|
||||
sh /opt/galaxy/start_mysql.sh && \
|
||||
echo "Prepopuating toolshed mysql database" && \
|
||||
echo "Prepopulating toolshed mysql database" && \
|
||||
GALAXY_CONFIG_DATABASE_CONNECTION="mysql://galaxy:galaxy@localhost/toolshed?unix_socket=/var/run/mysqld/mysqld.sock" sh create_db.sh tool_shed
|
||||
|
||||
# bcftools for Galaxy.
|
||||
|
||||
@@ -60,7 +60,7 @@ class TestDataManagers( ShedTwillTestCase ):
|
||||
owner=common.test_user_1_name,
|
||||
category_id=self.security.encode_id( category.id ),
|
||||
strings_displayed=[] )
|
||||
# Upload the edited tool dependency definition to the package_lapack_3_4_1440 repository.
|
||||
# Upload the data manager files to the repository.
|
||||
self.upload_file( repository,
|
||||
filename=data_manager_tar_file,
|
||||
filepath=None,
|
||||
|
||||
Reference in New Issue
Block a user