diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 0d442ef62c0..068e8806429 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -503,7 +503,7 @@ class AdminToolshed( AdminGalaxy ): if err_msg: message += err_msg status = 'error' - message += "Installed tool dependencies: %s" % ','.join( td.name for td in installed_tool_dependencies ) + message += "Installed tool dependencies: %s" % ', '.join( td.name for td in installed_tool_dependencies ) td_ids = [ trans.security.encode_id( td.id ) for td in tool_shed_repository.tool_dependencies ] return trans.response.send_redirect( web.url_for( controller='admin_toolshed', action='manage_tool_dependencies', @@ -1919,8 +1919,8 @@ class AdminToolshed( AdminGalaxy ): if new_repository_dependencies == original_repository_dependencies: for new_repository_tup in new_repository_dependencies: # Make sure all dependencies are installed. - # TODO: Repository dependencies that are not installed should be displayed to to the user, - # giving them the option to install them or now. This is the same behavior as when initially + # TODO: Repository dependencies that are not installed should be displayed to the user, + # giving them the option to install them or not. This is the same behavior as when initially # installing and when re-installing. new_tool_shed, new_name, new_owner, new_changeset_revision, new_pir, new_oicct = \ common_util.parse_repository_dependency_tuple( new_repository_tup ) diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py index 63d5ac593d9..24245f76f88 100644 --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -126,10 +126,10 @@ class InstallToolDependencyManager( object ): # Since there was an installation error, update the tool dependency status to Error. The remove_installation_path option must # be left False here. error_message = '%s\n%s' % ( self.format_traceback(), str( e ) ) - tool_dependency = tool_dependency_util.handle_tool_dependency_installation_error( self.app, - tool_dependency, - error_message, - remove_installation_path=False ) + tool_dependency = tool_dependency_util.set_tool_dependency_attributes(self.app, + tool_dependency=tool_dependency, + status=self.app.install_model.ToolDependency.installation_status.ERROR, + error_message=error_message) tool_dependency = self.mark_tool_dependency_installed( tool_dependency ) return tool_dependency @@ -204,10 +204,10 @@ class InstallToolDependencyManager( object ): # Since there was an installation error, update the tool dependency status to Error. The # remove_installation_path option must be left False here. tool_dependency = \ - tool_dependency_util.handle_tool_dependency_installation_error( self.app, - tool_dependency, - error_message, - remove_installation_path=False ) + tool_dependency_util.set_tool_dependency_attributes(self.app, + tool_dependency=tool_dependency, + status=self.app.install_model.ToolDependency.installation_status.ERROR, + error_message=error_message) if tool_dependency and tool_dependency.status in [ self.install_model.ToolDependency.installation_status.INSTALLED, self.install_model.ToolDependency.installation_status.ERROR ]: installed_packages.append( tool_dependency ) @@ -422,9 +422,7 @@ class InstallToolDependencyManager( object ): status = self.install_model.ToolDependency.installation_status.INSTALLED tool_dependency = tool_dependency_util.set_tool_dependency_attributes( self.app, tool_dependency=tool_dependency, - status=status, - error_message=None, - remove_from_disk=False ) + status=status ) return tool_dependency diff --git a/lib/tool_shed/galaxy_install/repair_repository_manager.py b/lib/tool_shed/galaxy_install/repair_repository_manager.py index 56f65020a6a..f41be595739 100644 --- a/lib/tool_shed/galaxy_install/repair_repository_manager.py +++ b/lib/tool_shed/galaxy_install/repair_repository_manager.py @@ -204,9 +204,7 @@ class RepairRepositoryManager(): tool_dependency = \ tool_dependency_util.set_tool_dependency_attributes( self.app, tool_dependency=tool_dependency, - status=self.app.install_model.ToolDependency.installation_status.UNINSTALLED, - error_message=None, - remove_from_disk=True ) + status=self.app.install_model.ToolDependency.installation_status.UNINSTALLED ) # Install tool dependencies. irm.update_tool_shed_repository_status( repository, self.app.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py index c776fdbb67a..0118cde2605 100644 --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py @@ -19,7 +19,7 @@ from galaxy.util import unicodify from tool_shed.galaxy_install.tool_dependencies.recipe import asynchronous_reader -from tool_shed.util import basic_util +from tool_shed.util import basic_util, tool_dependency_util log = logging.getLogger( __name__ ) @@ -113,7 +113,6 @@ class InstallEnvironment( object ): def handle_command( self, tool_dependency, cmd, return_output=False, job_name="" ): """Handle a command and log the results.""" - context = self.app.install_model.context command = str( cmd ) output = self.handle_complex_command( command, job_name=job_name ) self.log_results( cmd, output, os.path.join( self.install_dir, basic_util.INSTALLATION_LOG ) ) @@ -126,17 +125,19 @@ class InstallEnvironment( object ): log.warn( "Length of stderr > %s, so only a portion will be saved in the database." % str( DATABASE_MAX_STRING_SIZE_PRETTY ) ) stderr = shrink_string_by_size( stderr, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True ) if output.return_code not in [ 0 ]: - tool_dependency.status = self.app.install_model.ToolDependency.installation_status.ERROR + status = self.app.install_model.ToolDependency.installation_status.ERROR if stderr: - tool_dependency.error_message = unicodify( stderr ) + error_message = unicodify( stderr ) elif stdout: - tool_dependency.error_message = unicodify( stdout ) + error_message = unicodify( stdout ) else: # We have a problem if there was no stdout and no stderr. - tool_dependency.error_message = "Unknown error occurred executing shell command %s, return_code: %s" % \ + error_message = "Unknown error occurred executing shell command %s, return_code: %s" % \ ( str( cmd ), str( output.return_code ) ) - context.add( tool_dependency ) - context.flush() + tool_dependency_util.set_tool_dependency_attributes(self.app, + tool_dependency=tool_dependency, + status=status, + error_message=error_message) if return_output: return output return output.return_code diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py index e4e3e3a65cb..591bce87fb3 100755 --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -284,8 +284,7 @@ class AssertDirectoryExecutable( RecipeStep ): tool_dependency = tool_dependency_util.set_tool_dependency_attributes( self.app, tool_dependency, status=status, - error_message=error_message, - remove_from_disk=False ) + error_message=error_message ) return tool_dependency, None, None def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): @@ -330,8 +329,7 @@ class AssertDirectoryExists( RecipeStep ): tool_dependency = tool_dependency_util.set_tool_dependency_attributes( self.app, tool_dependency, status=status, - error_message=error_message, - remove_from_disk=False ) + error_message=error_message ) return tool_dependency, None, None def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): @@ -380,8 +378,7 @@ class AssertFileExecutable( RecipeStep ): tool_dependency = tool_dependency_util.set_tool_dependency_attributes( self.app, tool_dependency, status=status, - error_message=error_message, - remove_from_disk=False ) + error_message=error_message ) return tool_dependency, None, None def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): @@ -427,8 +424,7 @@ class AssertFileExists( RecipeStep ): tool_dependency = tool_dependency_util.set_tool_dependency_attributes( self.app, tool_dependency, status=status, - error_message=error_message, - remove_from_disk=False ) + error_message=error_message ) return tool_dependency, None, None def prepare_step( self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download ): diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py index 4cc6104df63..f18588e152b 100644 --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py @@ -180,10 +180,10 @@ class Install( RecipeTag, SyncDatabase ): error_message += 'tag set.' # Since there was an installation error, update the tool dependency status to Error. # The remove_installation_path option must be left False here. - tool_dependency = tool_dependency_util.handle_tool_dependency_installation_error( self.app, - tool_dependency, - error_message, - remove_installation_path=False ) + tool_dependency = tool_dependency_util.set_tool_dependency_attributes(self.app, + tool_dependency=tool_dependency, + status=self.app.install_model.ToolDependency.installation_status.ERROR, + error_message=error_message) else: raise NotImplementedError( 'Only install version 1.0 is currently supported (i.e., change your tag to be ).' ) return tool_dependency, proceed_with_install, actions_elem_tuples @@ -213,9 +213,7 @@ class Package( RecipeTag ): tool_dependency = \ tool_dependency_util.set_tool_dependency_attributes( self.app, tool_dependency=tool_dependency, - status=self.app.install_model.ToolDependency.installation_status.ERROR, - error_message=None, - remove_from_disk=False ) + status=self.app.install_model.ToolDependency.installation_status.ERROR ) else: proceed_with_install = True return tool_dependency, proceed_with_install, action_elem_tuples @@ -329,18 +327,18 @@ class Repository( RecipeTag, SyncDatabase ): error_message = 'Error defining env.sh file for package %s, return_code: %s' % \ ( str( package_name ), str( return_code ) ) tool_dependency = \ - tool_dependency_util.handle_tool_dependency_installation_error( self.app, - tool_dependency, - error_message, - remove_installation_path=False ) + tool_dependency_util.set_tool_dependency_attributes(self.app, + tool_dependency=tool_dependency, + status=self.app.install_model.ToolDependency.installation_status.ERROR, + error_message=error_message) elif required_tool_dependency is not None and required_tool_dependency.in_error_state: error_message = "This tool dependency's required tool dependency %s version %s has status %s." % \ ( str( required_tool_dependency.name ), str( required_tool_dependency.version ), str( required_tool_dependency.status ) ) tool_dependency = \ - tool_dependency_util.handle_tool_dependency_installation_error( self.app, - tool_dependency, - error_message, - remove_installation_path=False ) + tool_dependency_util.set_tool_dependency_attributes(self.app, + tool_dependency=tool_dependency, + status=self.app.install_model.ToolDependency.installation_status.ERROR, + error_message=error_message) else: tool_dependency = \ tool_dependency_util.set_tool_dependency_attributes( self.app, @@ -556,18 +554,10 @@ class SetEnvironment( RecipeTag ): else: attr_tups_of_dependencies_for_install = [ ( td.name, td.version, td.type ) for td in tool_dependency_db_records ] try: - tool_dependencies = self.set_environment( package_elem, tool_shed_repository, attr_tups_of_dependencies_for_install ) + self.set_environment( package_elem, tool_shed_repository, attr_tups_of_dependencies_for_install ) except Exception, e: error_message = "Error setting environment for tool dependency: %s" % str( e ) log.debug( error_message ) - for tool_dependency in tool_dependencies: - if tool_dependency and tool_dependency.status == self.app.install_model.ToolDependency.installation_status.ERROR: - # Since there was an installation error, update the tool dependency status to Error. The - # remove_installation_path option must be left False here. - tool_dependency = tool_dependency_util.handle_tool_dependency_installation_error( self.app, - tool_dependency, - error_message, - remove_installation_path=False ) return tool_dependency, proceed_with_install, action_elem_tuples def set_environment( self, elem, tool_shed_repository, attr_tups_of_dependencies_for_install ): @@ -613,7 +603,7 @@ class SetEnvironment( RecipeTag ): # R_SCRIPT_PATH). env_var_action = env_var_elem.get( 'action', None ) if env_var_name and env_var_action: - # Tool dependencies of type "set_environmnet" always have the version attribute set to None. + # Tool dependencies of type "set_environment" always have the version attribute set to None. attr_tup = ( env_var_name, None, 'set_environment' ) if attr_tup in attr_tups_of_dependencies_for_install: install_dir = \ @@ -654,8 +644,7 @@ class SetEnvironment( RecipeTag ): tool_dependency_util.set_tool_dependency_attributes( self.app, tool_dependency=tool_dependency, status=status, - error_message=error_message, - remove_from_disk=False ) + error_message=error_message ) else: if tool_dependency.status not in [ self.app.install_model.ToolDependency.installation_status.ERROR, self.app.install_model.ToolDependency.installation_status.INSTALLED ]: @@ -663,9 +652,7 @@ class SetEnvironment( RecipeTag ): tool_dependency = \ tool_dependency_util.set_tool_dependency_attributes( self.app, tool_dependency=tool_dependency, - status=status, - error_message=None, - remove_from_disk=False ) + status=status ) log.debug( 'Environment variable %s set in %s for tool dependency %s.' % ( str( env_var_name ), str( install_dir ), str( tool_dependency.name ) ) ) else: @@ -675,7 +662,6 @@ class SetEnvironment( RecipeTag ): tool_dependency_util.set_tool_dependency_attributes( self.app, tool_dependency=tool_dependency, status=status, - error_message=error_message, - remove_from_disk=False ) + error_message=error_message ) tool_dependencies.append( tool_dependency ) return tool_dependencies diff --git a/lib/tool_shed/util/tool_dependency_util.py b/lib/tool_shed/util/tool_dependency_util.py index e41decf6dde..50a77d9de0e 100644 --- a/lib/tool_shed/util/tool_dependency_util.py +++ b/lib/tool_shed/util/tool_dependency_util.py @@ -48,14 +48,7 @@ def create_or_update_tool_dependency( app, tool_shed_repository, name, version, # In some cases we should not override the current status of an existing tool_dependency, so do so only # if set_status is True. if set_status: - if str( tool_dependency.status ) != str( status ): - debug_msg = 'Updating an existing record for version %s of tool dependency %s for revision %s of repository %s ' % \ - ( str( version ), str( name ), str( tool_shed_repository.changeset_revision ), str( tool_shed_repository.name ) ) - debug_msg += 'by updating the status from %s to %s.' % ( str( tool_dependency.status ), str( status ) ) - log.debug( debug_msg ) - tool_dependency.status = status - context.add( tool_dependency ) - context.flush() + set_tool_dependency_attributes(app, tool_dependency=tool_dependency, status=status) else: # Create a new tool_dependency record for the tool_shed_repository. debug_msg = 'Creating a new record for version %s of tool dependency %s for revision %s of repository %s. ' % \ @@ -227,38 +220,6 @@ def get_tool_dependency_install_dir( app, repository_name, repository_owner, rep repository_changeset_revision ) ) -def handle_tool_dependency_installation_error( app, tool_dependency, error_message, remove_installation_path=False ): - # Since there was an installation error, remove the installation directory because the install_package method uses - # this: "if os.path.exists( install_dir ):". Setting remove_installation_path to True should rarely occur. It is - # currently set to True only to handle issues with installing tool dependencies into an Amazon S3 bucket. - sa_session = app.install_model.context - tool_shed_repository = tool_dependency.tool_shed_repository - install_dir = get_tool_dependency_install_dir( app=app, - repository_name=tool_shed_repository.name, - repository_owner=tool_shed_repository.owner, - repository_changeset_revision=tool_shed_repository.installed_changeset_revision, - tool_dependency_type=tool_dependency.type, - tool_dependency_name=tool_dependency.name, - tool_dependency_version=tool_dependency.version ) - if remove_installation_path: - # This will be True only in the case where an exception was encountered during the installation process after - # the installation path was created but before any information was written to the installation log and the - # tool dependency status was not set to "Installed" or "Error". - if os.path.exists( install_dir ): - log.debug( 'Attempting to remove installation directory %s for version %s of tool dependency %s %s' % - ( str( install_dir ), str( tool_dependency.version ), str( tool_dependency.type ), str( tool_dependency.name ) ) ) - log.debug( 'due to the following installation error:\n%s' % str( error_message ) ) - try: - shutil.rmtree( install_dir ) - except Exception, e: - log.exception( 'Error removing existing installation directory %s: %s', (install_dir, str(e)) ) - tool_dependency.status = app.install_model.ToolDependency.installation_status.ERROR - tool_dependency.error_message = error_message - sa_session.add( tool_dependency ) - sa_session.flush() - return tool_dependency - - def parse_package_elem( package_elem, platform_info_dict=None, include_after_install_actions=True ): """ Parse a element within a tool dependency definition and return a list of action tuples. @@ -405,19 +366,25 @@ def remove_tool_dependency_installation_directory( dependency_install_dir ): except Exception, e: removed = False error_message = "Error removing tool dependency installation directory %s: %s" % ( str( dependency_install_dir ), str( e ) ) - log.debug( error_message ) + log.warn( error_message ) else: removed = True error_message = '' return removed, error_message -def set_tool_dependency_attributes( app, tool_dependency, status, error_message=None, remove_from_disk=False ): +def set_tool_dependency_attributes( app, tool_dependency, status, error_message=None ): sa_session = app.install_model.context - if remove_from_disk: + if status == app.install_model.ToolDependency.installation_status.UNINSTALLED: installation_directory = tool_dependency.installation_directory( app ) - removed, err_msg = remove_tool_dependency_installation_directory( installation_directory ) + remove_tool_dependency_installation_directory( installation_directory ) tool_dependency.error_message = error_message + if str( tool_dependency.status ) != str( status ): + tool_shed_repository = tool_dependency.tool_shed_repository + debug_msg = 'Updating an existing record for version %s of tool dependency %s for revision %s of repository %s ' % \ + ( str( tool_dependency.version ), str( tool_dependency.name ), str( tool_shed_repository.changeset_revision ), str( tool_shed_repository.name ) ) + debug_msg += 'by updating the status from %s to %s.' % ( str( tool_dependency.status ), str( status ) ) + log.debug( debug_msg ) tool_dependency.status = status sa_session.add( tool_dependency ) sa_session.flush() diff --git a/test/docker/base/Dockerfile b/test/docker/base/Dockerfile index 77099f8340f..0e9d65c48bd 100644 --- a/test/docker/base/Dockerfile +++ b/test/docker/base/Dockerfile @@ -59,22 +59,22 @@ RUN mkdir /etc/galaxy && cd /tmp/ansible && mkdir roles && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* RUN cd $GALAXY_ROOT && \ - echo "Prepopuating postgres database" && \ + echo "Prepopulating postgres database" && \ su -c '/usr/lib/postgresql/9.3/bin/pg_ctl -o "-F" start -D /opt/galaxy/db' postgres && \ sleep 3 && \ GALAXY_CONFIG_DATABASE_CONNECTION="postgresql://root@localhost:5930/galaxy" sh create_db.sh && \ - echo "Prepopuating sqlite database" && \ + echo "Prepopulating sqlite database" && \ GALAXY_CONFIG_DATABASE_CONNECTION="sqlite:////opt/galaxy/galaxy.sqlite" sh create_db.sh && \ sh /opt/galaxy/start_mysql.sh && \ - echo "Prepopuating mysql database" && \ + echo "Prepopulating mysql database" && \ GALAXY_CONFIG_DATABASE_CONNECTION="mysql://galaxy:galaxy@localhost/galaxy?unix_socket=/var/run/mysqld/mysqld.sock" sh create_db.sh && \ echo "Prepopulating toolshed postgres database" && \ su -c '/usr/lib/postgresql/9.3/bin/pg_ctl -o "-F" start -D /opt/galaxy/db' postgres && \ GALAXY_CONFIG_DATABASE_CONNECTION="postgresql://root@localhost:5930/toolshed" sh create_db.sh tool_shed && \ - echo "Prepopuating toolshed sqlite database" && \ + echo "Prepopulating toolshed sqlite database" && \ GALAXY_CONFIG_DATABASE_CONNECTION="sqlite:////opt/galaxy/toolshed.sqlite" sh create_db.sh tool_shed && \ sh /opt/galaxy/start_mysql.sh && \ - echo "Prepopuating toolshed mysql database" && \ + echo "Prepopulating toolshed mysql database" && \ GALAXY_CONFIG_DATABASE_CONNECTION="mysql://galaxy:galaxy@localhost/toolshed?unix_socket=/var/run/mysqld/mysqld.sock" sh create_db.sh tool_shed # bcftools for Galaxy. diff --git a/test/tool_shed/functional/test_1460_data_managers.py b/test/tool_shed/functional/test_1460_data_managers.py index bccb77f8de2..6b1ebff8367 100644 --- a/test/tool_shed/functional/test_1460_data_managers.py +++ b/test/tool_shed/functional/test_1460_data_managers.py @@ -60,7 +60,7 @@ class TestDataManagers( ShedTwillTestCase ): owner=common.test_user_1_name, category_id=self.security.encode_id( category.id ), strings_displayed=[] ) - # Upload the edited tool dependency definition to the package_lapack_3_4_1440 repository. + # Upload the data manager files to the repository. self.upload_file( repository, filename=data_manager_tar_file, filepath=None,