opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-10-25 10:24:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add a DependencyDisplayer class for Galaxy installs from the Tool Shed.

Browse Source
This commit is contained in:
Greg Von Kuster
2014-07-16 14:30:36 -04:00
parent a44672251b
commit ecece2807f
9 changed files with 895 additions and 808 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py
+61 -53
View File
@@ -3,16 +3,12 @@ import os
import shutil
from admin import AdminGalaxy
from galaxy import eggs
from galaxy import web
from galaxy import util
from galaxy.web.form_builder import CheckboxField
from galaxy.web.framework.helpers import grids
from galaxy.web.framework.helpers import iff
from galaxy.util import json
from galaxy.model.orm import or_
import tool_shed.util.shed_util_common as suc
import tool_shed.repository_types.util as rt_util
from tool_shed.util import common_util
@@ -23,11 +19,14 @@ from tool_shed.util import hg_util
from tool_shed.util import metadata_util
from tool_shed.util import readme_util
from tool_shed.util import repository_maintenance_util
from tool_shed.util import shed_util_common as suc
from tool_shed.util import tool_dependency_util
from tool_shed.util import tool_util
from tool_shed.util import workflow_util
from tool_shed.util import xml_util
from tool_shed.galaxy_install import dependency_display
from tool_shed.galaxy_install import install_manager
from tool_shed.galaxy_install.repair_repository_manager import RepairRepositoryManager
import tool_shed.galaxy_install.grids.admin_toolshed_grids as admin_toolshed_grids
from tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager
@@ -221,17 +220,20 @@ class AdminToolshed( AdminGalaxy ):
@web.require_admin
def deactivate_or_uninstall_repository( self, trans, **kwd ):
"""
Handle all changes when a tool shed repository is being deactivated or uninstalled. Notice that if the repository contents include
a file named tool_data_table_conf.xml.sample, its entries are not removed from the defined config.shed_tool_data_table_config. This
is because it becomes a bit complex to determine if other installed repositories include tools that require the same entry. For now
we'll never delete entries from config.shed_tool_data_table_config, but we may choose to do so in the future if it becomes necessary.
Handle all changes when a tool shed repository is being deactivated or uninstalled. Notice
that if the repository contents include a file named tool_data_table_conf.xml.sample, its
entries are not removed from the defined config.shed_tool_data_table_config. This is because
it becomes a bit complex to determine if other installed repositories include tools that
require the same entry. For now we'll never delete entries from config.shed_tool_data_table_config,
but we may choose to do so in the future if it becomes necessary.
"""
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
remove_from_disk = kwd.get( 'remove_from_disk', '' )
remove_from_disk_checked = CheckboxField.is_checked( remove_from_disk )
tool_shed_repository = suc.get_installed_tool_shed_repository( trans.app, kwd[ 'id' ] )
shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, tool_shed_repository )
shed_tool_conf, tool_path, relative_install_dir = \
suc.get_tool_panel_config_tool_path_install_dir( trans.app, tool_shed_repository )
if relative_install_dir:
if tool_path:
relative_install_dir = os.path.join( tool_path, relative_install_dir )
@@ -247,7 +249,10 @@ class AdminToolshed( AdminGalaxy ):
data_manager_util.remove_from_data_manager( trans.app, tool_shed_repository )
if tool_shed_repository.includes_datatypes:
# Deactivate proprietary datatypes.
installed_repository_dict = datatype_util.load_installed_datatypes( trans.app, tool_shed_repository, repository_install_dir, deactivate=True )
installed_repository_dict = datatype_util.load_installed_datatypes( trans.app,
tool_shed_repository,
repository_install_dir,
deactivate=True )
if installed_repository_dict:
converter_path = installed_repository_dict.get( 'converter_path' )
if converter_path is not None:
@@ -779,12 +784,12 @@ class AdminToolshed( AdminGalaxy ):
trans.install_model.context.add( repository )
trans.install_model.context.flush()
message = "The repository information has been updated."
containers_dict = metadata_util.populate_containers_dict_from_repository_metadata( app=trans.app,
tool_shed_url=tool_shed_url,
tool_path=tool_path,
repository=repository,
reinstalling=False,
required_repo_info_dicts=None )
dd = dependency_display.DependencyDisplayer( trans.app )
containers_dict = dd.populate_containers_dict_from_repository_metadata( tool_shed_url=tool_shed_url,
tool_path=tool_path,
repository=repository,
reinstalling=False,
required_repo_info_dicts=None )
return trans.fill_template( '/admin/tool_shed_repository/manage_repository.mako',
repository=repository,
description=description,
@@ -1049,6 +1054,7 @@ class AdminToolshed( AdminGalaxy ):
includes_tool_dependencies = util.string_as_bool( repo_information_dict.get( 'includes_tool_dependencies', False ) )
encoded_repo_info_dicts = util.listify( repo_information_dict.get( 'repo_info_dicts', [] ) )
repo_info_dicts = [ encoding_util.tool_shed_decode( encoded_repo_info_dict ) for encoded_repo_info_dict in encoded_repo_info_dicts ]
dd = dependency_display.DependencyDisplayer( trans.app )
install_repository_manager = install_manager.InstallRepositoryManager( trans.app )
if ( not includes_tools_for_display_in_tool_panel and kwd.get( 'select_shed_tool_panel_config_button', False ) ) or \
( includes_tools_for_display_in_tool_panel and kwd.get( 'select_tool_panel_section_button', False ) ):
@@ -1149,14 +1155,14 @@ class AdminToolshed( AdminGalaxy ):
# defined repository (and possibly tool) dependencies. In this case, merging will result in newly defined
# dependencies to be lost. We pass the updating parameter to make sure merging occurs only when appropriate.
containers_dict = \
install_repository_manager.populate_containers_dict_for_new_install( tool_shed_url=tool_shed_url,
tool_path=tool_path,
readme_files_dict=readme_files_dict,
installed_repository_dependencies=installed_repository_dependencies,
missing_repository_dependencies=missing_repository_dependencies,
installed_tool_dependencies=installed_tool_dependencies,
missing_tool_dependencies=missing_tool_dependencies,
updating=updating )
dd.populate_containers_dict_for_new_install( tool_shed_url=tool_shed_url,
tool_path=tool_path,
readme_files_dict=readme_files_dict,
installed_repository_dependencies=installed_repository_dependencies,
missing_repository_dependencies=missing_repository_dependencies,
installed_tool_dependencies=installed_tool_dependencies,
missing_tool_dependencies=missing_tool_dependencies,
updating=updating )
else:
# We're installing a list of repositories, each of which may have tool dependencies or repository dependencies.
containers_dicts = []
@@ -1184,17 +1190,17 @@ class AdminToolshed( AdminGalaxy ):
name = dependencies_for_repository_dict.get( 'name', None )
repository_owner = dependencies_for_repository_dict.get( 'repository_owner', None )
containers_dict = \
install_repository_manager.populate_containers_dict_for_new_install( tool_shed_url=tool_shed_url,
tool_path=tool_path,
readme_files_dict=None,
installed_repository_dependencies=installed_repository_dependencies,
missing_repository_dependencies=missing_repository_dependencies,
installed_tool_dependencies=installed_tool_dependencies,
missing_tool_dependencies=missing_tool_dependencies,
updating=updating )
dd.populate_containers_dict_for_new_install( tool_shed_url=tool_shed_url,
tool_path=tool_path,
readme_files_dict=None,
installed_repository_dependencies=installed_repository_dependencies,
missing_repository_dependencies=missing_repository_dependencies,
installed_tool_dependencies=installed_tool_dependencies,
missing_tool_dependencies=missing_tool_dependencies,
updating=updating )
containers_dicts.append( containers_dict )
# Merge all containers into a single container.
containers_dict = install_repository_manager.merge_containers_dicts_for_new_install( containers_dicts )
containers_dict = dd.merge_containers_dicts_for_new_install( containers_dicts )
# Handle tool dependencies check box.
if trans.app.config.tool_dependency_dir is None:
if includes_tool_dependencies:
@@ -1647,19 +1653,20 @@ class AdminToolshed( AdminGalaxy ):
original_section_name = ''
tool_panel_section_select_field = None
shed_tool_conf_select_field = tool_util.build_shed_tool_conf_select_field( trans )
irm = install_manager.InstallRepositoryManager( trans.app )
dd = dependency_display.DependencyDisplayer( trans.app )
containers_dict = \
irm.populate_containers_dict_for_new_install( tool_shed_url=tool_shed_url,
tool_path=tool_path,
readme_files_dict=readme_files_dict,
installed_repository_dependencies=installed_repository_dependencies,
missing_repository_dependencies=missing_repository_dependencies,
installed_tool_dependencies=installed_tool_dependencies,
missing_tool_dependencies=missing_tool_dependencies,
updating=False )
# Since we're reinstalling we'll merge the list of missing repository dependencies into the list of installed repository dependencies since each displayed
# repository dependency will display a status, whether installed or missing.
containers_dict = irm.merge_missing_repository_dependencies_to_installed_container( containers_dict )
dd.populate_containers_dict_for_new_install( tool_shed_url=tool_shed_url,
tool_path=tool_path,
readme_files_dict=readme_files_dict,
installed_repository_dependencies=installed_repository_dependencies,
missing_repository_dependencies=missing_repository_dependencies,
installed_tool_dependencies=installed_tool_dependencies,
missing_tool_dependencies=missing_tool_dependencies,
updating=False )
# Since we're reinstalling we'll merge the list of missing repository dependencies into the list of
# installed repository dependencies since each displayed repository dependency will display a status,
# whether installed or missing.
containers_dict = dd.merge_missing_repository_dependencies_to_installed_container( containers_dict )
# Handle repository dependencies check box.
install_repository_dependencies_check_box = CheckboxField( 'install_repository_dependencies', checked=True )
# Handle tool dependencies check box.
@@ -1796,12 +1803,12 @@ class AdminToolshed( AdminGalaxy ):
status = 'error'
shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository )
repo_files_dir = os.path.abspath( os.path.join( relative_install_dir, repository.name ) )
containers_dict = metadata_util.populate_containers_dict_from_repository_metadata( app=trans.app,
tool_shed_url=tool_shed_url,
tool_path=tool_path,
repository=repository,
reinstalling=False,
required_repo_info_dicts=None )
dd = dependency_display.DependencyDisplayer( trans.app )
containers_dict = dd.populate_containers_dict_from_repository_metadata( tool_shed_url=tool_shed_url,
tool_path=tool_path,
repository=repository,
reinstalling=False,
required_repo_info_dicts=None )
return trans.fill_template( '/admin/tool_shed_repository/manage_repository.mako',
repository=repository,
description=repository.description,
@@ -1860,7 +1867,8 @@ class AdminToolshed( AdminGalaxy ):
message = "Error attempting to uninstall tool dependencies: %s" % message
status = 'error'
else:
message = "These tool dependencies have been uninstalled: %s" % ','.join( td.name for td in tool_dependencies_for_uninstallation )
message = "These tool dependencies have been uninstalled: %s" % \
','.join( td.name for td in tool_dependencies_for_uninstallation )
td_ids = [ trans.security.encode_id( td.id ) for td in tool_shed_repository.tool_dependencies ]
return trans.response.send_redirect( web.url_for( controller='admin_toolshed',
action='manage_repository_tool_dependencies',
lib/galaxy/webapps/tool_shed/controllers/repository.py
+12 -9
View File
@@ -6,16 +6,20 @@ from time import gmtime
from time import strftime
from datetime import date
from datetime import datetime
from galaxy import util
from galaxy import web
from galaxy.util.odict import odict
from galaxy.web.base.controller import BaseUIController
from galaxy.web.form_builder import CheckboxField
from galaxy.web.framework.helpers import grids
from galaxy.util import json
from galaxy.model.orm import and_
from tool_shed.capsule import capsule_manager
from tool_shed.dependencies.repository import relation_builder
from tool_shed.galaxy_install import dependency_display
from tool_shed.util import basic_util
from tool_shed.util import common_util
from tool_shed.util import container_util
@@ -24,17 +28,16 @@ from tool_shed.util import hg_util
from tool_shed.util import metadata_util
from tool_shed.util import readme_util
from tool_shed.util import repository_maintenance_util
from tool_shed.util import review_util
from tool_shed.util import search_util
from tool_shed.util import shed_util_common as suc
from tool_shed.util import tool_dependency_util
from tool_shed.util import tool_util
from tool_shed.util import workflow_util
from tool_shed.galaxy_install import install_manager
from galaxy.webapps.tool_shed.util import ratings_util
import galaxy.tools
import tool_shed.grids.repository_grids as repository_grids
import tool_shed.grids.util as grids_util
import tool_shed.repository_types.util as rt_util
from galaxy import eggs
@@ -2421,9 +2424,8 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
status = 'warning'
else:
# Handle messaging for orphan tool dependency definitions.
orphan_message = tool_dependency_util.generate_message_for_orphan_tool_dependencies( trans,
repository,
metadata )
dd = dependency_display.DependencyDisplayer( trans.app )
orphan_message = dd.generate_message_for_orphan_tool_dependencies( repository, metadata )
if orphan_message:
message += orphan_message
status = 'warning'
@@ -3313,7 +3315,8 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
repository_dependencies = rb.get_repository_dependencies_for_changeset_revision()
if str( repository.type ) != rt_util.TOOL_DEPENDENCY_DEFINITION:
# Handle messaging for orphan tool dependency definitions.
orphan_message = tool_dependency_util.generate_message_for_orphan_tool_dependencies( trans, repository, metadata )
dd = dependency_display.DependencyDisplayer( trans.app )
orphan_message = dd.generate_message_for_orphan_tool_dependencies( repository, metadata )
if orphan_message:
message += orphan_message
status = 'warning'
lib/galaxy/webapps/tool_shed/controllers/upload.py
+8 -7
View File
@@ -4,18 +4,21 @@ import shutil
import tarfile
import tempfile
import urllib
from galaxy.web.base.controller import BaseUIController
from galaxy import util
from galaxy import web
from galaxy.datatypes import checkers
from galaxy.web.base.controller import BaseUIController
from tool_shed.dependencies import attribute_handlers
from tool_shed.galaxy_install import dependency_display
import tool_shed.repository_types.util as rt_util
from tool_shed.util import basic_util
from tool_shed.util import commit_util
from tool_shed.util import hg_util
from tool_shed.util import metadata_util
from tool_shed.util import shed_util_common as suc
from tool_shed.util import tool_dependency_util
from tool_shed.util import tool_util
from tool_shed.util import xml_util
@@ -274,6 +277,7 @@ class UploadController( BaseUIController ):
metadata_dict = repository.metadata_revisions[ 0 ].metadata
else:
metadata_dict = {}
dd = dependency_display.DependencyDisplayer( trans.app )
if str( repository.type ) not in [ rt_util.REPOSITORY_SUITE_DEFINITION,
rt_util.TOOL_DEPENDENCY_DEFINITION ]:
change_repository_type_message = rt_util.generate_message_for_repository_type_change( trans.app,
@@ -288,15 +292,12 @@ class UploadController( BaseUIController ):
# repository), so warning messages are important because orphans are always valid. The repository
# owner must be warned in case they did not intend to define an orphan dependency, but simply
# provided incorrect information (tool shed, name owner, changeset_revision) for the definition.
orphan_message = tool_dependency_util.generate_message_for_orphan_tool_dependencies( trans,
repository,
metadata_dict )
orphan_message = dd.generate_message_for_orphan_tool_dependencies( repository, metadata_dict )
if orphan_message:
message += orphan_message
status = 'warning'
# Handle messaging for invalid tool dependencies.
invalid_tool_dependencies_message = \
tool_dependency_util.generate_message_for_invalid_tool_dependencies( metadata_dict )
invalid_tool_dependencies_message = dd.generate_message_for_invalid_tool_dependencies( metadata_dict )
if invalid_tool_dependencies_message:
message += invalid_tool_dependencies_message
status = 'error'
lib/tool_shed/galaxy_install/dependency_display.py
+575
View File
@@ -0,0 +1,575 @@
import logging
import os
import threading
from tool_shed.util import common_util
from tool_shed.util import container_util
from tool_shed.util import readme_util
from tool_shed.util import shed_util_common as suc
from tool_shed.util import tool_dependency_util
log = logging.getLogger( __name__ )
class DependencyDisplayer( object ):
def __init__( self, app ):
self.app = app
def add_installation_directories_to_tool_dependencies( self, tool_dependencies ):
"""
Determine the path to the installation directory for each of the received
tool dependencies. This path will be displayed within the tool dependencies
container on the select_tool_panel_section or reselect_tool_panel_section
pages when installing or reinstalling repositories that contain tools with
the defined tool dependencies. The list of tool dependencies may be associated
with more than a single repository.
"""
for dependency_key, requirements_dict in tool_dependencies.items():
if dependency_key in [ 'set_environment' ]:
continue
repository_name = requirements_dict.get( 'repository_name', 'unknown' )
repository_owner = requirements_dict.get( 'repository_owner', 'unknown' )
changeset_revision = requirements_dict.get( 'changeset_revision', 'unknown' )
dependency_name = requirements_dict[ 'name' ]
version = requirements_dict[ 'version' ]
type = requirements_dict[ 'type' ]
if self.app.config.tool_dependency_dir:
root_dir = self.app.config.tool_dependency_dir
else:
root_dir = '<set your tool_dependency_dir in your Galaxy configuration file>'
install_dir = os.path.join( root_dir,
dependency_name,
version,
repository_owner,
repository_name,
changeset_revision )
requirements_dict[ 'install_dir' ] = install_dir
tool_dependencies[ dependency_key ] = requirements_dict
return tool_dependencies
def generate_message_for_invalid_tool_dependencies( self, metadata_dict ):
"""
Tool dependency definitions can only be invalid if they include a definition for a complex
repository dependency and the repository dependency definition is invalid. This method
retrieves the error message associated with the invalid tool dependency for display in the
caller.
"""
message = ''
if metadata_dict:
invalid_tool_dependencies = metadata_dict.get( 'invalid_tool_dependencies', None )
if invalid_tool_dependencies:
for td_key, requirement_dict in invalid_tool_dependencies.items():
error = requirement_dict.get( 'error', None )
if error:
message = '%s ' % str( error )
return message
def generate_message_for_orphan_tool_dependencies( self, repository, metadata_dict ):
"""
The designation of a ToolDependency into the "orphan" category has evolved over time,
and is significantly restricted since the introduction of the TOOL_DEPENDENCY_DEFINITION
repository type. This designation is still critical, however, in that it handles the
case where a repository contains both tools and a tool_dependencies.xml file, but the
definition in the tool_dependencies.xml file is in no way related to anything defined
by any of the contained tool's requirements tag sets. This is important in that it is
often a result of a typo (e.g., dependency name or version) that differs between the tool
dependency definition within the tool_dependencies.xml file and what is defined in the
tool config's <requirements> tag sets. In these cases, the user should be presented with
a warning message, and this warning message is is in fact displayed if the following
is_orphan attribute is True. This is tricky because in some cases it may be intentional,
and tool dependencies that are categorized as "orphan" are in fact valid.
"""
has_orphan_package_dependencies = False
has_orphan_set_environment_dependencies = False
message = ''
package_orphans_str = ''
set_environment_orphans_str = ''
# Tool dependencies are categorized as orphan only if the repository contains tools.
if metadata_dict:
tools = metadata_dict.get( 'tools', [] )
invalid_tools = metadata_dict.get( 'invalid_tools', [] )
tool_dependencies = metadata_dict.get( 'tool_dependencies', {} )
# The use of the orphan_tool_dependencies category in metadata has been deprecated,
# but we still need to check in case the metadata is out of date.
orphan_tool_dependencies = metadata_dict.get( 'orphan_tool_dependencies', {} )
# Updating should cause no problems here since a tool dependency cannot be included
# in both dictionaries.
tool_dependencies.update( orphan_tool_dependencies )
if tool_dependencies and ( tools or invalid_tools ):
for td_key, requirements_dict in tool_dependencies.items():
if td_key == 'set_environment':
# "set_environment": [{"name": "R_SCRIPT_PATH", "type": "set_environment"}]
for env_requirements_dict in requirements_dict:
name = env_requirements_dict[ 'name' ]
type = env_requirements_dict[ 'type' ]
if self.tool_dependency_is_orphan( type, name, None, tools ):
if not has_orphan_set_environment_dependencies:
has_orphan_set_environment_dependencies = True
set_environment_orphans_str += "<b>* name:</b> %s, <b>type:</b> %s<br/>" % \
( str( name ), str( type ) )
else:
# "R/2.15.1": {"name": "R", "readme": "some string", "type": "package", "version": "2.15.1"}
name = requirements_dict[ 'name' ]
type = requirements_dict[ 'type' ]
version = requirements_dict[ 'version' ]
if self.tool_dependency_is_orphan( type, name, version, tools ):
if not has_orphan_package_dependencies:
has_orphan_package_dependencies = True
package_orphans_str += "<b>* name:</b> %s, <b>type:</b> %s, <b>version:</b> %s<br/>" % \
( str( name ), str( type ), str( version ) )
if has_orphan_package_dependencies:
message += "The settings for <b>name</b>, <b>version</b> and <b>type</b> from a "
message += "contained tool configuration file's <b>requirement</b> tag does not match "
message += "the information for the following tool dependency definitions in the "
message += "<b>tool_dependencies.xml</b> file, so these tool dependencies have no "
message += "relationship with any tools within this repository.<br/>"
message += package_orphans_str
if has_orphan_set_environment_dependencies:
message += "The settings for <b>name</b> and <b>type</b> from a contained tool "
message += "configuration file's <b>requirement</b> tag does not match the information "
message += "for the following tool dependency definitions in the <b>tool_dependencies.xml</b> "
message += "file, so these tool dependencies have no relationship with any tools within "
message += "this repository.<br/>"
message += set_environment_orphans_str
return message
def get_installed_and_missing_tool_dependencies_for_installed_repository( self, repository, all_tool_dependencies ):
"""
Return the lists of installed tool dependencies and missing tool dependencies for a Tool Shed
repository that has been installed into Galaxy.
"""
if all_tool_dependencies:
tool_dependencies = {}
missing_tool_dependencies = {}
for td_key, val in all_tool_dependencies.items():
if td_key in [ 'set_environment' ]:
for index, td_info_dict in enumerate( val ):
name = td_info_dict[ 'name' ]
version = None
type = td_info_dict[ 'type' ]
tool_dependency = tool_dependency_util.get_tool_dependency_by_name_type_repository( self.app,
repository,
name,
type )
if tool_dependency:
td_info_dict[ 'repository_id' ] = repository.id
td_info_dict[ 'tool_dependency_id' ] = tool_dependency.id
if tool_dependency.status:
tool_dependency_status = str( tool_dependency.status )
else:
tool_dependency_status = 'Never installed'
td_info_dict[ 'status' ] = tool_dependency_status
val[ index ] = td_info_dict
if tool_dependency.status == self.app.install_model.ToolDependency.installation_status.INSTALLED:
tool_dependencies[ td_key ] = val
else:
missing_tool_dependencies[ td_key ] = val
else:
name = val[ 'name' ]
version = val[ 'version' ]
type = val[ 'type' ]
tool_dependency = tool_dependency_util.get_tool_dependency_by_name_version_type_repository( self.app,
repository,
name,
version,
type )
if tool_dependency:
val[ 'repository_id' ] = repository.id
val[ 'tool_dependency_id' ] = tool_dependency.id
if tool_dependency.status:
tool_dependency_status = str( tool_dependency.status )
else:
tool_dependency_status = 'Never installed'
val[ 'status' ] = tool_dependency_status
if tool_dependency.status == self.app.install_model.ToolDependency.installation_status.INSTALLED:
tool_dependencies[ td_key ] = val
else:
missing_tool_dependencies[ td_key ] = val
else:
tool_dependencies = None
missing_tool_dependencies = None
return tool_dependencies, missing_tool_dependencies
def merge_containers_dicts_for_new_install( self, containers_dicts ):
"""
When installing one or more tool shed repositories for the first time, the received list of
containers_dicts contains a containers_dict for each repository being installed. Since the
repositories are being installed for the first time, all entries are None except the repository
dependencies and tool dependencies. The entries for missing dependencies are all None since
they have previously been merged into the installed dependencies. This method will merge the
dependencies entries into a single container and return it for display.
"""
new_containers_dict = dict( readme_files=None,
datatypes=None,
missing_repository_dependencies=None,
repository_dependencies=None,
missing_tool_dependencies=None,
tool_dependencies=None,
invalid_tools=None,
valid_tools=None,
workflows=None )
if containers_dicts:
lock = threading.Lock()
lock.acquire( True )
try:
repository_dependencies_root_folder = None
tool_dependencies_root_folder = None
# Use a unique folder id (hopefully the following is).
folder_id = 867
for old_container_dict in containers_dicts:
# Merge repository_dependencies.
old_container_repository_dependencies_root = old_container_dict[ 'repository_dependencies' ]
if old_container_repository_dependencies_root:
if repository_dependencies_root_folder is None:
repository_dependencies_root_folder = container_util.Folder( id=folder_id,
key='root',
label='root',
parent=None )
folder_id += 1
repository_dependencies_folder = container_util.Folder( id=folder_id,
key='merged',
label='Repository dependencies',
parent=repository_dependencies_root_folder )
folder_id += 1
# The old_container_repository_dependencies_root will be a root folder containing a single sub_folder.
old_container_repository_dependencies_folder = old_container_repository_dependencies_root.folders[ 0 ]
# Change the folder id so it won't confict with others being merged.
old_container_repository_dependencies_folder.id = folder_id
folder_id += 1
repository_components_tuple = container_util.get_components_from_key( old_container_repository_dependencies_folder.key )
components_list = suc.extract_components_from_tuple( repository_components_tuple )
name = components_list[ 1 ]
# Generate the label by retrieving the repository name.
old_container_repository_dependencies_folder.label = str( name )
repository_dependencies_folder.folders.append( old_container_repository_dependencies_folder )
# Merge tool_dependencies.
old_container_tool_dependencies_root = old_container_dict[ 'tool_dependencies' ]
if old_container_tool_dependencies_root:
if tool_dependencies_root_folder is None:
tool_dependencies_root_folder = container_util.Folder( id=folder_id,
key='root',
label='root',
parent=None )
folder_id += 1
tool_dependencies_folder = container_util.Folder( id=folder_id,
key='merged',
label='Tool dependencies',
parent=tool_dependencies_root_folder )
folder_id += 1
else:
td_list = [ td.listify for td in tool_dependencies_folder.tool_dependencies ]
# The old_container_tool_dependencies_root will be a root folder containing a single sub_folder.
old_container_tool_dependencies_folder = old_container_tool_dependencies_root.folders[ 0 ]
for td in old_container_tool_dependencies_folder.tool_dependencies:
if td.listify not in td_list:
tool_dependencies_folder.tool_dependencies.append( td )
if repository_dependencies_root_folder:
repository_dependencies_root_folder.folders.append( repository_dependencies_folder )
new_containers_dict[ 'repository_dependencies' ] = repository_dependencies_root_folder
if tool_dependencies_root_folder:
tool_dependencies_root_folder.folders.append( tool_dependencies_folder )
new_containers_dict[ 'tool_dependencies' ] = tool_dependencies_root_folder
except Exception, e:
log.debug( "Exception in merge_containers_dicts_for_new_install: %s" % str( e ) )
finally:
lock.release()
return new_containers_dict
def merge_missing_repository_dependencies_to_installed_container( self, containers_dict ):
"""
Merge the list of missing repository dependencies into the list of installed
repository dependencies.
"""
missing_rd_container_root = containers_dict.get( 'missing_repository_dependencies', None )
if missing_rd_container_root:
# The missing_rd_container_root will be a root folder containing a single sub_folder.
missing_rd_container = missing_rd_container_root.folders[ 0 ]
installed_rd_container_root = containers_dict.get( 'repository_dependencies', None )
# The installed_rd_container_root will be a root folder containing a single sub_folder.
if installed_rd_container_root:
installed_rd_container = installed_rd_container_root.folders[ 0 ]
installed_rd_container.label = 'Repository dependencies'
for index, rd in enumerate( missing_rd_container.repository_dependencies ):
# Skip the header row.
if index == 0:
continue
installed_rd_container.repository_dependencies.append( rd )
installed_rd_container_root.folders = [ installed_rd_container ]
containers_dict[ 'repository_dependencies' ] = installed_rd_container_root
else:
# Change the folder label from 'Missing repository dependencies' to be
# 'Repository dependencies' for display.
root_container = containers_dict[ 'missing_repository_dependencies' ]
for sub_container in root_container.folders:
# There should only be 1 sub-folder.
sub_container.label = 'Repository dependencies'
containers_dict[ 'repository_dependencies' ] = root_container
containers_dict[ 'missing_repository_dependencies' ] = None
return containers_dict
def merge_missing_tool_dependencies_to_installed_container( self, containers_dict ):
"""
Merge the list of missing tool dependencies into the list of installed tool
dependencies.
"""
missing_td_container_root = containers_dict.get( 'missing_tool_dependencies', None )
if missing_td_container_root:
# The missing_td_container_root will be a root folder containing a single sub_folder.
missing_td_container = missing_td_container_root.folders[ 0 ]
installed_td_container_root = containers_dict.get( 'tool_dependencies', None )
# The installed_td_container_root will be a root folder containing a single sub_folder.
if installed_td_container_root:
installed_td_container = installed_td_container_root.folders[ 0 ]
installed_td_container.label = 'Tool dependencies'
for index, td in enumerate( missing_td_container.tool_dependencies ):
# Skip the header row.
if index == 0:
continue
installed_td_container.tool_dependencies.append( td )
installed_td_container_root.folders = [ installed_td_container ]
containers_dict[ 'tool_dependencies' ] = installed_td_container_root
else:
# Change the folder label from 'Missing tool dependencies' to be
# 'Tool dependencies' for display.
root_container = containers_dict[ 'missing_tool_dependencies' ]
for sub_container in root_container.folders:
# There should only be 1 subfolder.
sub_container.label = 'Tool dependencies'
containers_dict[ 'tool_dependencies' ] = root_container
containers_dict[ 'missing_tool_dependencies' ] = None
return containers_dict
def populate_containers_dict_for_new_install( self, tool_shed_url, tool_path, readme_files_dict,
installed_repository_dependencies, missing_repository_dependencies,
installed_tool_dependencies, missing_tool_dependencies,
updating=False ):
"""
Return the populated containers for a repository being installed for the first time
or for an installed repository that is being updated and the updates include newly
defined repository (and possibly tool) dependencies.
"""
installed_tool_dependencies, missing_tool_dependencies = \
self.populate_tool_dependencies_dicts( tool_shed_url=tool_shed_url,
tool_path=tool_path,
repository_installed_tool_dependencies=installed_tool_dependencies,
repository_missing_tool_dependencies=missing_tool_dependencies,
required_repo_info_dicts=None )
# Most of the repository contents are set to None since we don't yet know what they are.
containers_dict = \
container_util.build_repository_containers_for_galaxy( app=self.app,
repository=None,
datatypes=None,
invalid_tools=None,
missing_repository_dependencies=missing_repository_dependencies,
missing_tool_dependencies=missing_tool_dependencies,
readme_files_dict=readme_files_dict,
repository_dependencies=installed_repository_dependencies,
tool_dependencies=installed_tool_dependencies,
valid_tools=None,
workflows=None,
valid_data_managers=None,
invalid_data_managers=None,
data_managers_errors=None,
new_install=True,
reinstalling=False )
if not updating:
# If we installing a new repository and not updaing an installed repository, we can merge
# the missing_repository_dependencies container contents to the installed_repository_dependencies
# container. When updating an installed repository, merging will result in losing newly defined
# dependencies included in the updates.
containers_dict = self.merge_missing_repository_dependencies_to_installed_container( containers_dict )
# Merge the missing_tool_dependencies container contents to the installed_tool_dependencies container.
containers_dict = self.merge_missing_tool_dependencies_to_installed_container( containers_dict )
return containers_dict
def populate_containers_dict_from_repository_metadata( self, tool_shed_url, tool_path, repository, reinstalling=False,
required_repo_info_dicts=None ):
"""
Retrieve necessary information from the received repository's metadata to populate the
containers_dict for display. This method is called only from Galaxy (not the tool shed)
when displaying repository dependencies for installed repositories and when displaying
them for uninstalled repositories that are being reinstalled.
"""
metadata = repository.metadata
if metadata:
# Handle proprietary datatypes.
datatypes = metadata.get( 'datatypes', None )
# Handle invalid tools.
invalid_tools = metadata.get( 'invalid_tools', None )
# Handle README files.
if repository.has_readme_files:
if reinstalling or repository.status not in \
[ self.app.install_model.ToolShedRepository.installation_status.DEACTIVATED,
self.app.install_model.ToolShedRepository.installation_status.INSTALLED ]:
# Since we're reinstalling, we need to send a request to the tool shed to get the README files.
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, tool_shed_url )
params = '?name=%s&owner=%s&changeset_revision=%s' % ( str( repository.name ),
str( repository.owner ),
str( repository.installed_changeset_revision ) )
url = common_util.url_join( tool_shed_url,
'repository/get_readme_files%s' % params )
raw_text = common_util.tool_shed_get( self.app, tool_shed_url, url )
readme_files_dict = json.from_json_string( raw_text )
else:
readme_files_dict = readme_util.build_readme_files_dict( self.app,
repository,
repository.changeset_revision,
repository.metadata, tool_path )
else:
readme_files_dict = None
# Handle repository dependencies.
installed_repository_dependencies, missing_repository_dependencies = \
self.app.installed_repository_manager.get_installed_and_missing_repository_dependencies( repository )
# Handle the current repository's tool dependencies.
repository_tool_dependencies = metadata.get( 'tool_dependencies', None )
# Make sure to display missing tool dependencies as well.
repository_invalid_tool_dependencies = metadata.get( 'invalid_tool_dependencies', None )
if repository_invalid_tool_dependencies is not None:
if repository_tool_dependencies is None:
repository_tool_dependencies = {}
repository_tool_dependencies.update( repository_invalid_tool_dependencies )
repository_installed_tool_dependencies, repository_missing_tool_dependencies = \
self.get_installed_and_missing_tool_dependencies_for_installed_repository( repository,
repository_tool_dependencies )
if reinstalling:
installed_tool_dependencies, missing_tool_dependencies = \
self.populate_tool_dependencies_dicts( tool_shed_url,
tool_path,
repository_installed_tool_dependencies,
repository_missing_tool_dependencies,
required_repo_info_dicts )
else:
installed_tool_dependencies = repository_installed_tool_dependencies
missing_tool_dependencies = repository_missing_tool_dependencies
# Handle valid tools.
valid_tools = metadata.get( 'tools', None )
# Handle workflows.
workflows = metadata.get( 'workflows', None )
# Handle Data Managers
valid_data_managers = None
invalid_data_managers = None
data_managers_errors = None
if 'data_manager' in metadata:
valid_data_managers = metadata['data_manager'].get( 'data_managers', None )
invalid_data_managers = metadata['data_manager'].get( 'invalid_data_managers', None )
data_managers_errors = metadata['data_manager'].get( 'messages', None )
containers_dict = \
container_util.build_repository_containers_for_galaxy( app=self.app,
repository=repository,
datatypes=datatypes,
invalid_tools=invalid_tools,
missing_repository_dependencies=missing_repository_dependencies,
missing_tool_dependencies=missing_tool_dependencies,
readme_files_dict=readme_files_dict,
repository_dependencies=installed_repository_dependencies,
tool_dependencies=installed_tool_dependencies,
valid_tools=valid_tools,
workflows=workflows,
valid_data_managers=valid_data_managers,
invalid_data_managers=invalid_data_managers,
data_managers_errors=data_managers_errors,
new_install=False,
reinstalling=reinstalling )
else:
containers_dict = dict( datatypes=None,
invalid_tools=None,
readme_files_dict=None,
repository_dependencies=None,
tool_dependencies=None,
valid_tools=None,
workflows=None )
return containers_dict
def populate_tool_dependencies_dicts( self, tool_shed_url, tool_path, repository_installed_tool_dependencies,
repository_missing_tool_dependencies, required_repo_info_dicts ):
"""
Return the populated installed_tool_dependencies and missing_tool_dependencies dictionaries
for all repositories defined by entries in the received required_repo_info_dicts.
"""
installed_tool_dependencies = None
missing_tool_dependencies = None
if repository_installed_tool_dependencies is None:
repository_installed_tool_dependencies = {}
else:
# Add the install_dir attribute to the tool_dependencies.
repository_installed_tool_dependencies = \
self.add_installation_directories_to_tool_dependencies( repository_installed_tool_dependencies )
if repository_missing_tool_dependencies is None:
repository_missing_tool_dependencies = {}
else:
# Add the install_dir attribute to the tool_dependencies.
repository_missing_tool_dependencies = \
self.add_installation_directories_to_tool_dependencies( repository_missing_tool_dependencies )
if required_repo_info_dicts:
# Handle the tool dependencies defined for each of the repository's repository dependencies.
for rid in required_repo_info_dicts:
for name, repo_info_tuple in rid.items():
description, \
repository_clone_url, \
changeset_revision, \
ctx_rev, \
repository_owner, \
repository_dependencies, \
tool_dependencies = \
suc.get_repo_info_tuple_contents( repo_info_tuple )
if tool_dependencies:
# Add the install_dir attribute to the tool_dependencies.
tool_dependencies = self.add_installation_directories_to_tool_dependencies( tool_dependencies )
# The required_repository may have been installed with a different changeset revision.
required_repository, installed_changeset_revision = \
suc.repository_was_previously_installed( self.app,
tool_shed_url,
name,
repo_info_tuple,
from_tip=False )
if required_repository:
required_repository_installed_tool_dependencies, required_repository_missing_tool_dependencies = \
self.get_installed_and_missing_tool_dependencies_for_installed_repository( required_repository,
tool_dependencies )
if required_repository_installed_tool_dependencies:
# Add the install_dir attribute to the tool_dependencies.
required_repository_installed_tool_dependencies = \
self.add_installation_directories_to_tool_dependencies( required_repository_installed_tool_dependencies )
for td_key, td_dict in required_repository_installed_tool_dependencies.items():
if td_key not in repository_installed_tool_dependencies:
repository_installed_tool_dependencies[ td_key ] = td_dict
if required_repository_missing_tool_dependencies:
# Add the install_dir attribute to the tool_dependencies.
required_repository_missing_tool_dependencies = \
self.add_installation_directories_to_tool_dependencies( required_repository_missing_tool_dependencies )
for td_key, td_dict in required_repository_missing_tool_dependencies.items():
if td_key not in repository_missing_tool_dependencies:
repository_missing_tool_dependencies[ td_key ] = td_dict
if repository_installed_tool_dependencies:
installed_tool_dependencies = repository_installed_tool_dependencies
if repository_missing_tool_dependencies:
missing_tool_dependencies = repository_missing_tool_dependencies
return installed_tool_dependencies, missing_tool_dependencies
def tool_dependency_is_orphan( self, type, name, version, tools ):
"""
Determine if the combination of the received type, name and version is defined in the <requirement>
tag for at least one tool in the received list of tools. If not, the tool dependency defined by the
combination is considered an orphan in its repository in the tool shed.
"""
if type == 'package':
if name and version:
for tool_dict in tools:
requirements = tool_dict.get( 'requirements', [] )
for requirement_dict in requirements:
req_name = requirement_dict.get( 'name', None )
req_version = requirement_dict.get( 'version', None )
req_type = requirement_dict.get( 'type', None )
if req_name == name and req_version == version and req_type == type:
return False
elif type == 'set_environment':
if name:
for tool_dict in tools:
requirements = tool_dict.get( 'requirements', [] )
for requirement_dict in requirements:
req_name = requirement_dict.get( 'name', None )
req_type = requirement_dict.get( 'type', None )
if req_name == name and req_type == type:
return False
return True
lib/tool_shed/galaxy_install/install_manager.py
+6 -189
View File
@@ -3,7 +3,6 @@ import logging
import os
import sys
import tempfile
import threading
import traceback
from galaxy import exceptions
@@ -19,7 +18,6 @@ from galaxy.model.orm import or_
from tool_shed.util import basic_util
from tool_shed.util import common_util
from tool_shed.util import container_util
from tool_shed.util import data_manager_util
from tool_shed.util import datatype_util
from tool_shed.util import encoding_util
@@ -225,7 +223,8 @@ class InstallToolDependencyManager( object ):
installed_packages.append( tool_dependency )
if self.app.config.manage_dependency_relationships:
# Add the tool_dependency to the in-memory dictionaries in the installed_repository_manager.
self.app.installed_repository_manager.handle_tool_dependency_install( tool_shed_repository, tool_dependency )
self.app.installed_repository_manager.handle_tool_dependency_install( tool_shed_repository,
tool_dependency )
return installed_packages
def install_via_fabric( self, tool_shed_repository, tool_dependency, install_dir, package_name=None, custom_fabfile_path=None,
@@ -371,8 +370,8 @@ class InstallToolDependencyManager( object ):
# Delete contents of installation directory if attempt at binary installation failed.
installation_directory_contents = os.listdir( installation_directory )
if installation_directory_contents:
removed, error_message = tool_dependency_util.remove_tool_dependency( self.app,
tool_dependency )
removed, error_message = \
tool_dependency_util.remove_tool_dependency( self.app, tool_dependency )
if removed:
can_install_from_source = True
else:
@@ -894,7 +893,8 @@ class InstallRepositoryManager( object ):
tool_shed_repository,
install_model.ToolShedRepository.installation_status.INSTALLED )
if self.app.config.manage_dependency_relationships:
# Add the installed repository and any tool dependencies to the in-memory dictionaries in the installed_repository_manager.
# Add the installed repository and any tool dependencies to the in-memory dictionaries
# in the installed_repository_manager.
self.app.installed_repository_manager.handle_repository_install( tool_shed_repository )
else:
# An error occurred while cloning the repository, so reset everything necessary to enable another attempt.
@@ -906,147 +906,6 @@ class InstallRepositoryManager( object ):
uninstalled=False,
remove_from_disk=True )
def merge_containers_dicts_for_new_install( self, containers_dicts ):
"""
When installing one or more tool shed repositories for the first time, the received list of
containers_dicts contains a containers_dict for each repository being installed. Since the
repositories are being installed for the first time, all entries are None except the repository
dependencies and tool dependencies. The entries for missing dependencies are all None since
they have previously been merged into the installed dependencies. This method will merge the
dependencies entries into a single container and return it for display.
"""
new_containers_dict = dict( readme_files=None,
datatypes=None,
missing_repository_dependencies=None,
repository_dependencies=None,
missing_tool_dependencies=None,
tool_dependencies=None,
invalid_tools=None,
valid_tools=None,
workflows=None )
if containers_dicts:
lock = threading.Lock()
lock.acquire( True )
try:
repository_dependencies_root_folder = None
tool_dependencies_root_folder = None
# Use a unique folder id (hopefully the following is).
folder_id = 867
for old_container_dict in containers_dicts:
# Merge repository_dependencies.
old_container_repository_dependencies_root = old_container_dict[ 'repository_dependencies' ]
if old_container_repository_dependencies_root:
if repository_dependencies_root_folder is None:
repository_dependencies_root_folder = container_util.Folder( id=folder_id,
key='root',
label='root',
parent=None )
folder_id += 1
repository_dependencies_folder = container_util.Folder( id=folder_id,
key='merged',
label='Repository dependencies',
parent=repository_dependencies_root_folder )
folder_id += 1
# The old_container_repository_dependencies_root will be a root folder containing a single sub_folder.
old_container_repository_dependencies_folder = old_container_repository_dependencies_root.folders[ 0 ]
# Change the folder id so it won't confict with others being merged.
old_container_repository_dependencies_folder.id = folder_id
folder_id += 1
repository_components_tuple = container_util.get_components_from_key( old_container_repository_dependencies_folder.key )
components_list = suc.extract_components_from_tuple( repository_components_tuple )
name = components_list[ 1 ]
# Generate the label by retrieving the repository name.
old_container_repository_dependencies_folder.label = str( name )
repository_dependencies_folder.folders.append( old_container_repository_dependencies_folder )
# Merge tool_dependencies.
old_container_tool_dependencies_root = old_container_dict[ 'tool_dependencies' ]
if old_container_tool_dependencies_root:
if tool_dependencies_root_folder is None:
tool_dependencies_root_folder = container_util.Folder( id=folder_id,
key='root',
label='root',
parent=None )
folder_id += 1
tool_dependencies_folder = container_util.Folder( id=folder_id,
key='merged',
label='Tool dependencies',
parent=tool_dependencies_root_folder )
folder_id += 1
else:
td_list = [ td.listify for td in tool_dependencies_folder.tool_dependencies ]
# The old_container_tool_dependencies_root will be a root folder containing a single sub_folder.
old_container_tool_dependencies_folder = old_container_tool_dependencies_root.folders[ 0 ]
for td in old_container_tool_dependencies_folder.tool_dependencies:
if td.listify not in td_list:
tool_dependencies_folder.tool_dependencies.append( td )
if repository_dependencies_root_folder:
repository_dependencies_root_folder.folders.append( repository_dependencies_folder )
new_containers_dict[ 'repository_dependencies' ] = repository_dependencies_root_folder
if tool_dependencies_root_folder:
tool_dependencies_root_folder.folders.append( tool_dependencies_folder )
new_containers_dict[ 'tool_dependencies' ] = tool_dependencies_root_folder
except Exception, e:
log.debug( "Exception in merge_containers_dicts_for_new_install: %s" % str( e ) )
finally:
lock.release()
return new_containers_dict
def merge_missing_repository_dependencies_to_installed_container( self, containers_dict ):
"""Merge the list of missing repository dependencies into the list of installed repository dependencies."""
missing_rd_container_root = containers_dict.get( 'missing_repository_dependencies', None )
if missing_rd_container_root:
# The missing_rd_container_root will be a root folder containing a single sub_folder.
missing_rd_container = missing_rd_container_root.folders[ 0 ]
installed_rd_container_root = containers_dict.get( 'repository_dependencies', None )
# The installed_rd_container_root will be a root folder containing a single sub_folder.
if installed_rd_container_root:
installed_rd_container = installed_rd_container_root.folders[ 0 ]
installed_rd_container.label = 'Repository dependencies'
for index, rd in enumerate( missing_rd_container.repository_dependencies ):
# Skip the header row.
if index == 0:
continue
installed_rd_container.repository_dependencies.append( rd )
installed_rd_container_root.folders = [ installed_rd_container ]
containers_dict[ 'repository_dependencies' ] = installed_rd_container_root
else:
# Change the folder label from 'Missing repository dependencies' to be 'Repository dependencies' for display.
root_container = containers_dict[ 'missing_repository_dependencies' ]
for sub_container in root_container.folders:
# There should only be 1 sub-folder.
sub_container.label = 'Repository dependencies'
containers_dict[ 'repository_dependencies' ] = root_container
containers_dict[ 'missing_repository_dependencies' ] = None
return containers_dict
def merge_missing_tool_dependencies_to_installed_container( self, containers_dict ):
""" Merge the list of missing tool dependencies into the list of installed tool dependencies."""
missing_td_container_root = containers_dict.get( 'missing_tool_dependencies', None )
if missing_td_container_root:
# The missing_td_container_root will be a root folder containing a single sub_folder.
missing_td_container = missing_td_container_root.folders[ 0 ]
installed_td_container_root = containers_dict.get( 'tool_dependencies', None )
# The installed_td_container_root will be a root folder containing a single sub_folder.
if installed_td_container_root:
installed_td_container = installed_td_container_root.folders[ 0 ]
installed_td_container.label = 'Tool dependencies'
for index, td in enumerate( missing_td_container.tool_dependencies ):
# Skip the header row.
if index == 0:
continue
installed_td_container.tool_dependencies.append( td )
installed_td_container_root.folders = [ installed_td_container ]
containers_dict[ 'tool_dependencies' ] = installed_td_container_root
else:
# Change the folder label from 'Missing tool dependencies' to be 'Tool dependencies' for display.
root_container = containers_dict[ 'missing_tool_dependencies' ]
for sub_container in root_container.folders:
# There should only be 1 subfolder.
sub_container.label = 'Tool dependencies'
containers_dict[ 'tool_dependencies' ] = root_container
containers_dict[ 'missing_tool_dependencies' ] = None
return containers_dict
def order_components_for_installation( self, tsr_ids, repo_info_dicts, tool_panel_section_keys ):
"""
Some repositories may have repository dependencies that are required to be installed
@@ -1096,45 +955,3 @@ class InstallRepositoryManager( object ):
ordered_repo_info_dicts.append( repo_info_dict )
ordered_tool_panel_section_keys.append( tool_panel_section_key )
return ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys
def populate_containers_dict_for_new_install( self, tool_shed_url, tool_path, readme_files_dict, installed_repository_dependencies,
missing_repository_dependencies, installed_tool_dependencies, missing_tool_dependencies,
updating=False ):
"""
Return the populated containers for a repository being installed for the first time or for an installed repository
that is being updated and the updates include newly defined repository (and possibly tool) dependencies.
"""
installed_tool_dependencies, missing_tool_dependencies = \
tool_dependency_util.populate_tool_dependencies_dicts( app=self.app,
tool_shed_url=tool_shed_url,
tool_path=tool_path,
repository_installed_tool_dependencies=installed_tool_dependencies,
repository_missing_tool_dependencies=missing_tool_dependencies,
required_repo_info_dicts=None )
# Most of the repository contents are set to None since we don't yet know what they are.
containers_dict = \
container_util.build_repository_containers_for_galaxy( app=self.app,
repository=None,
datatypes=None,
invalid_tools=None,
missing_repository_dependencies=missing_repository_dependencies,
missing_tool_dependencies=missing_tool_dependencies,
readme_files_dict=readme_files_dict,
repository_dependencies=installed_repository_dependencies,
tool_dependencies=installed_tool_dependencies,
valid_tools=None,
workflows=None,
valid_data_managers=None,
invalid_data_managers=None,
data_managers_errors=None,
new_install=True,
reinstalling=False )
if not updating:
# If we installing a new repository and not updaing an installed repository, we can merge
# the missing_repository_dependencies container contents to the installed_repository_dependencies
# container. When updating an installed repository, merging will result in losing newly defined
# dependencies included in the updates.
containers_dict = self.merge_missing_repository_dependencies_to_installed_container( containers_dict )
# Merge the missing_tool_dependencies container contents to the installed_tool_dependencies container.
containers_dict = self.merge_missing_tool_dependencies_to_installed_container( containers_dict )
return containers_dict
lib/tool_shed/galaxy_install/installed_repository_manager.py
+96 -1
View File
@@ -395,7 +395,13 @@ class InstalledRepositoryManager( object ):
installed_repository_dependencies = {}
missing_rd_tups = []
installed_rd_tups = []
description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \
description, \
repository_clone_url, \
changeset_revision, \
ctx_rev, \
repository_owner, \
repository_dependencies, \
tool_dependencies = \
suc.get_repo_info_tuple_contents( repo_info_tuple )
if repository_dependencies:
description = repository_dependencies[ 'description' ]
@@ -614,6 +620,30 @@ class InstalledRepositoryManager( object ):
type = str( tool_dependency.type )
return ( tool_dependency.tool_shed_repository_id, str( tool_dependency.name ), str( tool_dependency.version ), type )
def handle_existing_tool_dependencies_that_changed_in_update( self, repository, original_dependency_dict,
new_dependency_dict ):
"""
This method is called when a Galaxy admin is getting updates for an installed tool shed
repository in order to cover the case where an existing tool dependency was changed (e.g.,
the version of the dependency was changed) but the tool version for which it is a dependency
was not changed. In this case, we only want to determine if any of the dependency information
defined in original_dependency_dict was changed in new_dependency_dict. We don't care if new
dependencies were added in new_dependency_dict since they will just be treated as missing
dependencies for the tool.
"""
updated_tool_dependency_names = []
deleted_tool_dependency_names = []
for original_dependency_key, original_dependency_val_dict in original_dependency_dict.items():
if original_dependency_key not in new_dependency_dict:
updated_tool_dependency = self.update_existing_tool_dependency( repository,
original_dependency_val_dict,
new_dependency_dict )
if updated_tool_dependency:
updated_tool_dependency_names.append( updated_tool_dependency.name )
else:
deleted_tool_dependency_names.append( original_dependency_val_dict[ 'name' ] )
return updated_tool_dependency_names, deleted_tool_dependency_names
def handle_repository_install( self, repository ):
"""Load the dependency relationships for a repository that was just installed or reinstalled."""
# Populate self.repository_dependencies_of_installed_repositories.
@@ -924,3 +954,68 @@ class InstalledRepositoryManager( object ):
repository_dependency.changeset_revision == changeset_revision ):
return True
return False
def update_existing_tool_dependency( self, repository, original_dependency_dict, new_dependencies_dict ):
"""
Update an exsiting tool dependency whose definition was updated in a change set
pulled by a Galaxy administrator when getting updates to an installed tool shed
repository. The original_dependency_dict is a single tool dependency definition,
an example of which is::
{"name": "bwa",
"readme": "\\nCompiling BWA requires zlib and libpthread to be present on your system.\\n ",
"type": "package",
"version": "0.6.2"}
The new_dependencies_dict is the dictionary generated by the metadata_util.generate_tool_dependency_metadata method.
"""
new_tool_dependency = None
original_name = original_dependency_dict[ 'name' ]
original_type = original_dependency_dict[ 'type' ]
original_version = original_dependency_dict[ 'version' ]
# Locate the appropriate tool_dependency associated with the repository.
tool_dependency = None
for tool_dependency in repository.tool_dependencies:
if tool_dependency.name == original_name and \
tool_dependency.type == original_type and \
tool_dependency.version == original_version:
break
if tool_dependency and tool_dependency.can_update:
dependency_install_dir = tool_dependency.installation_directory( self.app )
removed_from_disk, error_message = \
tool_dependency_util.remove_tool_dependency_installation_directory( dependency_install_dir )
if removed_from_disk:
context = self.app.install_model.context
new_dependency_name = None
new_dependency_type = None
new_dependency_version = None
for new_dependency_key, new_dependency_val_dict in new_dependencies_dict.items():
# Match on name only, hopefully this will be enough!
if original_name == new_dependency_val_dict[ 'name' ]:
new_dependency_name = new_dependency_val_dict[ 'name' ]
new_dependency_type = new_dependency_val_dict[ 'type' ]
new_dependency_version = new_dependency_val_dict[ 'version' ]
break
if new_dependency_name and new_dependency_type and new_dependency_version:
# Update all attributes of the tool_dependency record in the database.
log.debug( "Updating version %s of tool dependency %s %s to have new version %s and type %s." % \
( str( tool_dependency.version ),
str( tool_dependency.type ),
str( tool_dependency.name ),
str( new_dependency_version ),
str( new_dependency_type ) ) )
tool_dependency.type = new_dependency_type
tool_dependency.version = new_dependency_version
tool_dependency.status = self.app.install_model.ToolDependency.installation_status.UNINSTALLED
tool_dependency.error_message = None
context.add( tool_dependency )
context.flush()
new_tool_dependency = tool_dependency
else:
# We have no new tool dependency definition based on a matching dependency name, so remove
# the existing tool dependency record from the database.
log.debug( "Deleting version %s of tool dependency %s %s from the database since it is no longer defined." % \
( str( tool_dependency.version ), str( tool_dependency.type ), str( tool_dependency.name ) ) )
context.delete( tool_dependency )
context.flush()
return new_tool_dependency
lib/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py
+114 -21
View File
@@ -25,7 +25,89 @@ class RecipeTag( object ):
raise "Unimplemented Method"
class Install( RecipeTag ):
class SyncDatabase( object ):
def sync_database_with_file_system( self, app, tool_shed_repository, tool_dependency_name, tool_dependency_version,
tool_dependency_install_dir, tool_dependency_type='package' ):
"""
The installation directory defined by the received tool_dependency_install_dir exists, so check for
the presence of INSTALLATION_LOG. If the files exists, we'll assume the tool dependency is installed,
but not necessarily successfully (it could be in an error state on disk. However, we can justifiably
assume here that no matter the state, an associated database record will exist.
"""
# This method should be reached very rarely. It implies that either the Galaxy environment
# became corrupted (i.e., the database records for installed tool dependencies is not synchronized
# with tool dependencies on disk) or the Tool Shed's install and test framework is running. The Tool
# Shed's install and test framework installs repositories in 2 stages, those of type tool_dependency_definition
# followed by those containing valid tools and tool functional test components.
log.debug( "Synchronizing the database with the file system..." )
try:
log.debug( "The value of app.config.running_functional_tests is: %s" % \
str( app.config.running_functional_tests ) )
except:
pass
sa_session = app.install_model.context
can_install_tool_dependency = False
tool_dependency = get_tool_dependency_by_name_version_type_repository( app,
tool_shed_repository,
tool_dependency_name,
tool_dependency_version,
tool_dependency_type )
if tool_dependency.status == app.install_model.ToolDependency.installation_status.INSTALLING:
# The tool dependency is in an Installing state, so we don't want to do anything to it. If the tool
# dependency is being installed by someone else, we don't want to interfere with that. This assumes
# the installation by "someone else" is not hung in an Installing state, which is a weakness if that
# "someone else" never repaired it.
log.debug( 'Skipping installation of tool dependency %s version %s because it has a status of %s' % \
( str( tool_dependency.name ), str( tool_dependency.version ), str( tool_dependency.status ) ) )
else:
# We have a pre-existing installation directory on the file system, but our associated database record is
# in a state that allowed us to arrive here. At this point, we'll inspect the installation directory to
# see if we have a "proper installation" and if so, synchronize the database record rather than reinstalling
# the dependency if we're "running_functional_tests". If we're not "running_functional_tests, we'll set
# the tool dependency's installation status to ERROR.
tool_dependency_installation_directory_contents = os.listdir( tool_dependency_install_dir )
if basic_util.INSTALLATION_LOG in tool_dependency_installation_directory_contents:
# Since this tool dependency's installation directory contains an installation log, we consider it to be
# installed. In some cases the record may be missing from the database due to some activity outside of
# the control of the Tool Shed. Since a new record was created for it and we don't know the state of the
# files on disk, we will set it to an error state (unless we are running Tool Shed functional tests - see
# below).
log.debug( 'Skipping installation of tool dependency %s version %s because it is installed in %s' % \
( str( tool_dependency.name ), str( tool_dependency.version ), str( tool_dependency_install_dir ) ) )
if app.config.running_functional_tests:
# If we are running functional tests, the state will be set to Installed because previously compiled
# tool dependencies are not deleted by default, from the "install and test" framework..
tool_dependency.status = app.install_model.ToolDependency.installation_status.INSTALLED
else:
error_message = 'The installation directory for this tool dependency had contents but the database had no record. '
error_message += 'The installation log may show this tool dependency to be correctly installed, but due to the '
error_message += 'missing database record it is now being set to Error.'
tool_dependency.status = app.install_model.ToolDependency.installation_status.ERROR
tool_dependency.error_message = error_message
else:
error_message = '\nInstallation path %s for tool dependency %s version %s exists, but the expected file %s' % \
( str( tool_dependency_install_dir ),
str( tool_dependency_name ),
str( tool_dependency_version ),
str( basic_util.INSTALLATION_LOG ) )
error_message += ' is missing. This indicates an installation error so the tool dependency is being'
error_message += ' prepared for re-installation.'
print error_message
tool_dependency.status = app.install_model.ToolDependency.installation_status.NEVER_INSTALLED
basic_util.remove_dir( tool_dependency_install_dir )
can_install_tool_dependency = True
sa_session.add( tool_dependency )
sa_session.flush()
try:
log.debug( "Returning from sync_database_with_file_system with tool_dependency %s, can_install_tool_dependency %s." % \
( str( tool_dependency.name ), str( can_install_tool_dependency ) ) )
except Exception, e:
log.debug( str( e ) )
return tool_dependency, can_install_tool_dependency
class Install( RecipeTag, SyncDatabase ):
def __init__( self, app ):
self.app = app
@@ -57,13 +139,12 @@ class Install( RecipeTag ):
proceed_with_install = True
else:
# Notice that we'll throw away the following tool_dependency if it can be installed.
tool_dependency, proceed_with_install = \
tool_dependency_util.sync_database_with_file_system( self.app,
tool_shed_repository,
package_name,
package_version,
install_dir,
tool_dependency_type='package' )
tool_dependency, proceed_with_install = self.sync_database_with_file_system( self.app,
tool_shed_repository,
package_name,
package_version,
install_dir,
tool_dependency_type='package' )
if not proceed_with_install:
log.debug( "Tool dependency %s version %s cannot be installed (it was probably previously installed), so returning it." % \
( str( tool_dependency.name ), str( tool_dependency.version ) ) )
@@ -156,7 +237,7 @@ class ReadMe( RecipeTag ):
return tool_dependency, proceed_with_install, action_elem_tuples
class Repository( RecipeTag ):
class Repository( RecipeTag, SyncDatabase ):
def __init__( self, app ):
self.app = app
@@ -190,7 +271,8 @@ class Repository( RecipeTag ):
return None
def create_tool_dependency_with_initialized_env_sh_file( self, dependent_install_dir, tool_shed_repository,
required_repository, package_name, package_version, tool_dependencies_config ):
required_repository, package_name, package_version,
tool_dependencies_config ):
"""
Create or get a tool_dependency record that is defined by the received package_name and package_version.
An env.sh file will be created for the tool_dependency in the received dependent_install_dir.
@@ -238,10 +320,9 @@ class Repository( RecipeTag ):
# the path defined by required_tool_dependency_env_file_path. It doesn't matter if the required env.sh
# file currently exists..
required_tool_dependency_env_file_path = \
tool_dependency_util.get_required_repository_package_env_sh_path( self.app,
package_name,
package_version,
required_repository )
self.get_required_repository_package_env_sh_path( package_name,
package_version,
required_repository )
env_file_builder = EnvFileBuilder( tool_dependency.installation_directory( self.app ) )
env_file_builder.append_line( action="source", value=required_tool_dependency_env_file_path )
return_code = env_file_builder.return_code
@@ -269,6 +350,19 @@ class Repository( RecipeTag ):
tool_dependencies.append( tool_dependency )
return tool_dependencies
def get_required_repository_package_env_sh_path( self, package_name, package_version, required_repository ):
"""Return path to env.sh file in required repository if the required repository has been installed."""
env_sh_file_dir = \
tool_dependency_util.get_tool_dependency_install_dir( app=self.app,
repository_name=required_repository.name,
repository_owner=required_repository.owner,
repository_changeset_revision=required_repository.installed_changeset_revision,
tool_dependency_type='package',
tool_dependency_name=package_name,
tool_dependency_version=package_version )
env_sh_file_path = os.path.join( env_sh_file_dir, 'env.sh' )
return env_sh_file_path
def get_tool_shed_repository_by_tool_shed_name_owner_changeset_revision( self, tool_shed_url, name, owner, changeset_revision ):
sa_session = self.app.install_model.context
# The protocol is not stored, but the port is if it exists.
@@ -352,13 +446,12 @@ class Repository( RecipeTag ):
can_install_tool_dependency = True
else:
# Notice that we'll throw away the following tool_dependency if it can be installed.
tool_dependency, can_install_tool_dependency = \
tool_dependency_util.sync_database_with_file_system( self.app,
tool_shed_repository,
package_name,
package_version,
dependent_install_dir,
tool_dependency_type='package' )
tool_dependency, can_install_tool_dependency = self.sync_database_with_file_system( self.app,
tool_shed_repository,
package_name,
package_version,
dependent_install_dir,
tool_dependency_type='package' )
if not can_install_tool_dependency:
log.debug( "Tool dependency %s version %s cannot be installed (it was probably previously installed), " % \
( str( tool_dependency.name, str( tool_dependency.version ) ) ) )
lib/tool_shed/util/metadata_util.py
+22 -188
View File
@@ -3,7 +3,6 @@ import os
import tempfile
from galaxy import util
from galaxy import web
from galaxy.datatypes import checkers
from galaxy.model.orm import and_
from galaxy.tools.data_manager.manager import DataManager
@@ -11,17 +10,17 @@ from galaxy.util import inflector
from galaxy.util import json
from galaxy.web import url_for
import tool_shed.util.shed_util_common as suc
from tool_shed.repository_types.metadata import TipOnly
import tool_shed.repository_types.util as rt_util
from tool_shed.util import basic_util
from tool_shed.util import common_util
from tool_shed.util import container_util
from tool_shed.util import hg_util
from tool_shed.util import readme_util
from tool_shed.util import shed_util_common as suc
from tool_shed.util import tool_dependency_util
from tool_shed.util import tool_util
from tool_shed.util import xml_util
import tool_shed.repository_types.util as rt_util
log = logging.getLogger( __name__ )
@@ -1013,7 +1012,11 @@ def generate_tool_dependency_metadata( app, repository, changeset_revision, repo
description = root.get( 'description' )
for elem in root:
if elem.tag == 'package':
valid_tool_dependencies_dict, invalid_tool_dependencies_dict, repository_dependency_tup, repository_dependency_is_valid, message = \
valid_tool_dependencies_dict, \
invalid_tool_dependencies_dict, \
repository_dependency_tup, \
repository_dependency_is_valid, \
message = \
generate_package_dependency_metadata( app, elem, valid_tool_dependencies_dict, invalid_tool_dependencies_dict )
if repository_dependency_is_valid:
if repository_dependency_tup and repository_dependency_tup not in valid_repository_dependency_tups:
@@ -1024,7 +1027,13 @@ def generate_tool_dependency_metadata( app, repository, changeset_revision, repo
# We have an invalid complex repository dependency, so mark the tool dependency as invalid.
tool_dependency_is_valid = False
# Append the error message to the invalid repository dependency tuple.
toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = repository_dependency_tup
toolshed, \
name, \
owner, \
changeset_revision, \
prior_installation_required, \
only_if_compiling_contained_td \
= repository_dependency_tup
repository_dependency_tup = \
( toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, message )
invalid_repository_dependency_tups.append( repository_dependency_tup )
@@ -1033,9 +1042,13 @@ def generate_tool_dependency_metadata( app, repository, changeset_revision, repo
valid_tool_dependencies_dict = generate_environment_dependency_metadata( elem, valid_tool_dependencies_dict )
if valid_tool_dependencies_dict:
if original_valid_tool_dependencies_dict:
# We're generating metadata on an update pulled to a tool shed repository installed into a Galaxy instance, so handle changes to
# tool dependencies appropriately.
handle_existing_tool_dependencies_that_changed_in_update( app, repository, original_valid_tool_dependencies_dict, valid_tool_dependencies_dict )
# We're generating metadata on an update pulled to a tool shed repository installed
# into a Galaxy instance, so handle changes to tool dependencies appropriately.
irm = app.installed_repository_manager
updated_tool_dependency_names, deleted_tool_dependency_names = \
irm.handle_existing_tool_dependencies_that_changed_in_update( repository,
original_valid_tool_dependencies_dict,
valid_tool_dependencies_dict )
metadata_dict[ 'tool_dependencies' ] = valid_tool_dependencies_dict
if invalid_tool_dependencies_dict:
metadata_dict[ 'invalid_tool_dependencies' ] = invalid_tool_dependencies_dict
@@ -1296,25 +1309,6 @@ def get_sample_files_from_disk( repository_files_dir, tool_path=None, relative_i
sample_file_metadata_paths.append( relative_path_to_sample_file )
return sample_file_metadata_paths, sample_file_copy_paths
def handle_existing_tool_dependencies_that_changed_in_update( app, repository, original_dependency_dict, new_dependency_dict ):
"""
This method is called when a Galaxy admin is getting updates for an installed tool shed repository in order to cover the case where an
existing tool dependency was changed (e.g., the version of the dependency was changed) but the tool version for which it is a dependency
was not changed. In this case, we only want to determine if any of the dependency information defined in original_dependency_dict was
changed in new_dependency_dict. We don't care if new dependencies were added in new_dependency_dict since they will just be treated as
missing dependencies for the tool.
"""
updated_tool_dependency_names = []
deleted_tool_dependency_names = []
for original_dependency_key, original_dependency_val_dict in original_dependency_dict.items():
if original_dependency_key not in new_dependency_dict:
updated_tool_dependency = update_existing_tool_dependency( app, repository, original_dependency_val_dict, new_dependency_dict )
if updated_tool_dependency:
updated_tool_dependency_names.append( updated_tool_dependency.name )
else:
deleted_tool_dependency_names.append( original_dependency_val_dict[ 'name' ] )
return updated_tool_dependency_names, deleted_tool_dependency_names
def handle_repository_elem( app, repository_elem, only_if_compiling_contained_td=False, updating_installed_repository=False ):
"""
Process the received repository_elem which is a <repository> tag either from a repository_dependencies.xml
@@ -1737,104 +1731,6 @@ def new_workflow_metadata_required( repository_metadata, metadata_dict ):
# The received metadata_dict includes no metadata for workflows, so a new repository_metadata table record is not needed.
return False
def populate_containers_dict_from_repository_metadata( app, tool_shed_url, tool_path, repository,
reinstalling=False, required_repo_info_dicts=None ):
"""
Retrieve necessary information from the received repository's metadata to populate the
containers_dict for display. This method is called only from Galaxy (not the tool shed)
when displaying repository dependencies for installed repositories and when displaying
them for uninstalled repositories that are being reinstalled.
"""
metadata = repository.metadata
if metadata:
# Handle proprietary datatypes.
datatypes = metadata.get( 'datatypes', None )
# Handle invalid tools.
invalid_tools = metadata.get( 'invalid_tools', None )
# Handle README files.
if repository.has_readme_files:
if reinstalling or repository.status not in [ app.install_model.ToolShedRepository.installation_status.DEACTIVATED,
app.install_model.ToolShedRepository.installation_status.INSTALLED ]:
# Since we're reinstalling, we need to send a request to the tool shed to get the README files.
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url )
params = '?name=%s&owner=%s&changeset_revision=%s' % ( str( repository.name ),
str( repository.owner ),
str( repository.installed_changeset_revision ) )
url = common_util.url_join( tool_shed_url,
'repository/get_readme_files%s' % params )
raw_text = common_util.tool_shed_get( app, tool_shed_url, url )
readme_files_dict = json.from_json_string( raw_text )
else:
readme_files_dict = readme_util.build_readme_files_dict( app,
repository,
repository.changeset_revision,
repository.metadata, tool_path )
else:
readme_files_dict = None
# Handle repository dependencies.
installed_repository_dependencies, missing_repository_dependencies = \
app.installed_repository_manager.get_installed_and_missing_repository_dependencies( repository )
# Handle the current repository's tool dependencies.
repository_tool_dependencies = metadata.get( 'tool_dependencies', None )
# Make sure to display missing tool dependencies as well.
repository_invalid_tool_dependencies = metadata.get( 'invalid_tool_dependencies', None )
if repository_invalid_tool_dependencies is not None:
if repository_tool_dependencies is None:
repository_tool_dependencies = {}
repository_tool_dependencies.update( repository_invalid_tool_dependencies )
repository_installed_tool_dependencies, repository_missing_tool_dependencies = \
tool_dependency_util.get_installed_and_missing_tool_dependencies_for_installed_repository( app,
repository,
repository_tool_dependencies )
if reinstalling:
installed_tool_dependencies, missing_tool_dependencies = \
tool_dependency_util.populate_tool_dependencies_dicts( app,
tool_shed_url,
tool_path,
repository_installed_tool_dependencies,
repository_missing_tool_dependencies,
required_repo_info_dicts )
else:
installed_tool_dependencies = repository_installed_tool_dependencies
missing_tool_dependencies = repository_missing_tool_dependencies
# Handle valid tools.
valid_tools = metadata.get( 'tools', None )
# Handle workflows.
workflows = metadata.get( 'workflows', None )
# Handle Data Managers
valid_data_managers = None
invalid_data_managers = None
data_managers_errors = None
if 'data_manager' in metadata:
valid_data_managers = metadata['data_manager'].get( 'data_managers', None )
invalid_data_managers = metadata['data_manager'].get( 'invalid_data_managers', None )
data_managers_errors = metadata['data_manager'].get( 'messages', None )
containers_dict = container_util.build_repository_containers_for_galaxy( app=app,
repository=repository,
datatypes=datatypes,
invalid_tools=invalid_tools,
missing_repository_dependencies=missing_repository_dependencies,
missing_tool_dependencies=missing_tool_dependencies,
readme_files_dict=readme_files_dict,
repository_dependencies=installed_repository_dependencies,
tool_dependencies=installed_tool_dependencies,
valid_tools=valid_tools,
workflows=workflows,
valid_data_managers=valid_data_managers,
invalid_data_managers=invalid_data_managers,
data_managers_errors=data_managers_errors,
new_install=False,
reinstalling=reinstalling )
else:
containers_dict = dict( datatypes=None,
invalid_tools=None,
readme_files_dict=None,
repository_dependencies=None,
tool_dependencies=None,
valid_tools=None,
workflows=None )
return containers_dict
def reset_all_metadata_on_installed_repository( app, id ):
"""Reset all metadata on a single tool shed repository installed into a Galaxy instance."""
invalid_file_tups = []
@@ -2201,68 +2097,6 @@ def set_repository_metadata_due_to_new_tip( app, host, user, repository, content
error_message, status = set_repository_metadata( app, host, user, repository, content_alert_str=content_alert_str, **kwd )
return status, error_message
def update_existing_tool_dependency( app, repository, original_dependency_dict, new_dependencies_dict ):
"""
Update an exsiting tool dependency whose definition was updated in a change set
pulled by a Galaxy administrator when getting updates to an installed tool shed
repository. The original_dependency_dict is a single tool dependency definition,
an example of which is::
{"name": "bwa",
"readme": "\\nCompiling BWA requires zlib and libpthread to be present on your system.\\n ",
"type": "package",
"version": "0.6.2"}
The new_dependencies_dict is the dictionary generated by the metadata_util.generate_tool_dependency_metadata method.
"""
new_tool_dependency = None
original_name = original_dependency_dict[ 'name' ]
original_type = original_dependency_dict[ 'type' ]
original_version = original_dependency_dict[ 'version' ]
# Locate the appropriate tool_dependency associated with the repository.
tool_dependency = None
for tool_dependency in repository.tool_dependencies:
if tool_dependency.name == original_name and tool_dependency.type == original_type and tool_dependency.version == original_version:
break
if tool_dependency and tool_dependency.can_update:
dependency_install_dir = tool_dependency.installation_directory( app )
removed_from_disk, error_message = tool_dependency_util.remove_tool_dependency_installation_directory( dependency_install_dir )
if removed_from_disk:
context = app.install_model.context
new_dependency_name = None
new_dependency_type = None
new_dependency_version = None
for new_dependency_key, new_dependency_val_dict in new_dependencies_dict.items():
# Match on name only, hopefully this will be enough!
if original_name == new_dependency_val_dict[ 'name' ]:
new_dependency_name = new_dependency_val_dict[ 'name' ]
new_dependency_type = new_dependency_val_dict[ 'type' ]
new_dependency_version = new_dependency_val_dict[ 'version' ]
break
if new_dependency_name and new_dependency_type and new_dependency_version:
# Update all attributes of the tool_dependency record in the database.
log.debug( "Updating version %s of tool dependency %s %s to have new version %s and type %s." % \
( str( tool_dependency.version ),
str( tool_dependency.type ),
str( tool_dependency.name ),
str( new_dependency_version ),
str( new_dependency_type ) ) )
tool_dependency.type = new_dependency_type
tool_dependency.version = new_dependency_version
tool_dependency.status = app.install_model.ToolDependency.installation_status.UNINSTALLED
tool_dependency.error_message = None
context.add( tool_dependency )
context.flush()
new_tool_dependency = tool_dependency
else:
# We have no new tool dependency definition based on a matching dependency name, so remove
# the existing tool dependency record from the database.
log.debug( "Deleting version %s of tool dependency %s %s from the database since it is no longer defined." % \
( str( tool_dependency.version ), str( tool_dependency.type ), str( tool_dependency.name ) ) )
context.delete( tool_dependency )
context.flush()
return new_tool_dependency
def update_repository_dependencies_metadata( metadata, repository_dependency_tups, is_valid, description ):
if is_valid:
repository_dependencies_dict = metadata.get( 'repository_dependencies', None )
lib/tool_shed/util/tool_dependency_util.py
+1 -340
View File
@@ -1,48 +1,14 @@
import logging
import os
import shutil
from galaxy import eggs
from galaxy import util
from galaxy.model.orm import and_
from galaxy.model.orm import or_
import tool_shed.util.shed_util_common as suc
import tool_shed.repository_types.util as rt_util
from tool_shed.util import basic_util
from tool_shed.util import hg_util
from tool_shed.util import xml_util
log = logging.getLogger( __name__ )
def add_installation_directories_to_tool_dependencies( app, tool_dependencies ):
"""
Determine the path to the installation directory for each of the received tool dependencies. This path will be
displayed within the tool dependencies container on the select_tool_panel_section or reselect_tool_panel_section
pages when installing or reinstalling repositories that contain tools with the defined tool dependencies. The
list of tool dependencies may be associated with more than a single repository.
"""
for dependency_key, requirements_dict in tool_dependencies.items():
if dependency_key in [ 'set_environment' ]:
continue
repository_name = requirements_dict.get( 'repository_name', 'unknown' )
repository_owner = requirements_dict.get( 'repository_owner', 'unknown' )
changeset_revision = requirements_dict.get( 'changeset_revision', 'unknown' )
dependency_name = requirements_dict[ 'name' ]
version = requirements_dict[ 'version' ]
type = requirements_dict[ 'type' ]
if app.config.tool_dependency_dir:
root_dir = app.config.tool_dependency_dir
else:
root_dir = '<set your tool_dependency_dir in your Galaxy configuration file>'
install_dir = os.path.join( root_dir,
dependency_name,
version,
repository_owner,
repository_name,
changeset_revision )
requirements_dict[ 'install_dir' ] = install_dir
tool_dependencies[ dependency_key ] = requirements_dict
return tool_dependencies
def create_or_update_tool_dependency( app, tool_shed_repository, name, version, type, status, set_status=True ):
"""Create or update a tool_dependency record in the Galaxy database."""
# Called from Galaxy (never the tool shed) when a new repository is being installed or when an uninstalled
@@ -123,82 +89,6 @@ def create_tool_dependency_objects( app, tool_shed_repository, relative_install_
tool_dependency_objects.append( tool_dependency )
return tool_dependency_objects
def generate_message_for_invalid_tool_dependencies( metadata_dict ):
"""
Tool dependency definitions can only be invalid if they include a definition for a complex repository dependency and the repository
dependency definition is invalid. This method retrieves the error message associated with the invalid tool dependency for display
in the caller.
"""
message = ''
if metadata_dict:
invalid_tool_dependencies = metadata_dict.get( 'invalid_tool_dependencies', None )
if invalid_tool_dependencies:
for td_key, requirement_dict in invalid_tool_dependencies.items():
error = requirement_dict.get( 'error', None )
if error:
message = '%s ' % str( error )
return message
def generate_message_for_orphan_tool_dependencies( trans, repository, metadata_dict ):
"""
The designation of a ToolDependency into the "orphan" category has evolved over time, and is significantly restricted since the
introduction of the TOOL_DEPENDENCY_DEFINITION repository type. This designation is still critical, however, in that it handles
the case where a repository contains both tools and a tool_dependencies.xml file, but the definition in the tool_dependencies.xml
file is in no way related to anything defined by any of the contained tool's requirements tag sets. This is important in that it
is often a result of a typo (e.g., dependency name or version) that differs between the tool dependency definition within the
tool_dependencies.xml file and what is defined in the tool config's <requirements> tag sets. In these cases, the user should be
presented with a warning message, and this warning message is is in fact displayed if the following is_orphan attribute is True.
This is tricky because in some cases it may be intentional, and tool dependencies that are categorized as "orphan" are in fact valid.
"""
has_orphan_package_dependencies = False
has_orphan_set_environment_dependencies = False
message = ''
package_orphans_str = ''
set_environment_orphans_str = ''
# Tool dependencies are categorized as orphan only if the repository contains tools.
if metadata_dict:
tools = metadata_dict.get( 'tools', [] )
invalid_tools = metadata_dict.get( 'invalid_tools', [] )
tool_dependencies = metadata_dict.get( 'tool_dependencies', {} )
# The use of the orphan_tool_dependencies category in metadata has been deprecated, but we still need to check in case
# the metadata is out of date.
orphan_tool_dependencies = metadata_dict.get( 'orphan_tool_dependencies', {} )
# Updating should cause no problems here since a tool dependency cannot be included in both dictionaries.
tool_dependencies.update( orphan_tool_dependencies )
if tool_dependencies and ( tools or invalid_tools ):
for td_key, requirements_dict in tool_dependencies.items():
if td_key == 'set_environment':
# "set_environment": [{"name": "R_SCRIPT_PATH", "type": "set_environment"}]
for env_requirements_dict in requirements_dict:
name = env_requirements_dict[ 'name' ]
type = env_requirements_dict[ 'type' ]
if tool_dependency_is_orphan( type, name, None, tools ):
if not has_orphan_set_environment_dependencies:
has_orphan_set_environment_dependencies = True
set_environment_orphans_str += "<b>* name:</b> %s, <b>type:</b> %s<br/>" % ( str( name ), str( type ) )
else:
# "R/2.15.1": {"name": "R", "readme": "some string", "type": "package", "version": "2.15.1"}
name = requirements_dict[ 'name' ]
type = requirements_dict[ 'type' ]
version = requirements_dict[ 'version' ]
if tool_dependency_is_orphan( type, name, version, tools ):
if not has_orphan_package_dependencies:
has_orphan_package_dependencies = True
package_orphans_str += "<b>* name:</b> %s, <b>type:</b> %s, <b>version:</b> %s<br/>" % \
( str( name ), str( type ), str( version ) )
if has_orphan_package_dependencies:
message += "The settings for <b>name</b>, <b>version</b> and <b>type</b> from a contained tool configuration file's "
message += "<b>requirement</b> tag does not match the information for the following tool dependency definitions in the "
message += "<b>tool_dependencies.xml</b> file, so these tool dependencies have no relationship with any tools within "
message += "this repository.<br/>"
message += package_orphans_str
if has_orphan_set_environment_dependencies:
message += "The settings for <b>name</b> and <b>type</b> from a contained tool configuration file's <b>requirement</b> tag "
message += "does not match the information for the following tool dependency definitions in the <b>tool_dependencies.xml</b> "
message += "file, so these tool dependencies have no relationship with any tools within this repository.<br/>"
message += set_environment_orphans_str
return message
def get_download_url_for_platform( url_templates, platform_info_dict ):
'''
Compare the dict returned by get_platform_info() with the values specified in the url_template element. Return
@@ -230,56 +120,6 @@ def get_download_url_for_platform( url_templates, platform_info_dict ):
return url_template
return None
def get_installed_and_missing_tool_dependencies_for_installed_repository( app, repository, all_tool_dependencies ):
"""
Return the lists of installed tool dependencies and missing tool dependencies for a Tool Shed
repository that has been installed into Galaxy.
"""
if all_tool_dependencies:
tool_dependencies = {}
missing_tool_dependencies = {}
for td_key, val in all_tool_dependencies.items():
if td_key in [ 'set_environment' ]:
for index, td_info_dict in enumerate( val ):
name = td_info_dict[ 'name' ]
version = None
type = td_info_dict[ 'type' ]
tool_dependency = get_tool_dependency_by_name_type_repository( app, repository, name, type )
if tool_dependency:
td_info_dict[ 'repository_id' ] = repository.id
td_info_dict[ 'tool_dependency_id' ] = tool_dependency.id
if tool_dependency.status:
tool_dependency_status = str( tool_dependency.status )
else:
tool_dependency_status = 'Never installed'
td_info_dict[ 'status' ] = tool_dependency_status
val[ index ] = td_info_dict
if tool_dependency.status == app.install_model.ToolDependency.installation_status.INSTALLED:
tool_dependencies[ td_key ] = val
else:
missing_tool_dependencies[ td_key ] = val
else:
name = val[ 'name' ]
version = val[ 'version' ]
type = val[ 'type' ]
tool_dependency = get_tool_dependency_by_name_version_type_repository( app, repository, name, version, type )
if tool_dependency:
val[ 'repository_id' ] = repository.id
val[ 'tool_dependency_id' ] = tool_dependency.id
if tool_dependency.status:
tool_dependency_status = str( tool_dependency.status )
else:
tool_dependency_status = 'Never installed'
val[ 'status' ] = tool_dependency_status
if tool_dependency.status == app.install_model.ToolDependency.installation_status.INSTALLED:
tool_dependencies[ td_key ] = val
else:
missing_tool_dependencies[ td_key ] = val
else:
tool_dependencies = None
missing_tool_dependencies = None
return tool_dependencies, missing_tool_dependencies
def get_platform_info_dict():
'''Return a dict with information about the current platform.'''
platform_dict = {}
@@ -288,18 +128,6 @@ def get_platform_info_dict():
platform_dict[ 'architecture' ] = machine.lower()
return platform_dict
def get_required_repository_package_env_sh_path( app, package_name, package_version, required_repository ):
"""Return path to env.sh file in required repository if the required repository has been installed."""
env_sh_file_dir = get_tool_dependency_install_dir( app=app,
repository_name=required_repository.name,
repository_owner=required_repository.owner,
repository_changeset_revision=required_repository.installed_changeset_revision,
tool_dependency_type='package',
tool_dependency_name=package_name,
tool_dependency_version=package_version )
env_sh_file_path = os.path.join( env_sh_file_dir, 'env.sh' )
return env_sh_file_path
def get_tool_dependency( trans, id ):
"""Get a tool_dependency from the database via id"""
return trans.install_model.context.query( trans.install_model.ToolDependency ).get( trans.security.decode_id( id ) )
@@ -514,68 +342,6 @@ def parse_package_elem( package_elem, platform_info_dict=None, include_after_ins
continue
return actions_elem_tuples
def populate_tool_dependencies_dicts( app, tool_shed_url, tool_path, repository_installed_tool_dependencies,
repository_missing_tool_dependencies, required_repo_info_dicts ):
"""
Return the populated installed_tool_dependencies and missing_tool_dependencies dictionaries for all
repositories defined by entries in the received required_repo_info_dicts.
"""
installed_tool_dependencies = None
missing_tool_dependencies = None
if repository_installed_tool_dependencies is None:
repository_installed_tool_dependencies = {}
else:
# Add the install_dir attribute to the tool_dependencies.
repository_installed_tool_dependencies = \
add_installation_directories_to_tool_dependencies( app=app,
tool_dependencies=repository_installed_tool_dependencies )
if repository_missing_tool_dependencies is None:
repository_missing_tool_dependencies = {}
else:
# Add the install_dir attribute to the tool_dependencies.
repository_missing_tool_dependencies = \
add_installation_directories_to_tool_dependencies( app=app,
tool_dependencies=repository_missing_tool_dependencies )
if required_repo_info_dicts:
# Handle the tool dependencies defined for each of the repository's repository dependencies.
for rid in required_repo_info_dicts:
for name, repo_info_tuple in rid.items():
description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \
suc.get_repo_info_tuple_contents( repo_info_tuple )
if tool_dependencies:
# Add the install_dir attribute to the tool_dependencies.
tool_dependencies = add_installation_directories_to_tool_dependencies( app=app,
tool_dependencies=tool_dependencies )
# The required_repository may have been installed with a different changeset revision.
required_repository, installed_changeset_revision = \
suc.repository_was_previously_installed( app, tool_shed_url, name, repo_info_tuple, from_tip=False )
if required_repository:
required_repository_installed_tool_dependencies, required_repository_missing_tool_dependencies = \
get_installed_and_missing_tool_dependencies_for_installed_repository( app,
required_repository,
tool_dependencies )
if required_repository_installed_tool_dependencies:
# Add the install_dir attribute to the tool_dependencies.
required_repository_installed_tool_dependencies = \
add_installation_directories_to_tool_dependencies( app=app,
tool_dependencies=required_repository_installed_tool_dependencies )
for td_key, td_dict in required_repository_installed_tool_dependencies.items():
if td_key not in repository_installed_tool_dependencies:
repository_installed_tool_dependencies[ td_key ] = td_dict
if required_repository_missing_tool_dependencies:
# Add the install_dir attribute to the tool_dependencies.
required_repository_missing_tool_dependencies = \
add_installation_directories_to_tool_dependencies( app=app,
tool_dependencies=required_repository_missing_tool_dependencies )
for td_key, td_dict in required_repository_missing_tool_dependencies.items():
if td_key not in repository_missing_tool_dependencies:
repository_missing_tool_dependencies[ td_key ] = td_dict
if repository_installed_tool_dependencies:
installed_tool_dependencies = repository_installed_tool_dependencies
if repository_missing_tool_dependencies:
missing_tool_dependencies = repository_missing_tool_dependencies
return installed_tool_dependencies, missing_tool_dependencies
def remove_tool_dependency( app, tool_dependency ):
"""The received tool_dependency must be in an error state."""
context = app.install_model.context
@@ -617,108 +383,3 @@ def set_tool_dependency_attributes( app, tool_dependency, status, error_message=
sa_session.add( tool_dependency )
sa_session.flush()
return tool_dependency
def sync_database_with_file_system( app, tool_shed_repository, tool_dependency_name, tool_dependency_version,
tool_dependency_install_dir, tool_dependency_type='package' ):
"""
The installation directory defined by the received tool_dependency_install_dir exists, so check for
the presence of INSTALLATION_LOG. If the files exists, we'll assume the tool dependency is installed,
but not necessarily successfully (it could be in an error state on disk. However, we can justifiably
assume here that no matter the state, an associated database record will exist.
"""
# This method should be reached very rarely. It implies that either the Galaxy environment became corrupted (i.e.,
# the database records for installed tool dependencies is not synchronized with tool dependencies on disk) or the Tool
# Shed's install and test framework is running. The Tool Shed's install and test framework installs repositories
# in 2 stages, those of type tool_dependency_definition followed by those containing valid tools and tool functional
# test components.
log.debug( "Synchronizing the database with the file system..." )
try:
log.debug( "The value of app.config.running_functional_tests is: %s" % str( app.config.running_functional_tests ) )
except:
pass
sa_session = app.install_model.context
can_install_tool_dependency = False
tool_dependency = get_tool_dependency_by_name_version_type_repository( app,
tool_shed_repository,
tool_dependency_name,
tool_dependency_version,
tool_dependency_type )
if tool_dependency.status == app.install_model.ToolDependency.installation_status.INSTALLING:
# The tool dependency is in an Installing state, so we don't want to do anything to it. If the tool
# dependency is being installed by someone else, we don't want to interfere with that. This assumes
# the installation by "someone else" is not hung in an Installing state, which is a weakness if that
# "someone else" never repaired it.
log.debug( 'Skipping installation of tool dependency %s version %s because it has a status of %s' % \
( str( tool_dependency.name ), str( tool_dependency.version ), str( tool_dependency.status ) ) )
else:
# We have a pre-existing installation directory on the file system, but our associated database record is
# in a state that allowed us to arrive here. At this point, we'll inspect the installation directory to
# see if we have a "proper installation" and if so, synchronize the database record rather than reinstalling
# the dependency if we're "running_functional_tests". If we're not "running_functional_tests, we'll set
# the tool dependency's installation status to ERROR.
tool_dependency_installation_directory_contents = os.listdir( tool_dependency_install_dir )
if basic_util.INSTALLATION_LOG in tool_dependency_installation_directory_contents:
# Since this tool dependency's installation directory contains an installation log, we consider it to be
# installed. In some cases the record may be missing from the database due to some activity outside of
# the control of the Tool Shed. Since a new record was created for it and we don't know the state of the
# files on disk, we will set it to an error state (unless we are running Tool Shed functional tests - see
# below).
log.debug( 'Skipping installation of tool dependency %s version %s because it is installed in %s' % \
( str( tool_dependency.name ), str( tool_dependency.version ), str( tool_dependency_install_dir ) ) )
if app.config.running_functional_tests:
# If we are running functional tests, the state will be set to Installed because previously compiled
# tool dependencies are not deleted by default, from the "install and test" framework..
tool_dependency.status = app.install_model.ToolDependency.installation_status.INSTALLED
else:
error_message = 'The installation directory for this tool dependency had contents but the database had no record. '
error_message += 'The installation log may show this tool dependency to be correctly installed, but due to the '
error_message += 'missing database record it is now being set to Error.'
tool_dependency.status = app.install_model.ToolDependency.installation_status.ERROR
tool_dependency.error_message = error_message
else:
error_message = '\nInstallation path %s for tool dependency %s version %s exists, but the expected file %s' % \
( str( tool_dependency_install_dir ),
str( tool_dependency_name ),
str( tool_dependency_version ),
str( basic_util.INSTALLATION_LOG ) )
error_message += ' is missing. This indicates an installation error so the tool dependency is being'
error_message += ' prepared for re-installation.'
print error_message
tool_dependency.status = app.install_model.ToolDependency.installation_status.NEVER_INSTALLED
basic_util.remove_dir( tool_dependency_install_dir )
can_install_tool_dependency = True
sa_session.add( tool_dependency )
sa_session.flush()
try:
log.debug( "Returning from sync_database_with_file_system with tool_dependency %s, can_install_tool_dependency %s." % \
( str( tool_dependency.name ), str( can_install_tool_dependency ) ) )
except Exception, e:
log.debug( str( e ) )
return tool_dependency, can_install_tool_dependency
def tool_dependency_is_orphan( type, name, version, tools ):
"""
Determine if the combination of the received type, name and version is defined in the <requirement> tag for at least one tool in the
received list of tools. If not, the tool dependency defined by the combination is considered an orphan in its repository in the tool
shed.
"""
if type == 'package':
if name and version:
for tool_dict in tools:
requirements = tool_dict.get( 'requirements', [] )
for requirement_dict in requirements:
req_name = requirement_dict.get( 'name', None )
req_version = requirement_dict.get( 'version', None )
req_type = requirement_dict.get( 'type', None )
if req_name == name and req_version == version and req_type == type:
return False
elif type == 'set_environment':
if name:
for tool_dict in tools:
requirements = tool_dict.get( 'requirements', [] )
for requirement_dict in requirements:
req_name = requirement_dict.get( 'name', None )
req_type = requirement_dict.get( 'type', None )
if req_name == name and req_type == type:
return False
return True