mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
Merge pull request #14810 from jmchilton/cleanup_ts_controllers_3
More tool shed dependency and dead code cleanup.
This commit is contained in:
Björn Grüning
@@ -0,0 +1,292 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
from galaxy import util
|
||||
from galaxy.tool_shed.util import utility_container_manager
|
||||
from galaxy.util.tool_shed.common_util import parse_repository_dependency_tuple
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DependencyDisplayer:
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
|
||||
def _add_installation_directories_to_tool_dependencies(self, tool_dependencies):
|
||||
"""
|
||||
Determine the path to the installation directory for each of the received
|
||||
tool dependencies. This path will be displayed within the tool dependencies
|
||||
container on the select_tool_panel_section or reselect_tool_panel_section
|
||||
pages when installing or reinstalling repositories that contain tools with
|
||||
the defined tool dependencies. The list of tool dependencies may be associated
|
||||
with more than a single repository.
|
||||
"""
|
||||
for dependency_key, requirements_dict in tool_dependencies.items():
|
||||
if dependency_key in ["set_environment"]:
|
||||
continue
|
||||
repository_name = requirements_dict.get("repository_name", "unknown")
|
||||
repository_owner = requirements_dict.get("repository_owner", "unknown")
|
||||
changeset_revision = requirements_dict.get("changeset_revision", "unknown")
|
||||
dependency_name = requirements_dict["name"]
|
||||
version = requirements_dict["version"]
|
||||
if self.app.tool_dependency_dir:
|
||||
root_dir = self.app.tool_dependency_dir
|
||||
else:
|
||||
root_dir = "<set your tool_dependency_dir in your Galaxy configuration file>"
|
||||
install_dir = os.path.join(
|
||||
root_dir, dependency_name, version, repository_owner, repository_name, changeset_revision
|
||||
)
|
||||
requirements_dict["install_dir"] = install_dir
|
||||
tool_dependencies[dependency_key] = requirements_dict
|
||||
return tool_dependencies
|
||||
|
||||
def generate_message_for_invalid_repository_dependencies(self, metadata_dict, error_from_tuple=False):
|
||||
"""
|
||||
Get or generate and return an error message associated with an invalid repository dependency.
|
||||
"""
|
||||
message = ""
|
||||
if metadata_dict:
|
||||
if error_from_tuple:
|
||||
# Return the error messages associated with a set of one or more invalid repository
|
||||
# dependency tuples.
|
||||
invalid_repository_dependencies_dict = metadata_dict.get("invalid_repository_dependencies", None)
|
||||
if invalid_repository_dependencies_dict is not None:
|
||||
invalid_repository_dependencies = invalid_repository_dependencies_dict.get(
|
||||
"invalid_repository_dependencies", []
|
||||
)
|
||||
for repository_dependency_tup in invalid_repository_dependencies:
|
||||
(
|
||||
toolshed,
|
||||
name,
|
||||
owner,
|
||||
changeset_revision,
|
||||
prior_installation_required,
|
||||
only_if_compiling_contained_td,
|
||||
error,
|
||||
) = parse_repository_dependency_tuple(repository_dependency_tup, contains_error=True)
|
||||
if error:
|
||||
message += f"{error} "
|
||||
else:
|
||||
# The complete dependency hierarchy could not be determined for a repository being installed into
|
||||
# Galaxy. This is likely due to invalid repository dependency definitions, so we'll get them from
|
||||
# the metadata and parse them for display in an error message. This will hopefully communicate the
|
||||
# problem to the user in such a way that a resolution can be determined.
|
||||
message += (
|
||||
"The complete dependency hierarchy could not be determined for this repository, so no required "
|
||||
)
|
||||
message += "repositories will not be installed. This is likely due to invalid repository dependency definitions. "
|
||||
repository_dependencies_dict = metadata_dict.get("repository_dependencies", None)
|
||||
if repository_dependencies_dict is not None:
|
||||
rd_tups = repository_dependencies_dict.get("repository_dependencies", None)
|
||||
if rd_tups is not None:
|
||||
message += "Here are the attributes of the dependencies defined for this repository to help determine the "
|
||||
message += "cause of this problem.<br/>"
|
||||
message += '<table cellpadding="2" cellspacing="2">'
|
||||
message += (
|
||||
"<tr><th>Tool shed</th><th>Repository name</th><th>Owner</th><th>Changeset revision</th>"
|
||||
)
|
||||
message += "<th>Prior install required</th></tr>"
|
||||
for rd_tup in rd_tups:
|
||||
(
|
||||
tool_shed,
|
||||
name,
|
||||
owner,
|
||||
changeset_revision,
|
||||
pir,
|
||||
oicct,
|
||||
) = parse_repository_dependency_tuple(rd_tup)
|
||||
if util.asbool(pir):
|
||||
pir_str = "True"
|
||||
else:
|
||||
pir_str = ""
|
||||
message += f"<tr><td>{tool_shed}</td><td>{name}</td><td>{owner}</td><td>{changeset_revision}</td><td>{pir_str}</td></tr>"
|
||||
message += "</table>"
|
||||
return message
|
||||
|
||||
def generate_message_for_invalid_tool_dependencies(self, metadata_dict):
|
||||
"""
|
||||
Tool dependency definitions can only be invalid if they include a definition for a complex
|
||||
repository dependency and the repository dependency definition is invalid. This method
|
||||
retrieves the error message associated with the invalid tool dependency for display in the
|
||||
caller.
|
||||
"""
|
||||
message = ""
|
||||
if metadata_dict:
|
||||
invalid_tool_dependencies = metadata_dict.get("invalid_tool_dependencies", None)
|
||||
if invalid_tool_dependencies:
|
||||
for requirement_dict in invalid_tool_dependencies.values():
|
||||
error = requirement_dict.get("error", None)
|
||||
if error:
|
||||
message = f"{error} "
|
||||
return message
|
||||
|
||||
def generate_message_for_orphan_tool_dependencies(self, repository, metadata_dict):
|
||||
"""
|
||||
The designation of a ToolDependency into the "orphan" category has evolved over time,
|
||||
and is significantly restricted since the introduction of the TOOL_DEPENDENCY_DEFINITION
|
||||
repository type. This designation is still critical, however, in that it handles the
|
||||
case where a repository contains both tools and a tool_dependencies.xml file, but the
|
||||
definition in the tool_dependencies.xml file is in no way related to anything defined
|
||||
by any of the contained tool's requirements tag sets. This is important in that it is
|
||||
often a result of a typo (e.g., dependency name or version) that differs between the tool
|
||||
dependency definition within the tool_dependencies.xml file and what is defined in the
|
||||
tool config's <requirements> tag sets. In these cases, the user should be presented with
|
||||
a warning message, and this warning message is is in fact displayed if the following
|
||||
is_orphan attribute is True. This is tricky because in some cases it may be intentional,
|
||||
and tool dependencies that are categorized as "orphan" are in fact valid.
|
||||
"""
|
||||
has_orphan_package_dependencies = False
|
||||
has_orphan_set_environment_dependencies = False
|
||||
message = ""
|
||||
package_orphans_str = ""
|
||||
set_environment_orphans_str = ""
|
||||
# Tool dependencies are categorized as orphan only if the repository contains tools.
|
||||
if metadata_dict:
|
||||
tools = metadata_dict.get("tools", [])
|
||||
tool_dependencies = metadata_dict.get("tool_dependencies", {})
|
||||
# The use of the orphan_tool_dependencies category in metadata has been deprecated,
|
||||
# but we still need to check in case the metadata is out of date.
|
||||
orphan_tool_dependencies = metadata_dict.get("orphan_tool_dependencies", {})
|
||||
# Updating should cause no problems here since a tool dependency cannot be included
|
||||
# in both dictionaries.
|
||||
tool_dependencies.update(orphan_tool_dependencies)
|
||||
if tool_dependencies and tools:
|
||||
for td_key, requirements_dict in tool_dependencies.items():
|
||||
if td_key == "set_environment":
|
||||
# "set_environment": [{"name": "R_SCRIPT_PATH", "type": "set_environment"}]
|
||||
for env_requirements_dict in requirements_dict:
|
||||
name = env_requirements_dict["name"]
|
||||
type = env_requirements_dict["type"]
|
||||
if self._tool_dependency_is_orphan(type, name, None, tools):
|
||||
if not has_orphan_set_environment_dependencies:
|
||||
has_orphan_set_environment_dependencies = True
|
||||
set_environment_orphans_str += f"<b>* name:</b> {name}, <b>type:</b> {type}<br/>"
|
||||
else:
|
||||
# "R/2.15.1": {"name": "R", "readme": "some string", "type": "package", "version": "2.15.1"}
|
||||
name = requirements_dict["name"]
|
||||
type = requirements_dict["type"]
|
||||
version = requirements_dict["version"]
|
||||
if self._tool_dependency_is_orphan(type, name, version, tools):
|
||||
if not has_orphan_package_dependencies:
|
||||
has_orphan_package_dependencies = True
|
||||
package_orphans_str += (
|
||||
f"<b>* name:</b> {name}, <b>type:</b> {type}, <b>version:</b> {version}<br/>"
|
||||
)
|
||||
if has_orphan_package_dependencies:
|
||||
message += "The settings for <b>name</b>, <b>version</b> and <b>type</b> from a "
|
||||
message += "contained tool configuration file's <b>requirement</b> tag does not match "
|
||||
message += "the information for the following tool dependency definitions in the "
|
||||
message += "<b>tool_dependencies.xml</b> file, so these tool dependencies have no "
|
||||
message += "relationship with any tools within this repository.<br/>"
|
||||
message += package_orphans_str
|
||||
if has_orphan_set_environment_dependencies:
|
||||
message += "The settings for <b>name</b> and <b>type</b> from a contained tool "
|
||||
message += "configuration file's <b>requirement</b> tag does not match the information "
|
||||
message += "for the following tool dependency definitions in the <b>tool_dependencies.xml</b> "
|
||||
message += "file, so these tool dependencies have no relationship with any tools within "
|
||||
message += "this repository.<br/>"
|
||||
message += set_environment_orphans_str
|
||||
return message
|
||||
|
||||
def populate_containers_dict_from_repository_metadata(self, repository):
|
||||
"""
|
||||
Retrieve necessary information from the received repository's metadata to populate the
|
||||
containers_dict for display. This method is called only from Galaxy (not the tool shed)
|
||||
when displaying repository dependencies for installed repositories and when displaying
|
||||
them for uninstalled repositories that are being reinstalled.
|
||||
"""
|
||||
metadata = repository.metadata_
|
||||
if metadata:
|
||||
# Handle repository dependencies.
|
||||
(
|
||||
installed_repository_dependencies,
|
||||
missing_repository_dependencies,
|
||||
) = self.app.installed_repository_manager.get_installed_and_missing_repository_dependencies(repository)
|
||||
# Handle the current repository's tool dependencies.
|
||||
repository_tool_dependencies = metadata.get("tool_dependencies", None)
|
||||
# Make sure to display missing tool dependencies as well.
|
||||
repository_invalid_tool_dependencies = metadata.get("invalid_tool_dependencies", None)
|
||||
if repository_invalid_tool_dependencies is not None:
|
||||
if repository_tool_dependencies is None:
|
||||
repository_tool_dependencies = {}
|
||||
repository_tool_dependencies.update(repository_invalid_tool_dependencies)
|
||||
gucm = GalaxyUtilityContainerManager(self.app)
|
||||
containers_dict = gucm.build_repository_containers(
|
||||
missing_repository_dependencies=missing_repository_dependencies,
|
||||
repository_dependencies=installed_repository_dependencies,
|
||||
)
|
||||
else:
|
||||
containers_dict = dict(
|
||||
repository_dependencies=None,
|
||||
)
|
||||
return containers_dict
|
||||
|
||||
def _tool_dependency_is_orphan(self, type, name, version, tools):
|
||||
"""
|
||||
Determine if the combination of the received type, name and version is defined in the <requirement>
|
||||
tag for at least one tool in the received list of tools. If not, the tool dependency defined by the
|
||||
combination is considered an orphan in its repository in the tool shed.
|
||||
"""
|
||||
if type == "package":
|
||||
if name and version:
|
||||
for tool_dict in tools:
|
||||
requirements = tool_dict.get("requirements", [])
|
||||
for requirement_dict in requirements:
|
||||
req_name = requirement_dict.get("name", None)
|
||||
req_version = requirement_dict.get("version", None)
|
||||
req_type = requirement_dict.get("type", None)
|
||||
if req_name == name and req_version == version and req_type == type:
|
||||
return False
|
||||
elif type == "set_environment":
|
||||
if name:
|
||||
for tool_dict in tools:
|
||||
requirements = tool_dict.get("requirements", [])
|
||||
for requirement_dict in requirements:
|
||||
req_name = requirement_dict.get("name", None)
|
||||
req_type = requirement_dict.get("type", None)
|
||||
if req_name == name and req_type == type:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class GalaxyUtilityContainerManager(utility_container_manager.UtilityContainerManager):
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
|
||||
def build_repository_containers(
|
||||
self,
|
||||
missing_repository_dependencies,
|
||||
repository_dependencies,
|
||||
):
|
||||
"""
|
||||
Return a dictionary of containers for the received repository's dependencies and readme files for
|
||||
display during installation to Galaxy.
|
||||
"""
|
||||
containers_dict = dict(
|
||||
repository_dependencies=None,
|
||||
missing_repository_dependencies=None,
|
||||
)
|
||||
# Some of the tool dependency folders will include links to display tool dependency information, and
|
||||
# some of these links require the repository id. However we need to be careful because sometimes the
|
||||
# repository object is None.
|
||||
try:
|
||||
folder_id = 0
|
||||
# Installed repository dependencies container.
|
||||
if repository_dependencies:
|
||||
label = "Installed repository dependencies"
|
||||
folder_id, repository_dependencies_root_folder = self.build_repository_dependencies_folder(
|
||||
folder_id=folder_id, repository_dependencies=repository_dependencies, label=label, installed=True
|
||||
)
|
||||
containers_dict["repository_dependencies"] = repository_dependencies_root_folder
|
||||
# Missing repository dependencies container.
|
||||
if missing_repository_dependencies:
|
||||
folder_id, missing_repository_dependencies_root_folder = self.build_repository_dependencies_folder(
|
||||
folder_id=folder_id,
|
||||
repository_dependencies=missing_repository_dependencies,
|
||||
label="Missing repository dependencies",
|
||||
installed=False,
|
||||
)
|
||||
containers_dict["missing_repository_dependencies"] = missing_repository_dependencies_root_folder
|
||||
except Exception as e:
|
||||
log.debug(f"Exception in build_repository_containers: {str(e)}")
|
||||
return containers_dict
|
||||
@@ -12,13 +12,19 @@ from galaxy.exceptions import ConfigDoesNotAllowException
|
||||
from galaxy.tool_shed.galaxy_install import install_manager
|
||||
from galaxy.tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager
|
||||
from galaxy.tool_shed.galaxy_install.tools import tool_panel_manager
|
||||
from galaxy.tool_shed.util import tool_util
|
||||
from galaxy.tool_shed.util import dependency_display
|
||||
from galaxy.tool_shed.util.repository_util import (
|
||||
create_or_update_tool_shed_repository,
|
||||
get_absolute_path_to_file_in_repository,
|
||||
get_installed_tool_shed_repository,
|
||||
get_tool_shed_repository_by_id,
|
||||
)
|
||||
from galaxy.tool_shed.util.shed_util_common import (
|
||||
clean_dependency_relationships,
|
||||
get_ctx_rev,
|
||||
get_tool_panel_config_tool_path_install_dir,
|
||||
have_shed_tool_conf_for_install,
|
||||
)
|
||||
from galaxy.tool_util.deps import views
|
||||
from galaxy.util import unicodify
|
||||
from galaxy.util.tool_shed import (
|
||||
@@ -26,9 +32,12 @@ from galaxy.util.tool_shed import (
|
||||
encoding_util,
|
||||
)
|
||||
from galaxy.web.form_builder import CheckboxField
|
||||
from tool_shed.galaxy_install import dependency_display
|
||||
from tool_shed.util import shed_util_common as suc
|
||||
from tool_shed.util.repository_util import create_repo_info_dict
|
||||
from tool_shed.util.shed_util_common import (
|
||||
get_repository_type_from_tool_shed,
|
||||
get_requirements_from_repository,
|
||||
get_tool_dependency_definition_metadata_from_tool_shed,
|
||||
)
|
||||
from tool_shed.util.web_util import escape
|
||||
from .admin import AdminGalaxy
|
||||
|
||||
@@ -151,10 +160,8 @@ class AdminToolshed(AdminGalaxy):
|
||||
repository = get_installed_tool_shed_repository(trans.app, repository_id)
|
||||
if repository is None:
|
||||
return trans.show_error_message("Invalid repository specified.")
|
||||
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(trans.app, str(repository.tool_shed))
|
||||
description = kwd.get("description", repository.description)
|
||||
status = "ok"
|
||||
_, tool_path, _ = suc.get_tool_panel_config_tool_path_install_dir(trans.app, repository)
|
||||
if repository.in_error_state:
|
||||
message = "This repository is not installed correctly (see the <b>Repository installation error</b> below). Choose "
|
||||
message += "<b>Reset to install</b> from the <b>Repository Actions</b> menu, correct problems if necessary and try "
|
||||
@@ -171,14 +178,10 @@ class AdminToolshed(AdminGalaxy):
|
||||
message = "The repository information has been updated."
|
||||
dd = dependency_display.DependencyDisplayer(trans.app)
|
||||
containers_dict = dd.populate_containers_dict_from_repository_metadata(
|
||||
tool_shed_url=tool_shed_url,
|
||||
tool_path=tool_path,
|
||||
repository=repository,
|
||||
reinstalling=False,
|
||||
required_repo_info_dicts=None,
|
||||
)
|
||||
view = views.DependencyResolversView(self.app)
|
||||
tool_requirements_d = suc.get_requirements_from_repository(repository)
|
||||
tool_requirements_d = get_requirements_from_repository(repository)
|
||||
requirements_status = view.get_requirements_status(tool_requirements_d, repository.installed_tool_dependencies)
|
||||
management_dict = {
|
||||
"status": status,
|
||||
@@ -211,9 +214,9 @@ class AdminToolshed(AdminGalaxy):
|
||||
install_repository_dependencies = CheckboxField.is_checked(kwd.get("install_repository_dependencies", ""))
|
||||
install_tool_dependencies = CheckboxField.is_checked(kwd.get("install_tool_dependencies", ""))
|
||||
install_resolver_dependencies = CheckboxField.is_checked(kwd.get("install_resolver_dependencies", ""))
|
||||
if not suc.have_shed_tool_conf_for_install(trans.app):
|
||||
if not have_shed_tool_conf_for_install(trans.app):
|
||||
raise Exception("No valid shed tool configuration file available, please configure one")
|
||||
shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir(
|
||||
shed_tool_conf, tool_path, relative_install_dir = get_tool_panel_config_tool_path_install_dir(
|
||||
trans.app, tool_shed_repository
|
||||
)
|
||||
repository_clone_url = common_util.generate_clone_url_for_installed_repository(
|
||||
@@ -245,19 +248,19 @@ class AdminToolshed(AdminGalaxy):
|
||||
# Just in case the tool_section.id differs from tool_panel_section_id, which it shouldn't...
|
||||
tool_panel_section_id = str(tool_section.id)
|
||||
if tool_shed_repository.status == trans.install_model.ToolShedRepository.installation_status.UNINSTALLED:
|
||||
repository_type = suc.get_repository_type_from_tool_shed(
|
||||
repository_type = get_repository_type_from_tool_shed(
|
||||
trans.app, tool_shed_url, tool_shed_repository.name, tool_shed_repository.owner
|
||||
)
|
||||
if repository_type == rt_util.TOOL_DEPENDENCY_DEFINITION:
|
||||
# Repositories of type tool_dependency_definition must get the latest
|
||||
# metadata from the Tool Shed since they have only a single installable
|
||||
# revision.
|
||||
raw_text = suc.get_tool_dependency_definition_metadata_from_tool_shed(
|
||||
raw_text = get_tool_dependency_definition_metadata_from_tool_shed(
|
||||
trans.app, tool_shed_url, tool_shed_repository.name, tool_shed_repository.owner
|
||||
)
|
||||
new_meta = json.loads(raw_text)
|
||||
# Clean up old repository dependency and tool dependency relationships.
|
||||
suc.clean_dependency_relationships(trans, new_meta, tool_shed_repository, tool_shed_url)
|
||||
clean_dependency_relationships(trans, new_meta, tool_shed_repository, tool_shed_url)
|
||||
# The repository's status must be updated from 'Uninstalled' to 'New' when initiating reinstall
|
||||
# so the repository_installation_updater will function.
|
||||
tool_shed_repository = create_or_update_tool_shed_repository(
|
||||
@@ -273,7 +276,7 @@ class AdminToolshed(AdminGalaxy):
|
||||
owner=tool_shed_repository.owner,
|
||||
dist_to_shed=tool_shed_repository.dist_to_shed,
|
||||
)
|
||||
ctx_rev = suc.get_ctx_rev(
|
||||
ctx_rev = get_ctx_rev(
|
||||
trans.app,
|
||||
tool_shed_url,
|
||||
tool_shed_repository.name,
|
||||
@@ -368,172 +371,3 @@ class AdminToolshed(AdminGalaxy):
|
||||
except Exception:
|
||||
log.exception("Problem you know in here somewhere...")
|
||||
raise
|
||||
|
||||
@web.expose
|
||||
@web.require_admin
|
||||
@legacy_tool_shed_endpoint
|
||||
def reselect_tool_panel_section(self, trans, **kwd):
|
||||
"""
|
||||
Select or change the tool panel section to contain the tools included in the tool shed repository
|
||||
being reinstalled. If there are updates available for the repository in the tool shed, the
|
||||
tool_dependencies and repository_dependencies associated with the updated changeset revision will
|
||||
have been retrieved from the tool shed and passed in the received kwd. In this case, the stored
|
||||
tool shed repository metadata from the Galaxy database will not be used since it is outdated.
|
||||
"""
|
||||
message = ""
|
||||
status = "done"
|
||||
repository_id = kwd.get("id", None)
|
||||
latest_changeset_revision = kwd.get("latest_changeset_revision", None)
|
||||
latest_ctx_rev = kwd.get("latest_ctx_rev", None)
|
||||
tool_shed_repository = get_installed_tool_shed_repository(trans.app, repository_id)
|
||||
repository_clone_url = common_util.generate_clone_url_for_installed_repository(trans.app, tool_shed_repository)
|
||||
metadata = tool_shed_repository.metadata_
|
||||
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(
|
||||
trans.app, str(tool_shed_repository.tool_shed)
|
||||
)
|
||||
tool_path = tool_shed_repository.get_tool_relative_path(trans.app)[0]
|
||||
if latest_changeset_revision and latest_ctx_rev:
|
||||
# There are updates available in the tool shed for the repository, so use the received
|
||||
# dependency information which was retrieved from the tool shed.
|
||||
encoded_updated_repo_info_dict = kwd.get("updated_repo_info_dict", None)
|
||||
updated_repo_info_dict = encoding_util.tool_shed_decode(encoded_updated_repo_info_dict)
|
||||
readme_files_dict = updated_repo_info_dict.get("readme_files_dict", None)
|
||||
includes_data_managers = updated_repo_info_dict.get("includes_data_managers", False)
|
||||
includes_tool_dependencies = updated_repo_info_dict.get("includes_tool_dependencies", False)
|
||||
repo_info_dict = updated_repo_info_dict["repo_info_dict"]
|
||||
else:
|
||||
# There are no updates available from the tool shed for the repository, so use its locally stored metadata.
|
||||
includes_data_managers = False
|
||||
includes_tool_dependencies = False
|
||||
readme_files_dict = None
|
||||
tool_dependencies = None
|
||||
if metadata:
|
||||
if "data_manager" in metadata:
|
||||
includes_data_managers = True
|
||||
if "tool_dependencies" in metadata:
|
||||
includes_tool_dependencies = True
|
||||
# Since we're reinstalling, we need to send a request to the tool shed to get the README files.
|
||||
params = dict(
|
||||
name=tool_shed_repository.name,
|
||||
owner=tool_shed_repository.owner,
|
||||
changeset_revision=tool_shed_repository.installed_changeset_revision,
|
||||
)
|
||||
pathspec = ["repository", "get_readme_files"]
|
||||
raw_text = util.url_get(
|
||||
tool_shed_url,
|
||||
auth=self.app.tool_shed_registry.url_auth(tool_shed_url),
|
||||
pathspec=pathspec,
|
||||
params=params,
|
||||
)
|
||||
readme_files_dict = json.loads(raw_text)
|
||||
tool_dependencies = metadata.get("tool_dependencies", None)
|
||||
rdim = repository_dependency_manager.RepositoryDependencyInstallManager(trans.app)
|
||||
repository_dependencies = rdim.get_repository_dependencies_for_installed_tool_shed_repository(
|
||||
trans.app, tool_shed_repository
|
||||
)
|
||||
repo_info_dict = create_repo_info_dict(
|
||||
trans.app,
|
||||
repository_clone_url=repository_clone_url,
|
||||
changeset_revision=tool_shed_repository.installed_changeset_revision,
|
||||
ctx_rev=tool_shed_repository.ctx_rev,
|
||||
repository_owner=tool_shed_repository.owner,
|
||||
repository_name=tool_shed_repository.name,
|
||||
tool_dependencies=tool_dependencies,
|
||||
repository_dependencies=repository_dependencies,
|
||||
)
|
||||
irm = trans.app.installed_repository_manager
|
||||
dependencies_for_repository_dict = irm.get_dependencies_for_repository(
|
||||
tool_shed_url, repo_info_dict, includes_tool_dependencies, updating=True
|
||||
)
|
||||
includes_tool_dependencies = dependencies_for_repository_dict.get("includes_tool_dependencies", False)
|
||||
includes_tools = dependencies_for_repository_dict.get("includes_tools", False)
|
||||
includes_tools_for_display_in_tool_panel = dependencies_for_repository_dict.get(
|
||||
"includes_tools_for_display_in_tool_panel", False
|
||||
)
|
||||
installed_repository_dependencies = dependencies_for_repository_dict.get(
|
||||
"installed_repository_dependencies", None
|
||||
)
|
||||
installed_tool_dependencies = dependencies_for_repository_dict.get("installed_tool_dependencies", None)
|
||||
missing_repository_dependencies = dependencies_for_repository_dict.get("missing_repository_dependencies", None)
|
||||
missing_tool_dependencies = dependencies_for_repository_dict.get("missing_tool_dependencies", None)
|
||||
if installed_repository_dependencies or missing_repository_dependencies:
|
||||
has_repository_dependencies = True
|
||||
else:
|
||||
has_repository_dependencies = False
|
||||
if includes_tools_for_display_in_tool_panel:
|
||||
# Get the location in the tool panel in which the tools were originally loaded.
|
||||
if "tool_panel_section" in metadata:
|
||||
tool_panel_dict = metadata["tool_panel_section"]
|
||||
if tool_panel_dict:
|
||||
if tool_util.panel_entry_per_tool(tool_panel_dict):
|
||||
# The following forces everything to be loaded into 1 section (or no section) in the tool panel.
|
||||
tool_section_dicts = tool_panel_dict[next(iter(tool_panel_dict.keys()))]
|
||||
tool_section_dict = tool_section_dicts[0]
|
||||
original_section_name = tool_section_dict["name"]
|
||||
else:
|
||||
original_section_name = tool_panel_dict["name"]
|
||||
else:
|
||||
original_section_name = ""
|
||||
else:
|
||||
original_section_name = ""
|
||||
tool_panel_section_select_field = tool_util.build_tool_panel_section_select_field(trans.app)
|
||||
no_changes_check_box = CheckboxField("no_changes", value=True)
|
||||
if original_section_name:
|
||||
message += (
|
||||
"The tools contained in your <b>%s</b> repository were last loaded into the tool panel section <b>%s</b>. "
|
||||
% (escape(tool_shed_repository.name), original_section_name)
|
||||
)
|
||||
message += "Uncheck the <b>No changes</b> check box and select a different tool panel section to load the tools in a "
|
||||
message += "different section in the tool panel. "
|
||||
status = "warning"
|
||||
else:
|
||||
message += f"The tools contained in your <b>{escape(tool_shed_repository.name)}</b> repository were last loaded into the tool panel outside of any sections. "
|
||||
message += "Uncheck the <b>No changes</b> check box and select a tool panel section to load the tools into that section. "
|
||||
status = "warning"
|
||||
else:
|
||||
no_changes_check_box = None
|
||||
original_section_name = ""
|
||||
tool_panel_section_select_field = None
|
||||
shed_tool_conf_select_field = tool_util.build_shed_tool_conf_select_field(trans.app)
|
||||
dd = dependency_display.DependencyDisplayer(trans.app)
|
||||
containers_dict = dd.populate_containers_dict_for_new_install(
|
||||
tool_shed_url=tool_shed_url,
|
||||
tool_path=tool_path,
|
||||
readme_files_dict=readme_files_dict,
|
||||
installed_repository_dependencies=installed_repository_dependencies,
|
||||
missing_repository_dependencies=missing_repository_dependencies,
|
||||
installed_tool_dependencies=installed_tool_dependencies,
|
||||
missing_tool_dependencies=missing_tool_dependencies,
|
||||
updating=False,
|
||||
)
|
||||
# Since we're reinstalling we'll merge the list of missing repository dependencies into the list of
|
||||
# installed repository dependencies since each displayed repository dependency will display a status,
|
||||
# whether installed or missing.
|
||||
containers_dict = dd.merge_missing_repository_dependencies_to_installed_container(containers_dict)
|
||||
# Handle repository dependencies check box.
|
||||
install_repository_dependencies_check_box = CheckboxField("install_repository_dependencies", value=True)
|
||||
view = views.DependencyResolversView(self.app)
|
||||
if view.installable_resolvers:
|
||||
install_resolver_dependencies_check_box = CheckboxField("install_resolver_dependencies", value=True)
|
||||
else:
|
||||
install_resolver_dependencies_check_box = None
|
||||
return trans.fill_template(
|
||||
"/admin/tool_shed_repository/reselect_tool_panel_section.mako",
|
||||
repository=tool_shed_repository,
|
||||
no_changes_check_box=no_changes_check_box,
|
||||
original_section_name=original_section_name,
|
||||
includes_data_managers=includes_data_managers,
|
||||
includes_tools=includes_tools,
|
||||
includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel,
|
||||
includes_tool_dependencies=includes_tool_dependencies,
|
||||
has_repository_dependencies=has_repository_dependencies,
|
||||
install_repository_dependencies_check_box=install_repository_dependencies_check_box,
|
||||
install_resolver_dependencies_check_box=install_resolver_dependencies_check_box,
|
||||
containers_dict=containers_dict,
|
||||
tool_panel_section_select_field=tool_panel_section_select_field,
|
||||
shed_tool_conf_select_field=shed_tool_conf_select_field,
|
||||
encoded_repo_info_dict=encoding_util.tool_shed_encode(repo_info_dict),
|
||||
repo_info_dict=repo_info_dict,
|
||||
message=message,
|
||||
status=status,
|
||||
)
|
||||
|
||||
@@ -1,686 +0,0 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
|
||||
from galaxy import util
|
||||
from tool_shed.galaxy_install.utility_containers import GalaxyUtilityContainerManager
|
||||
from tool_shed.util import (
|
||||
common_util,
|
||||
container_util,
|
||||
readme_util,
|
||||
repository_util,
|
||||
tool_dependency_util,
|
||||
)
|
||||
from tool_shed.utility_containers import utility_container_manager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DependencyDisplayer:
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
|
||||
def add_installation_directories_to_tool_dependencies(self, tool_dependencies):
|
||||
"""
|
||||
Determine the path to the installation directory for each of the received
|
||||
tool dependencies. This path will be displayed within the tool dependencies
|
||||
container on the select_tool_panel_section or reselect_tool_panel_section
|
||||
pages when installing or reinstalling repositories that contain tools with
|
||||
the defined tool dependencies. The list of tool dependencies may be associated
|
||||
with more than a single repository.
|
||||
"""
|
||||
for dependency_key, requirements_dict in tool_dependencies.items():
|
||||
if dependency_key in ["set_environment"]:
|
||||
continue
|
||||
repository_name = requirements_dict.get("repository_name", "unknown")
|
||||
repository_owner = requirements_dict.get("repository_owner", "unknown")
|
||||
changeset_revision = requirements_dict.get("changeset_revision", "unknown")
|
||||
dependency_name = requirements_dict["name"]
|
||||
version = requirements_dict["version"]
|
||||
if self.app.tool_dependency_dir:
|
||||
root_dir = self.app.tool_dependency_dir
|
||||
else:
|
||||
root_dir = "<set your tool_dependency_dir in your Galaxy configuration file>"
|
||||
install_dir = os.path.join(
|
||||
root_dir, dependency_name, version, repository_owner, repository_name, changeset_revision
|
||||
)
|
||||
requirements_dict["install_dir"] = install_dir
|
||||
tool_dependencies[dependency_key] = requirements_dict
|
||||
return tool_dependencies
|
||||
|
||||
def generate_message_for_invalid_repository_dependencies(self, metadata_dict, error_from_tuple=False):
|
||||
"""
|
||||
Get or generate and return an error message associated with an invalid repository dependency.
|
||||
"""
|
||||
message = ""
|
||||
if metadata_dict:
|
||||
if error_from_tuple:
|
||||
# Return the error messages associated with a set of one or more invalid repository
|
||||
# dependency tuples.
|
||||
invalid_repository_dependencies_dict = metadata_dict.get("invalid_repository_dependencies", None)
|
||||
if invalid_repository_dependencies_dict is not None:
|
||||
invalid_repository_dependencies = invalid_repository_dependencies_dict.get(
|
||||
"invalid_repository_dependencies", []
|
||||
)
|
||||
for repository_dependency_tup in invalid_repository_dependencies:
|
||||
(
|
||||
toolshed,
|
||||
name,
|
||||
owner,
|
||||
changeset_revision,
|
||||
prior_installation_required,
|
||||
only_if_compiling_contained_td,
|
||||
error,
|
||||
) = common_util.parse_repository_dependency_tuple(
|
||||
repository_dependency_tup, contains_error=True
|
||||
)
|
||||
if error:
|
||||
message += f"{error} "
|
||||
else:
|
||||
# The complete dependency hierarchy could not be determined for a repository being installed into
|
||||
# Galaxy. This is likely due to invalid repository dependency definitions, so we'll get them from
|
||||
# the metadata and parse them for display in an error message. This will hopefully communicate the
|
||||
# problem to the user in such a way that a resolution can be determined.
|
||||
message += (
|
||||
"The complete dependency hierarchy could not be determined for this repository, so no required "
|
||||
)
|
||||
message += "repositories will not be installed. This is likely due to invalid repository dependency definitions. "
|
||||
repository_dependencies_dict = metadata_dict.get("repository_dependencies", None)
|
||||
if repository_dependencies_dict is not None:
|
||||
rd_tups = repository_dependencies_dict.get("repository_dependencies", None)
|
||||
if rd_tups is not None:
|
||||
message += "Here are the attributes of the dependencies defined for this repository to help determine the "
|
||||
message += "cause of this problem.<br/>"
|
||||
message += '<table cellpadding="2" cellspacing="2">'
|
||||
message += (
|
||||
"<tr><th>Tool shed</th><th>Repository name</th><th>Owner</th><th>Changeset revision</th>"
|
||||
)
|
||||
message += "<th>Prior install required</th></tr>"
|
||||
for rd_tup in rd_tups:
|
||||
(
|
||||
tool_shed,
|
||||
name,
|
||||
owner,
|
||||
changeset_revision,
|
||||
pir,
|
||||
oicct,
|
||||
) = common_util.parse_repository_dependency_tuple(rd_tup)
|
||||
if util.asbool(pir):
|
||||
pir_str = "True"
|
||||
else:
|
||||
pir_str = ""
|
||||
message += f"<tr><td>{tool_shed}</td><td>{name}</td><td>{owner}</td><td>{changeset_revision}</td><td>{pir_str}</td></tr>"
|
||||
message += "</table>"
|
||||
return message
|
||||
|
||||
def generate_message_for_invalid_tool_dependencies(self, metadata_dict):
|
||||
"""
|
||||
Tool dependency definitions can only be invalid if they include a definition for a complex
|
||||
repository dependency and the repository dependency definition is invalid. This method
|
||||
retrieves the error message associated with the invalid tool dependency for display in the
|
||||
caller.
|
||||
"""
|
||||
message = ""
|
||||
if metadata_dict:
|
||||
invalid_tool_dependencies = metadata_dict.get("invalid_tool_dependencies", None)
|
||||
if invalid_tool_dependencies:
|
||||
for requirement_dict in invalid_tool_dependencies.values():
|
||||
error = requirement_dict.get("error", None)
|
||||
if error:
|
||||
message = f"{error} "
|
||||
return message
|
||||
|
||||
def generate_message_for_orphan_tool_dependencies(self, repository, metadata_dict):
|
||||
"""
|
||||
The designation of a ToolDependency into the "orphan" category has evolved over time,
|
||||
and is significantly restricted since the introduction of the TOOL_DEPENDENCY_DEFINITION
|
||||
repository type. This designation is still critical, however, in that it handles the
|
||||
case where a repository contains both tools and a tool_dependencies.xml file, but the
|
||||
definition in the tool_dependencies.xml file is in no way related to anything defined
|
||||
by any of the contained tool's requirements tag sets. This is important in that it is
|
||||
often a result of a typo (e.g., dependency name or version) that differs between the tool
|
||||
dependency definition within the tool_dependencies.xml file and what is defined in the
|
||||
tool config's <requirements> tag sets. In these cases, the user should be presented with
|
||||
a warning message, and this warning message is is in fact displayed if the following
|
||||
is_orphan attribute is True. This is tricky because in some cases it may be intentional,
|
||||
and tool dependencies that are categorized as "orphan" are in fact valid.
|
||||
"""
|
||||
has_orphan_package_dependencies = False
|
||||
has_orphan_set_environment_dependencies = False
|
||||
message = ""
|
||||
package_orphans_str = ""
|
||||
set_environment_orphans_str = ""
|
||||
# Tool dependencies are categorized as orphan only if the repository contains tools.
|
||||
if metadata_dict:
|
||||
tools = metadata_dict.get("tools", [])
|
||||
tool_dependencies = metadata_dict.get("tool_dependencies", {})
|
||||
# The use of the orphan_tool_dependencies category in metadata has been deprecated,
|
||||
# but we still need to check in case the metadata is out of date.
|
||||
orphan_tool_dependencies = metadata_dict.get("orphan_tool_dependencies", {})
|
||||
# Updating should cause no problems here since a tool dependency cannot be included
|
||||
# in both dictionaries.
|
||||
tool_dependencies.update(orphan_tool_dependencies)
|
||||
if tool_dependencies and tools:
|
||||
for td_key, requirements_dict in tool_dependencies.items():
|
||||
if td_key == "set_environment":
|
||||
# "set_environment": [{"name": "R_SCRIPT_PATH", "type": "set_environment"}]
|
||||
for env_requirements_dict in requirements_dict:
|
||||
name = env_requirements_dict["name"]
|
||||
type = env_requirements_dict["type"]
|
||||
if self.tool_dependency_is_orphan(type, name, None, tools):
|
||||
if not has_orphan_set_environment_dependencies:
|
||||
has_orphan_set_environment_dependencies = True
|
||||
set_environment_orphans_str += f"<b>* name:</b> {name}, <b>type:</b> {type}<br/>"
|
||||
else:
|
||||
# "R/2.15.1": {"name": "R", "readme": "some string", "type": "package", "version": "2.15.1"}
|
||||
name = requirements_dict["name"]
|
||||
type = requirements_dict["type"]
|
||||
version = requirements_dict["version"]
|
||||
if self.tool_dependency_is_orphan(type, name, version, tools):
|
||||
if not has_orphan_package_dependencies:
|
||||
has_orphan_package_dependencies = True
|
||||
package_orphans_str += (
|
||||
f"<b>* name:</b> {name}, <b>type:</b> {type}, <b>version:</b> {version}<br/>"
|
||||
)
|
||||
if has_orphan_package_dependencies:
|
||||
message += "The settings for <b>name</b>, <b>version</b> and <b>type</b> from a "
|
||||
message += "contained tool configuration file's <b>requirement</b> tag does not match "
|
||||
message += "the information for the following tool dependency definitions in the "
|
||||
message += "<b>tool_dependencies.xml</b> file, so these tool dependencies have no "
|
||||
message += "relationship with any tools within this repository.<br/>"
|
||||
message += package_orphans_str
|
||||
if has_orphan_set_environment_dependencies:
|
||||
message += "The settings for <b>name</b> and <b>type</b> from a contained tool "
|
||||
message += "configuration file's <b>requirement</b> tag does not match the information "
|
||||
message += "for the following tool dependency definitions in the <b>tool_dependencies.xml</b> "
|
||||
message += "file, so these tool dependencies have no relationship with any tools within "
|
||||
message += "this repository.<br/>"
|
||||
message += set_environment_orphans_str
|
||||
return message
|
||||
|
||||
def get_installed_and_missing_tool_dependencies_for_installed_repository(self, repository, all_tool_dependencies):
|
||||
"""
|
||||
Return the lists of installed tool dependencies and missing tool dependencies for a Tool Shed
|
||||
repository that has been installed into Galaxy.
|
||||
"""
|
||||
if all_tool_dependencies:
|
||||
tool_dependencies = {}
|
||||
missing_tool_dependencies = {}
|
||||
for td_key, val in all_tool_dependencies.items():
|
||||
if td_key in ["set_environment"]:
|
||||
for index, td_info_dict in enumerate(val):
|
||||
name = td_info_dict["name"]
|
||||
version = None
|
||||
type = td_info_dict["type"]
|
||||
tool_dependency = tool_dependency_util.get_tool_dependency_by_name_type_repository(
|
||||
self.app, repository, name, type
|
||||
)
|
||||
if tool_dependency:
|
||||
td_info_dict["repository_id"] = repository.id
|
||||
td_info_dict["tool_dependency_id"] = tool_dependency.id
|
||||
if tool_dependency.status:
|
||||
tool_dependency_status = str(tool_dependency.status)
|
||||
else:
|
||||
tool_dependency_status = "Never installed"
|
||||
td_info_dict["status"] = tool_dependency_status
|
||||
val[index] = td_info_dict
|
||||
if (
|
||||
tool_dependency.status
|
||||
== self.app.install_model.ToolDependency.installation_status.INSTALLED
|
||||
):
|
||||
tool_dependencies[td_key] = val
|
||||
else:
|
||||
missing_tool_dependencies[td_key] = val
|
||||
else:
|
||||
name = val["name"]
|
||||
version = val["version"]
|
||||
type = val["type"]
|
||||
tool_dependency = tool_dependency_util.get_tool_dependency_by_name_version_type_repository(
|
||||
self.app, repository, name, version, type
|
||||
)
|
||||
if tool_dependency:
|
||||
val["repository_id"] = repository.id
|
||||
val["tool_dependency_id"] = tool_dependency.id
|
||||
if tool_dependency.status:
|
||||
tool_dependency_status = str(tool_dependency.status)
|
||||
else:
|
||||
tool_dependency_status = "Never installed"
|
||||
val["status"] = tool_dependency_status
|
||||
if (
|
||||
tool_dependency.status
|
||||
== self.app.install_model.ToolDependency.installation_status.INSTALLED
|
||||
):
|
||||
tool_dependencies[td_key] = val
|
||||
else:
|
||||
missing_tool_dependencies[td_key] = val
|
||||
else:
|
||||
tool_dependencies = None
|
||||
missing_tool_dependencies = None
|
||||
return tool_dependencies, missing_tool_dependencies
|
||||
|
||||
def merge_containers_dicts_for_new_install(self, containers_dicts):
|
||||
"""
|
||||
When installing one or more tool shed repositories for the first time, the received list of
|
||||
containers_dicts contains a containers_dict for each repository being installed. Since the
|
||||
repositories are being installed for the first time, all entries are None except the repository
|
||||
dependencies and tool dependencies. The entries for missing dependencies are all None since
|
||||
they have previously been merged into the installed dependencies. This method will merge the
|
||||
dependencies entries into a single container and return it for display.
|
||||
"""
|
||||
new_containers_dict = dict(
|
||||
readme_files=None,
|
||||
datatypes=None,
|
||||
missing_repository_dependencies=None,
|
||||
repository_dependencies=None,
|
||||
missing_tool_dependencies=None,
|
||||
tool_dependencies=None,
|
||||
invalid_tools=None,
|
||||
valid_tools=None,
|
||||
workflows=None,
|
||||
)
|
||||
if containers_dicts:
|
||||
lock = threading.Lock()
|
||||
lock.acquire(True)
|
||||
try:
|
||||
repository_dependencies_root_folder = None
|
||||
tool_dependencies_root_folder = None
|
||||
# Use a unique folder id (hopefully the following is).
|
||||
folder_id = 867
|
||||
for old_container_dict in containers_dicts:
|
||||
# Merge repository_dependencies.
|
||||
old_container_repository_dependencies_root = old_container_dict["repository_dependencies"]
|
||||
if old_container_repository_dependencies_root:
|
||||
if repository_dependencies_root_folder is None:
|
||||
repository_dependencies_root_folder = utility_container_manager.Folder(
|
||||
id=folder_id, key="root", label="root", parent=None
|
||||
)
|
||||
folder_id += 1
|
||||
repository_dependencies_folder = utility_container_manager.Folder(
|
||||
id=folder_id,
|
||||
key="merged",
|
||||
label="Repository dependencies",
|
||||
parent=repository_dependencies_root_folder,
|
||||
)
|
||||
folder_id += 1
|
||||
# The old_container_repository_dependencies_root will be a root folder containing a single sub_folder.
|
||||
old_container_repository_dependencies_folder = (
|
||||
old_container_repository_dependencies_root.folders[0]
|
||||
)
|
||||
# Change the folder id so it won't confict with others being merged.
|
||||
old_container_repository_dependencies_folder.id = folder_id
|
||||
folder_id += 1
|
||||
repository_components_tuple = container_util.get_components_from_key(
|
||||
old_container_repository_dependencies_folder.key
|
||||
)
|
||||
components_list = repository_util.extract_components_from_tuple(repository_components_tuple)
|
||||
name = components_list[1]
|
||||
# Generate the label by retrieving the repository name.
|
||||
old_container_repository_dependencies_folder.label = str(name)
|
||||
repository_dependencies_folder.folders.append(old_container_repository_dependencies_folder)
|
||||
# Merge tool_dependencies.
|
||||
old_container_tool_dependencies_root = old_container_dict["tool_dependencies"]
|
||||
if old_container_tool_dependencies_root:
|
||||
if tool_dependencies_root_folder is None:
|
||||
tool_dependencies_root_folder = utility_container_manager.Folder(
|
||||
id=folder_id, key="root", label="root", parent=None
|
||||
)
|
||||
folder_id += 1
|
||||
tool_dependencies_folder = utility_container_manager.Folder(
|
||||
id=folder_id,
|
||||
key="merged",
|
||||
label="Tool dependencies",
|
||||
parent=tool_dependencies_root_folder,
|
||||
)
|
||||
folder_id += 1
|
||||
else:
|
||||
td_list = [td.listify for td in tool_dependencies_folder.tool_dependencies]
|
||||
# The old_container_tool_dependencies_root will be a root folder containing a single sub_folder.
|
||||
old_container_tool_dependencies_folder = old_container_tool_dependencies_root.folders[0]
|
||||
for td in old_container_tool_dependencies_folder.tool_dependencies:
|
||||
if td.listify not in td_list:
|
||||
tool_dependencies_folder.tool_dependencies.append(td)
|
||||
if repository_dependencies_root_folder:
|
||||
repository_dependencies_root_folder.folders.append(repository_dependencies_folder)
|
||||
new_containers_dict["repository_dependencies"] = repository_dependencies_root_folder
|
||||
if tool_dependencies_root_folder:
|
||||
tool_dependencies_root_folder.folders.append(tool_dependencies_folder)
|
||||
new_containers_dict["tool_dependencies"] = tool_dependencies_root_folder
|
||||
except Exception as e:
|
||||
log.debug(f"Exception in merge_containers_dicts_for_new_install: {e}")
|
||||
finally:
|
||||
lock.release()
|
||||
return new_containers_dict
|
||||
|
||||
def merge_missing_repository_dependencies_to_installed_container(self, containers_dict):
|
||||
"""
|
||||
Merge the list of missing repository dependencies into the list of installed
|
||||
repository dependencies.
|
||||
"""
|
||||
missing_rd_container_root = containers_dict.get("missing_repository_dependencies", None)
|
||||
if missing_rd_container_root:
|
||||
# The missing_rd_container_root will be a root folder containing a single sub_folder.
|
||||
missing_rd_container = missing_rd_container_root.folders[0]
|
||||
installed_rd_container_root = containers_dict.get("repository_dependencies", None)
|
||||
# The installed_rd_container_root will be a root folder containing a single sub_folder.
|
||||
if installed_rd_container_root:
|
||||
installed_rd_container = installed_rd_container_root.folders[0]
|
||||
installed_rd_container.label = "Repository dependencies"
|
||||
for index, rd in enumerate(missing_rd_container.repository_dependencies):
|
||||
# Skip the header row.
|
||||
if index == 0:
|
||||
continue
|
||||
installed_rd_container.repository_dependencies.append(rd)
|
||||
installed_rd_container_root.folders = [installed_rd_container]
|
||||
containers_dict["repository_dependencies"] = installed_rd_container_root
|
||||
else:
|
||||
# Change the folder label from 'Missing repository dependencies' to be
|
||||
# 'Repository dependencies' for display.
|
||||
root_container = containers_dict["missing_repository_dependencies"]
|
||||
for sub_container in root_container.folders:
|
||||
# There should only be 1 sub-folder.
|
||||
sub_container.label = "Repository dependencies"
|
||||
containers_dict["repository_dependencies"] = root_container
|
||||
containers_dict["missing_repository_dependencies"] = None
|
||||
return containers_dict
|
||||
|
||||
def merge_missing_tool_dependencies_to_installed_container(self, containers_dict):
|
||||
"""
|
||||
Merge the list of missing tool dependencies into the list of installed tool
|
||||
dependencies.
|
||||
"""
|
||||
missing_td_container_root = containers_dict.get("missing_tool_dependencies", None)
|
||||
if missing_td_container_root:
|
||||
# The missing_td_container_root will be a root folder containing a single sub_folder.
|
||||
missing_td_container = missing_td_container_root.folders[0]
|
||||
installed_td_container_root = containers_dict.get("tool_dependencies", None)
|
||||
# The installed_td_container_root will be a root folder containing a single sub_folder.
|
||||
if installed_td_container_root:
|
||||
installed_td_container = installed_td_container_root.folders[0]
|
||||
installed_td_container.label = "Tool dependencies"
|
||||
for index, td in enumerate(missing_td_container.tool_dependencies):
|
||||
# Skip the header row.
|
||||
if index == 0:
|
||||
continue
|
||||
installed_td_container.tool_dependencies.append(td)
|
||||
installed_td_container_root.folders = [installed_td_container]
|
||||
containers_dict["tool_dependencies"] = installed_td_container_root
|
||||
else:
|
||||
# Change the folder label from 'Missing tool dependencies' to be
|
||||
# 'Tool dependencies' for display.
|
||||
root_container = containers_dict["missing_tool_dependencies"]
|
||||
for sub_container in root_container.folders:
|
||||
# There should only be 1 subfolder.
|
||||
sub_container.label = "Tool dependencies"
|
||||
containers_dict["tool_dependencies"] = root_container
|
||||
containers_dict["missing_tool_dependencies"] = None
|
||||
return containers_dict
|
||||
|
||||
def populate_containers_dict_for_new_install(
|
||||
self,
|
||||
tool_shed_url,
|
||||
tool_path,
|
||||
readme_files_dict,
|
||||
installed_repository_dependencies,
|
||||
missing_repository_dependencies,
|
||||
installed_tool_dependencies,
|
||||
missing_tool_dependencies,
|
||||
updating=False,
|
||||
):
|
||||
"""
|
||||
Return the populated containers for a repository being installed for the first time
|
||||
or for an installed repository that is being updated and the updates include newly
|
||||
defined repository (and possibly tool) dependencies.
|
||||
"""
|
||||
installed_tool_dependencies, missing_tool_dependencies = self.populate_tool_dependencies_dicts(
|
||||
tool_shed_url=tool_shed_url,
|
||||
tool_path=tool_path,
|
||||
repository_installed_tool_dependencies=installed_tool_dependencies,
|
||||
repository_missing_tool_dependencies=missing_tool_dependencies,
|
||||
required_repo_info_dicts=None,
|
||||
)
|
||||
# Most of the repository contents are set to None since we don't yet know what they are.
|
||||
gucm = GalaxyUtilityContainerManager(self.app)
|
||||
containers_dict = gucm.build_repository_containers(
|
||||
repository=None,
|
||||
invalid_tools=None,
|
||||
missing_repository_dependencies=missing_repository_dependencies,
|
||||
missing_tool_dependencies=missing_tool_dependencies,
|
||||
readme_files_dict=readme_files_dict,
|
||||
repository_dependencies=installed_repository_dependencies,
|
||||
tool_dependencies=installed_tool_dependencies,
|
||||
valid_tools=None,
|
||||
valid_data_managers=None,
|
||||
invalid_data_managers=None,
|
||||
data_managers_errors=None,
|
||||
new_install=True,
|
||||
reinstalling=False,
|
||||
)
|
||||
if not updating:
|
||||
# If we installing a new repository and not updaing an installed repository, we can merge
|
||||
# the missing_repository_dependencies container contents to the installed_repository_dependencies
|
||||
# container. When updating an installed repository, merging will result in losing newly defined
|
||||
# dependencies included in the updates.
|
||||
containers_dict = self.merge_missing_repository_dependencies_to_installed_container(containers_dict)
|
||||
# Merge the missing_tool_dependencies container contents to the installed_tool_dependencies container.
|
||||
containers_dict = self.merge_missing_tool_dependencies_to_installed_container(containers_dict)
|
||||
return containers_dict
|
||||
|
||||
def populate_containers_dict_from_repository_metadata(
|
||||
self, tool_shed_url, tool_path, repository, reinstalling=False, required_repo_info_dicts=None
|
||||
):
|
||||
"""
|
||||
Retrieve necessary information from the received repository's metadata to populate the
|
||||
containers_dict for display. This method is called only from Galaxy (not the tool shed)
|
||||
when displaying repository dependencies for installed repositories and when displaying
|
||||
them for uninstalled repositories that are being reinstalled.
|
||||
"""
|
||||
metadata = repository.metadata_
|
||||
if metadata:
|
||||
# Handle invalid tools.
|
||||
invalid_tools = metadata.get("invalid_tools", None)
|
||||
# Handle README files.
|
||||
if repository.has_readme_files:
|
||||
if reinstalling or repository.status not in [
|
||||
self.app.install_model.ToolShedRepository.installation_status.DEACTIVATED,
|
||||
self.app.install_model.ToolShedRepository.installation_status.INSTALLED,
|
||||
]:
|
||||
# Since we're reinstalling, we need to send a request to the tool shed to get the README files.
|
||||
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(self.app, tool_shed_url)
|
||||
params = dict(
|
||||
name=str(repository.name),
|
||||
owner=str(repository.owner),
|
||||
changeset_revision=str(repository.installed_changeset_revision),
|
||||
)
|
||||
pathspec = ["repository", "get_readme_files"]
|
||||
raw_text = util.url_get(
|
||||
tool_shed_url,
|
||||
auth=self.app.tool_shed_registry.url_auth(tool_shed_url),
|
||||
pathspec=pathspec,
|
||||
params=params,
|
||||
)
|
||||
readme_files_dict = json.loads(raw_text)
|
||||
else:
|
||||
readme_files_dict = readme_util.build_readme_files_dict(
|
||||
self.app, repository, repository.changeset_revision, repository.metadata_, tool_path
|
||||
)
|
||||
else:
|
||||
readme_files_dict = None
|
||||
# Handle repository dependencies.
|
||||
(
|
||||
installed_repository_dependencies,
|
||||
missing_repository_dependencies,
|
||||
) = self.app.installed_repository_manager.get_installed_and_missing_repository_dependencies(repository)
|
||||
# Handle the current repository's tool dependencies.
|
||||
repository_tool_dependencies = metadata.get("tool_dependencies", None)
|
||||
# Make sure to display missing tool dependencies as well.
|
||||
repository_invalid_tool_dependencies = metadata.get("invalid_tool_dependencies", None)
|
||||
if repository_invalid_tool_dependencies is not None:
|
||||
if repository_tool_dependencies is None:
|
||||
repository_tool_dependencies = {}
|
||||
repository_tool_dependencies.update(repository_invalid_tool_dependencies)
|
||||
(
|
||||
repository_installed_tool_dependencies,
|
||||
repository_missing_tool_dependencies,
|
||||
) = self.get_installed_and_missing_tool_dependencies_for_installed_repository(
|
||||
repository, repository_tool_dependencies
|
||||
)
|
||||
if reinstalling:
|
||||
installed_tool_dependencies, missing_tool_dependencies = self.populate_tool_dependencies_dicts(
|
||||
tool_shed_url,
|
||||
tool_path,
|
||||
repository_installed_tool_dependencies,
|
||||
repository_missing_tool_dependencies,
|
||||
required_repo_info_dicts,
|
||||
)
|
||||
else:
|
||||
installed_tool_dependencies = repository_installed_tool_dependencies
|
||||
missing_tool_dependencies = repository_missing_tool_dependencies
|
||||
# Handle valid tools.
|
||||
valid_tools = metadata.get("tools", None)
|
||||
# Handle Data Managers
|
||||
valid_data_managers = None
|
||||
invalid_data_managers = None
|
||||
data_managers_errors = None
|
||||
if "data_manager" in metadata:
|
||||
valid_data_managers = metadata["data_manager"].get("data_managers", None)
|
||||
invalid_data_managers = metadata["data_manager"].get("invalid_data_managers", None)
|
||||
data_managers_errors = metadata["data_manager"].get("messages", None)
|
||||
gucm = GalaxyUtilityContainerManager(self.app)
|
||||
containers_dict = gucm.build_repository_containers(
|
||||
repository=repository,
|
||||
invalid_tools=invalid_tools,
|
||||
missing_repository_dependencies=missing_repository_dependencies,
|
||||
missing_tool_dependencies=missing_tool_dependencies,
|
||||
readme_files_dict=readme_files_dict,
|
||||
repository_dependencies=installed_repository_dependencies,
|
||||
tool_dependencies=installed_tool_dependencies,
|
||||
valid_tools=valid_tools,
|
||||
valid_data_managers=valid_data_managers,
|
||||
invalid_data_managers=invalid_data_managers,
|
||||
data_managers_errors=data_managers_errors,
|
||||
new_install=False,
|
||||
reinstalling=reinstalling,
|
||||
)
|
||||
else:
|
||||
containers_dict = dict(
|
||||
datatypes=None,
|
||||
invalid_tools=None,
|
||||
readme_files_dict=None,
|
||||
repository_dependencies=None,
|
||||
tool_dependencies=None,
|
||||
valid_tools=None,
|
||||
workflows=None,
|
||||
)
|
||||
return containers_dict
|
||||
|
||||
def populate_tool_dependencies_dicts(
|
||||
self,
|
||||
tool_shed_url,
|
||||
tool_path,
|
||||
repository_installed_tool_dependencies,
|
||||
repository_missing_tool_dependencies,
|
||||
required_repo_info_dicts,
|
||||
):
|
||||
"""
|
||||
Return the populated installed_tool_dependencies and missing_tool_dependencies dictionaries
|
||||
for all repositories defined by entries in the received required_repo_info_dicts.
|
||||
"""
|
||||
installed_tool_dependencies = None
|
||||
missing_tool_dependencies = None
|
||||
if repository_installed_tool_dependencies is None:
|
||||
repository_installed_tool_dependencies = {}
|
||||
else:
|
||||
# Add the install_dir attribute to the tool_dependencies.
|
||||
repository_installed_tool_dependencies = self.add_installation_directories_to_tool_dependencies(
|
||||
repository_installed_tool_dependencies
|
||||
)
|
||||
if repository_missing_tool_dependencies is None:
|
||||
repository_missing_tool_dependencies = {}
|
||||
else:
|
||||
# Add the install_dir attribute to the tool_dependencies.
|
||||
repository_missing_tool_dependencies = self.add_installation_directories_to_tool_dependencies(
|
||||
repository_missing_tool_dependencies
|
||||
)
|
||||
if required_repo_info_dicts:
|
||||
# Handle the tool dependencies defined for each of the repository's repository dependencies.
|
||||
for rid in required_repo_info_dicts:
|
||||
for name, repo_info_tuple in rid.items():
|
||||
(
|
||||
description,
|
||||
repository_clone_url,
|
||||
changeset_revision,
|
||||
ctx_rev,
|
||||
repository_owner,
|
||||
repository_dependencies,
|
||||
tool_dependencies,
|
||||
) = repository_util.get_repo_info_tuple_contents(repo_info_tuple)
|
||||
if tool_dependencies:
|
||||
# Add the install_dir attribute to the tool_dependencies.
|
||||
tool_dependencies = self.add_installation_directories_to_tool_dependencies(tool_dependencies)
|
||||
# The required_repository may have been installed with a different changeset revision.
|
||||
(
|
||||
required_repository,
|
||||
installed_changeset_revision,
|
||||
) = repository_util.repository_was_previously_installed(
|
||||
self.app, tool_shed_url, name, repo_info_tuple, from_tip=False
|
||||
)
|
||||
if required_repository:
|
||||
(
|
||||
required_repository_installed_tool_dependencies,
|
||||
required_repository_missing_tool_dependencies,
|
||||
) = self.get_installed_and_missing_tool_dependencies_for_installed_repository(
|
||||
required_repository, tool_dependencies
|
||||
)
|
||||
if required_repository_installed_tool_dependencies:
|
||||
# Add the install_dir attribute to the tool_dependencies.
|
||||
required_repository_installed_tool_dependencies = (
|
||||
self.add_installation_directories_to_tool_dependencies(
|
||||
required_repository_installed_tool_dependencies
|
||||
)
|
||||
)
|
||||
for td_key, td_dict in required_repository_installed_tool_dependencies.items():
|
||||
if td_key not in repository_installed_tool_dependencies:
|
||||
repository_installed_tool_dependencies[td_key] = td_dict
|
||||
if required_repository_missing_tool_dependencies:
|
||||
# Add the install_dir attribute to the tool_dependencies.
|
||||
required_repository_missing_tool_dependencies = (
|
||||
self.add_installation_directories_to_tool_dependencies(
|
||||
required_repository_missing_tool_dependencies
|
||||
)
|
||||
)
|
||||
for td_key, td_dict in required_repository_missing_tool_dependencies.items():
|
||||
if td_key not in repository_missing_tool_dependencies:
|
||||
repository_missing_tool_dependencies[td_key] = td_dict
|
||||
if repository_installed_tool_dependencies:
|
||||
installed_tool_dependencies = repository_installed_tool_dependencies
|
||||
if repository_missing_tool_dependencies:
|
||||
missing_tool_dependencies = repository_missing_tool_dependencies
|
||||
return installed_tool_dependencies, missing_tool_dependencies
|
||||
|
||||
def tool_dependency_is_orphan(self, type, name, version, tools):
|
||||
"""
|
||||
Determine if the combination of the received type, name and version is defined in the <requirement>
|
||||
tag for at least one tool in the received list of tools. If not, the tool dependency defined by the
|
||||
combination is considered an orphan in its repository in the tool shed.
|
||||
"""
|
||||
if type == "package":
|
||||
if name and version:
|
||||
for tool_dict in tools:
|
||||
requirements = tool_dict.get("requirements", [])
|
||||
for requirement_dict in requirements:
|
||||
req_name = requirement_dict.get("name", None)
|
||||
req_version = requirement_dict.get("version", None)
|
||||
req_type = requirement_dict.get("type", None)
|
||||
if req_name == name and req_version == version and req_type == type:
|
||||
return False
|
||||
elif type == "set_environment":
|
||||
if name:
|
||||
for tool_dict in tools:
|
||||
requirements = tool_dict.get("requirements", [])
|
||||
for requirement_dict in requirements:
|
||||
req_name = requirement_dict.get("name", None)
|
||||
req_type = requirement_dict.get("type", None)
|
||||
if req_name == name and req_type == type:
|
||||
return False
|
||||
return True
|
||||
@@ -1,138 +0,0 @@
|
||||
import logging
|
||||
import threading
|
||||
|
||||
from tool_shed.utility_containers import utility_container_manager
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class GalaxyUtilityContainerManager(utility_container_manager.UtilityContainerManager):
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
|
||||
def build_repository_containers(
|
||||
self,
|
||||
repository,
|
||||
invalid_tools,
|
||||
missing_repository_dependencies,
|
||||
missing_tool_dependencies,
|
||||
readme_files_dict,
|
||||
repository_dependencies,
|
||||
tool_dependencies,
|
||||
valid_tools,
|
||||
valid_data_managers,
|
||||
invalid_data_managers,
|
||||
data_managers_errors,
|
||||
new_install=False,
|
||||
reinstalling=False,
|
||||
):
|
||||
"""
|
||||
Return a dictionary of containers for the received repository's dependencies and readme files for
|
||||
display during installation to Galaxy.
|
||||
"""
|
||||
containers_dict = dict(
|
||||
datatypes=None,
|
||||
invalid_tools=None,
|
||||
missing_tool_dependencies=None,
|
||||
readme_files=None,
|
||||
repository_dependencies=None,
|
||||
missing_repository_dependencies=None,
|
||||
tool_dependencies=None,
|
||||
valid_tools=None,
|
||||
workflows=None,
|
||||
valid_data_managers=None,
|
||||
invalid_data_managers=None,
|
||||
)
|
||||
# Some of the tool dependency folders will include links to display tool dependency information, and
|
||||
# some of these links require the repository id. However we need to be careful because sometimes the
|
||||
# repository object is None.
|
||||
if repository:
|
||||
changeset_revision = repository.changeset_revision
|
||||
else:
|
||||
changeset_revision = None
|
||||
lock = threading.Lock()
|
||||
lock.acquire(True)
|
||||
try:
|
||||
folder_id = 0
|
||||
# Invalid tools container.
|
||||
if invalid_tools:
|
||||
folder_id, invalid_tools_root_folder = self.build_invalid_tools_folder(
|
||||
folder_id, invalid_tools, changeset_revision, repository=repository, label="Invalid tools"
|
||||
)
|
||||
containers_dict["invalid_tools"] = invalid_tools_root_folder
|
||||
# Readme files container.
|
||||
if readme_files_dict:
|
||||
folder_id, readme_files_root_folder = self.build_readme_files_folder(folder_id, readme_files_dict)
|
||||
containers_dict["readme_files"] = readme_files_root_folder
|
||||
# Installed repository dependencies container.
|
||||
if repository_dependencies:
|
||||
if new_install:
|
||||
label = "Repository dependencies"
|
||||
else:
|
||||
label = "Installed repository dependencies"
|
||||
folder_id, repository_dependencies_root_folder = self.build_repository_dependencies_folder(
|
||||
folder_id=folder_id, repository_dependencies=repository_dependencies, label=label, installed=True
|
||||
)
|
||||
containers_dict["repository_dependencies"] = repository_dependencies_root_folder
|
||||
# Missing repository dependencies container.
|
||||
if missing_repository_dependencies:
|
||||
folder_id, missing_repository_dependencies_root_folder = self.build_repository_dependencies_folder(
|
||||
folder_id=folder_id,
|
||||
repository_dependencies=missing_repository_dependencies,
|
||||
label="Missing repository dependencies",
|
||||
installed=False,
|
||||
)
|
||||
containers_dict["missing_repository_dependencies"] = missing_repository_dependencies_root_folder
|
||||
# Installed tool dependencies container.
|
||||
if tool_dependencies:
|
||||
if new_install:
|
||||
label = "Tool dependencies"
|
||||
else:
|
||||
label = "Installed tool dependencies"
|
||||
# We only want to display the Status column if the tool_dependency is missing.
|
||||
folder_id, tool_dependencies_root_folder = self.build_tool_dependencies_folder(
|
||||
folder_id,
|
||||
tool_dependencies,
|
||||
label=label,
|
||||
missing=False,
|
||||
new_install=new_install,
|
||||
reinstalling=reinstalling,
|
||||
)
|
||||
containers_dict["tool_dependencies"] = tool_dependencies_root_folder
|
||||
# Missing tool dependencies container.
|
||||
if missing_tool_dependencies:
|
||||
# We only want to display the Status column if the tool_dependency is missing.
|
||||
folder_id, missing_tool_dependencies_root_folder = self.build_tool_dependencies_folder(
|
||||
folder_id,
|
||||
missing_tool_dependencies,
|
||||
label="Missing tool dependencies",
|
||||
missing=True,
|
||||
new_install=new_install,
|
||||
reinstalling=reinstalling,
|
||||
)
|
||||
containers_dict["missing_tool_dependencies"] = missing_tool_dependencies_root_folder
|
||||
# Valid tools container.
|
||||
if valid_tools:
|
||||
folder_id, valid_tools_root_folder = self.build_tools_folder(
|
||||
folder_id, valid_tools, repository, changeset_revision, label="Valid tools"
|
||||
)
|
||||
containers_dict["valid_tools"] = valid_tools_root_folder
|
||||
# Workflows container.
|
||||
if valid_data_managers:
|
||||
folder_id, valid_data_managers_root_folder = self.build_data_managers_folder(
|
||||
folder_id=folder_id, data_managers=valid_data_managers, label="Valid Data Managers"
|
||||
)
|
||||
containers_dict["valid_data_managers"] = valid_data_managers_root_folder
|
||||
if invalid_data_managers or data_managers_errors:
|
||||
folder_id, invalid_data_managers_root_folder = self.build_invalid_data_managers_folder(
|
||||
folder_id=folder_id,
|
||||
data_managers=invalid_data_managers,
|
||||
error_messages=data_managers_errors,
|
||||
label="Invalid Data Managers",
|
||||
)
|
||||
containers_dict["invalid_data_managers"] = invalid_data_managers_root_folder
|
||||
except Exception as e:
|
||||
log.debug(f"Exception in build_repository_containers: {str(e)}")
|
||||
finally:
|
||||
lock.release()
|
||||
return containers_dict
|
||||
@@ -4,20 +4,21 @@ import os
|
||||
|
||||
from mako.template import Template
|
||||
|
||||
import tool_shed.util.shed_util_common as suc
|
||||
from galaxy import web
|
||||
from galaxy.tool_shed.util import basic_util
|
||||
from galaxy.tool_shed.util.hg_util import (
|
||||
get_changectx_for_changeset,
|
||||
get_file_context_from_ctx,
|
||||
)
|
||||
from galaxy.tool_shed.util.repository_util import get_repo_info_tuple_contents
|
||||
from galaxy.tool_shed.util.shed_util_common import set_image_paths
|
||||
from galaxy.util import (
|
||||
rst_to_html,
|
||||
unicodify,
|
||||
url_get,
|
||||
)
|
||||
from tool_shed.util import (
|
||||
basic_util,
|
||||
common_util,
|
||||
hg_util,
|
||||
metadata_util,
|
||||
repository_util,
|
||||
)
|
||||
from galaxy.util.tool_shed.common_util import get_tool_shed_url_from_tool_shed_registry
|
||||
from tool_shed.util.metadata_util import get_latest_downloadable_changeset_revision
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -31,9 +32,7 @@ def build_readme_files_dict(app, repository, changeset_revision, metadata, tool_
|
||||
if app.name == "galaxy":
|
||||
can_use_disk_files = True
|
||||
else:
|
||||
latest_downloadable_changeset_revision = metadata_util.get_latest_downloadable_changeset_revision(
|
||||
app, repository
|
||||
)
|
||||
latest_downloadable_changeset_revision = get_latest_downloadable_changeset_revision(app, repository)
|
||||
can_use_disk_files = changeset_revision == latest_downloadable_changeset_revision
|
||||
readme_files_dict = {}
|
||||
if metadata:
|
||||
@@ -59,7 +58,7 @@ def build_readme_files_dict(app, repository, changeset_revision, metadata, tool_
|
||||
if text_of_reasonable_length.find(".. image:: ") >= 0:
|
||||
# Handle image display for README files that are contained in repositories in the tool shed or installed into Galaxy.
|
||||
try:
|
||||
text_of_reasonable_length = suc.set_image_paths(
|
||||
text_of_reasonable_length = set_image_paths(
|
||||
app,
|
||||
text_of_reasonable_length,
|
||||
encoded_repository_id=app.security.encode_id(repository.id),
|
||||
@@ -85,9 +84,9 @@ def build_readme_files_dict(app, repository, changeset_revision, metadata, tool_
|
||||
else:
|
||||
# We must be in the tool shed and have an old changeset_revision, so we need to retrieve the file contents from the repository manifest.
|
||||
repo = repository.hg_repo
|
||||
ctx = hg_util.get_changectx_for_changeset(repo, changeset_revision)
|
||||
ctx = get_changectx_for_changeset(repo, changeset_revision)
|
||||
if ctx:
|
||||
fctx = hg_util.get_file_context_from_ctx(ctx, readme_file_name)
|
||||
fctx = get_file_context_from_ctx(ctx, readme_file_name)
|
||||
if fctx and fctx not in ["DELETED"]:
|
||||
try:
|
||||
text = unicodify(fctx.data())
|
||||
@@ -115,9 +114,9 @@ def get_readme_files_dict_for_display(app, tool_shed_url, repo_info_dict):
|
||||
repository_owner,
|
||||
repository_dependencies,
|
||||
installed_td,
|
||||
) = repository_util.get_repo_info_tuple_contents(repo_info_tuple)
|
||||
) = get_repo_info_tuple_contents(repo_info_tuple)
|
||||
# Handle changing HTTP protocols over time.
|
||||
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(app, tool_shed_url)
|
||||
tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, tool_shed_url)
|
||||
params = dict(name=name, owner=repository_owner, changeset_revision=changeset_revision)
|
||||
pathspec = ["repository", "get_readme_files"]
|
||||
raw_text = url_get(
|
||||
|
||||
@@ -2,12 +2,13 @@ import logging
|
||||
import threading
|
||||
|
||||
from galaxy import util
|
||||
from tool_shed.util import (
|
||||
common_util,
|
||||
container_util,
|
||||
readme_util,
|
||||
from galaxy.tool_shed.util import utility_container_manager
|
||||
from galaxy.tool_shed.util.container_util import (
|
||||
generate_repository_dependencies_key_for_repository,
|
||||
STRSEP,
|
||||
)
|
||||
from . import utility_container_manager
|
||||
from galaxy.util.tool_shed.common_util import parse_repository_dependency_tuple
|
||||
from tool_shed.util.readme_util import build_readme_files_dict
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -166,8 +167,8 @@ class ToolShedUtilityContainerManager(utility_container_manager.UtilityContainer
|
||||
prior_installation_required,
|
||||
only_if_compiling_contained_td,
|
||||
error,
|
||||
) = common_util.parse_repository_dependency_tuple(invalid_repository_dependency, contains_error=True)
|
||||
key = container_util.generate_repository_dependencies_key_for_repository(
|
||||
) = parse_repository_dependency_tuple(invalid_repository_dependency, contains_error=True)
|
||||
key = generate_repository_dependencies_key_for_repository(
|
||||
toolshed,
|
||||
name,
|
||||
owner,
|
||||
@@ -300,9 +301,7 @@ class ToolShedUtilityContainerManager(utility_container_manager.UtilityContainer
|
||||
# Readme files container.
|
||||
if metadata:
|
||||
if "readme_files" not in exclude and "readme_files" in metadata:
|
||||
readme_files_dict = readme_util.build_readme_files_dict(
|
||||
self.app, repository, changeset_revision, metadata
|
||||
)
|
||||
readme_files_dict = build_readme_files_dict(self.app, repository, changeset_revision, metadata)
|
||||
folder_id, readme_files_root_folder = self.build_readme_files_folder(
|
||||
folder_id, readme_files_dict
|
||||
)
|
||||
@@ -363,4 +362,4 @@ class ToolShedUtilityContainerManager(utility_container_manager.UtilityContainer
|
||||
return containers_dict
|
||||
|
||||
def generate_tool_dependencies_key(self, name, version, type):
|
||||
return f"{str(name)}{container_util.STRSEP}{str(version)}{container_util.STRSEP}{str(type)}"
|
||||
return f"{str(name)}{STRSEP}{str(version)}{STRSEP}{str(type)}"
|
||||
|
||||
@@ -24,6 +24,7 @@ from galaxy import (
|
||||
util,
|
||||
web,
|
||||
)
|
||||
from galaxy.tool_shed.util import dependency_display
|
||||
from galaxy.tools.repositories import ValidationContext
|
||||
from galaxy.web.form_builder import (
|
||||
CheckboxField,
|
||||
@@ -32,7 +33,6 @@ from galaxy.web.form_builder import (
|
||||
from galaxy.web.legacy_framework import grids
|
||||
from galaxy.webapps.base.controller import BaseUIController
|
||||
from tool_shed.dependencies.repository import relation_builder
|
||||
from tool_shed.galaxy_install import dependency_display
|
||||
from tool_shed.metadata import repository_metadata_manager
|
||||
from tool_shed.tools import (
|
||||
tool_validator,
|
||||
|
||||
@@ -10,10 +10,10 @@ from galaxy import (
|
||||
util,
|
||||
web,
|
||||
)
|
||||
from galaxy.tool_shed.util import dependency_display
|
||||
from galaxy.util import checkers
|
||||
from galaxy.webapps.base.controller import BaseUIController
|
||||
from tool_shed.dependencies import attribute_handlers
|
||||
from tool_shed.galaxy_install import dependency_display
|
||||
from tool_shed.metadata import repository_metadata_manager
|
||||
from tool_shed.repository_types import util as rt_util
|
||||
from tool_shed.tools.data_table_manager import ShedToolDataTableManager
|
||||
|
||||
@@ -1,101 +0,0 @@
|
||||
<%inherit file="/base.mako"/>
|
||||
<%namespace file="/message.mako" import="render_msg" />
|
||||
<%namespace file="/admin/tool_shed_repository/common.mako" import="render_dependencies_section" />
|
||||
<%namespace file="/admin/tool_shed_repository/common.mako" import="render_readme_section" />
|
||||
<%namespace file="/webapps/tool_shed/repository/common.mako" import="*" />
|
||||
<%namespace file="/webapps/tool_shed/common/common.mako" import="*" />
|
||||
|
||||
<%def name="stylesheets()">
|
||||
${parent.stylesheets()}
|
||||
${h.css( "library" )}
|
||||
</%def>
|
||||
|
||||
<%def name="javascripts()">
|
||||
${parent.javascripts()}
|
||||
${container_javascripts()}
|
||||
</%def>
|
||||
|
||||
%if message:
|
||||
${render_msg( message, status )}
|
||||
%endif
|
||||
|
||||
<div class="card">
|
||||
<div class="card-body">
|
||||
<form name="reselect_tool_panel_section" id="reselect_tool_panel_section" action="${h.url_for( controller='admin_toolshed', action='reinstall_repository', id=trans.security.encode_id( repository.id ) )}" method="post" >
|
||||
<div class="form-row">
|
||||
<input type="hidden" name="repo_info_dict" value="${encoded_repo_info_dict}" />
|
||||
</div>
|
||||
<div style="clear: both"></div>
|
||||
<% readme_files_dict = containers_dict.get( 'readme_files', None ) %>
|
||||
%if readme_files_dict:
|
||||
<div class="form-row">
|
||||
<table class="colored" width="100%">
|
||||
<th bgcolor="#EBD9B2">Repository README files - may contain important installation or license information</th>
|
||||
</table>
|
||||
</div>
|
||||
${render_readme_section( containers_dict )}
|
||||
<div style="clear: both"></div>
|
||||
%endif
|
||||
%if has_repository_dependencies or includes_tool_dependencies:
|
||||
<div class="form-row">
|
||||
<table class="colored" width="100%">
|
||||
<th bgcolor="#EBD9B2">Confirm dependency installation</th>
|
||||
</table>
|
||||
</div>
|
||||
${render_dependencies_section( install_resolver_dependencies_check_box, install_repository_dependencies_check_box, containers_dict, revision_label=None, export=False )}
|
||||
%endif
|
||||
%if shed_tool_conf_select_field:
|
||||
<div class="form-row">
|
||||
<table class="colored" width="100%">
|
||||
<th bgcolor="#EBD9B2">Choose the configuration file whose tool_path setting will be used for installing repositories</th>
|
||||
</table>
|
||||
</div>
|
||||
<%
|
||||
if len( shed_tool_conf_select_field.options ) == 1:
|
||||
select_help = "Your Galaxy instance is configured with 1 shed-related tool configuration file, so repositories will be "
|
||||
select_help += "installed using its <b>tool_path</b> setting."
|
||||
else:
|
||||
select_help = "Your Galaxy instance is configured with %d shed-related tool configuration files, " % len( shed_tool_conf_select_field.options )
|
||||
select_help += "so select the file whose <b>tool_path</b> setting you want used for installing repositories."
|
||||
%>
|
||||
<div class="form-row">
|
||||
<label>Shed tool configuration file:</label>
|
||||
${render_select(shed_tool_conf_select_field)}
|
||||
<div class="toolParamHelp" style="clear: both;">
|
||||
${select_help|h}
|
||||
</div>
|
||||
</div>
|
||||
<div style="clear: both"></div>
|
||||
%else:
|
||||
<input type="hidden" name="shed_tool_conf" value="${shed_tool_conf|h}"/>
|
||||
%endif
|
||||
%if includes_tools_for_display_in_tool_panel:
|
||||
<div style="clear: both"></div>
|
||||
<div class="form-row">
|
||||
${render_checkbox(no_changes_check_box)}
|
||||
<label style="display: inline;">No changes</label>
|
||||
<div class="toolParamHelp" style="clear: both;">
|
||||
Uncheck and select a different tool panel section to load the tools into a different section in the tool panel.
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-row">
|
||||
<label>Add new tool panel section:</label>
|
||||
<input name="new_tool_panel_section_label" type="textfield" value="" size="40"/>
|
||||
<div class="toolParamHelp" style="clear: both;">
|
||||
Add a new tool panel section to contain the installed tools (optional).
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-row">
|
||||
<label>Select existing tool panel section:</label>
|
||||
${render_select(tool_panel_section_select_field)}
|
||||
<div class="toolParamHelp" style="clear: both;">
|
||||
Choose an existing section in your tool panel to contain the installed tools (optional).
|
||||
</div>
|
||||
</div>
|
||||
%endif
|
||||
<div class="form-row">
|
||||
<input type="submit" name="select_tool_panel_section_button" value="Install"/>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
@@ -1,188 +0,0 @@
|
||||
<%inherit file="/base.mako"/>
|
||||
<%namespace file="/message.mako" import="render_msg" />
|
||||
<%namespace file="/admin/tool_shed_repository/common.mako" import="render_dependencies_section" />
|
||||
<%namespace file="/admin/tool_shed_repository/common.mako" import="render_readme_section" />
|
||||
<%namespace file="/webapps/tool_shed/repository/common.mako" import="*" />
|
||||
<%namespace file="/webapps/tool_shed/common/common.mako" import="*" />
|
||||
|
||||
<%def name="stylesheets()">
|
||||
${parent.stylesheets()}
|
||||
${h.css( "library" )}
|
||||
</%def>
|
||||
|
||||
<%def name="javascripts()">
|
||||
${parent.javascripts()}
|
||||
${container_javascripts()}
|
||||
</%def>
|
||||
|
||||
<%
|
||||
# Handle the case where an uninstalled repository encountered errors during the process of being reinstalled. In
|
||||
# this case, the repository metadata is an empty dictionary, but one or both of has_repository_dependencies
|
||||
# and includes_tool_dependencies may be True. If either of these are True but we have no metadata, we cannot install
|
||||
# repository dependencies on this pass.
|
||||
if has_repository_dependencies:
|
||||
repository_dependencies = containers_dict[ 'repository_dependencies' ]
|
||||
missing_repository_dependencies = containers_dict[ 'missing_repository_dependencies' ]
|
||||
if repository_dependencies or missing_repository_dependencies:
|
||||
can_display_repository_dependencies = True
|
||||
else:
|
||||
can_display_repository_dependencies = False
|
||||
else:
|
||||
can_display_repository_dependencies = False
|
||||
if includes_tool_dependencies:
|
||||
tool_dependencies = containers_dict[ 'tool_dependencies' ]
|
||||
missing_tool_dependencies = containers_dict[ 'missing_tool_dependencies' ]
|
||||
if tool_dependencies or missing_tool_dependencies:
|
||||
can_display_tool_dependencies = True
|
||||
else:
|
||||
can_display_tool_dependencies = False
|
||||
else:
|
||||
can_display_tool_dependencies = False
|
||||
can_display_resolver_installation = install_resolver_dependencies_check_box is not None
|
||||
%>
|
||||
|
||||
%if message:
|
||||
${render_msg( message, status )}
|
||||
%endif
|
||||
|
||||
<div class="warningmessage">
|
||||
<p>
|
||||
The Galaxy development team does not maintain the contents of many Galaxy Tool Shed repositories. Some
|
||||
repository tools may include code that produces malicious behavior, so be aware of what you are installing.
|
||||
</p>
|
||||
<p>
|
||||
If you discover a repository that causes problems after installation, contact <a href="https://galaxyproject.org/support" target="_blank">Galaxy support</a>,
|
||||
sending all necessary information, and appropriate action will be taken.
|
||||
</p>
|
||||
<p>
|
||||
<a href="https://galaxyproject.org/toolshed/repository-features/#contact-repository-owner" target="_blank">Contact the repository owner</a> for
|
||||
general questions or concerns.
|
||||
</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div class="card-body">
|
||||
<form name="select_tool_panel_section" id="select_tool_panel_section" action="${h.url_for( controller='admin_toolshed', action='prepare_for_install' )}" method="post" >
|
||||
<div class="form-row">
|
||||
<input type="hidden" name="includes_tools" value="${includes_tools}" />
|
||||
<input type="hidden" name="includes_tool_dependencies" value="${includes_tool_dependencies}" />
|
||||
<input type="hidden" name="requirements_status" value="${requirements_status}" />
|
||||
<input type="hidden" name="includes_tools_for_display_in_tool_panel" value="${includes_tools_for_display_in_tool_panel}" />
|
||||
<input type="hidden" name="tool_shed_url" value="${tool_shed_url}" />
|
||||
<input type="hidden" name="encoded_repo_info_dicts" value="${encoded_repo_info_dicts}" />
|
||||
<input type="hidden" name="updating" value="${updating}" />
|
||||
<input type="hidden" name="updating_repository_id" value="${updating_repository_id}" />
|
||||
<input type="hidden" name="updating_to_ctx_rev" value="${updating_to_ctx_rev}" />
|
||||
<input type="hidden" name="updating_to_changeset_revision" value="${updating_to_changeset_revision}" />
|
||||
<input type="hidden" name="encoded_updated_metadata" value="${encoded_updated_metadata}" />
|
||||
</div>
|
||||
<div style="clear: both"></div>
|
||||
<% readme_files_dict = containers_dict.get( 'readme_files', None ) %>
|
||||
%if readme_files_dict:
|
||||
<div class="form-row">
|
||||
<table class="colored" width="100%">
|
||||
<th bgcolor="#EBD9B2">Repository README file - may contain important installation or license information</th>
|
||||
</table>
|
||||
</div>
|
||||
${render_readme_section( containers_dict )}
|
||||
<div style="clear: both"></div>
|
||||
%endif
|
||||
<%
|
||||
if requirements_status and install_resolver_dependencies_check_box or includes_tool_dependencies:
|
||||
display_dependency_confirmation = True
|
||||
else:
|
||||
display_dependency_confirmation = False
|
||||
%>
|
||||
%if requirements_status:
|
||||
%if not install_resolver_dependencies_check_box and not includes_tool_dependencies:
|
||||
<div class="form-row">
|
||||
<table class="colored" width="100%">
|
||||
<head>
|
||||
<th>
|
||||
<img src="${h.url_for('/static')}/images/icon_error_sml.gif" title='Cannot install dependencies'/>
|
||||
This repository requires dependencies that cannot be installed through the Tool Shed
|
||||
</th>
|
||||
</head>
|
||||
</table>
|
||||
</div>
|
||||
<div class="form-row">
|
||||
<p>This repository defines tool requirements that cannot be installed through the Tool Shed.</p>
|
||||
<p>Please activate Conda dependency resolution, activate Docker dependency resolution, setup Environment Modules
|
||||
or manually satisfy the dependencies listed below.</p>
|
||||
<p>For details see <a target="_blank" href="https://docs.galaxyproject.org/en/master/admin/dependency_resolvers.html">the dependency resolver documentation.</a></p>
|
||||
</div>
|
||||
%endif
|
||||
<div class="form-row">
|
||||
<table class="colored" width="100%">
|
||||
<th bgcolor="#EBD9B2">The following tool dependencies are required by the current repository</th>
|
||||
</table>
|
||||
</div>
|
||||
<div class="form-row">
|
||||
${render_tool_dependency_resolver( requirements_status, prepare_for_install=True )}
|
||||
</div>
|
||||
<div style="clear: both"></div>
|
||||
%endif
|
||||
%if can_display_repository_dependencies or display_dependency_confirmation:
|
||||
<div class="form-row">
|
||||
<table class="colored" width="100%">
|
||||
<th bgcolor="#EBD9B2">Confirm dependency installation</th>
|
||||
</table>
|
||||
</div>
|
||||
${render_dependencies_section( install_resolver_dependencies_check_box, install_repository_dependencies_check_box, containers_dict, revision_label=None, export=False, requirements_status=requirements_status )}
|
||||
<div style="clear: both"></div>
|
||||
%endif
|
||||
%if shed_tool_conf_select_field:
|
||||
%if includes_tools_for_display_in_tool_panel:
|
||||
<div class="form-row">
|
||||
<table class="colored" width="100%">
|
||||
<th bgcolor="#EBD9B2">Choose the tool panel section to contain the installed tools (optional)</th>
|
||||
</table>
|
||||
</div>
|
||||
<div class="detail-section">
|
||||
%endif
|
||||
<%
|
||||
if len( shed_tool_conf_select_field.options ) == 1:
|
||||
select_help = "Your Galaxy instance is configured with 1 shed-related tool configuration file, so repositories will be "
|
||||
select_help += "installed using its <b>tool_path</b> setting."
|
||||
else:
|
||||
select_help = "Your Galaxy instance is configured with %d shed-related tool configuration files, " % len( shed_tool_conf_select_field.options )
|
||||
select_help += "so select the file whose <b>tool_path</b> setting you want used for installing repositories."
|
||||
%>
|
||||
<div class="form-row">
|
||||
<label>Shed tool configuration file:</label>
|
||||
${render_select(shed_tool_conf_select_field)}
|
||||
<div class="toolParamHelp" style="clear: both;">
|
||||
${select_help|h}
|
||||
</div>
|
||||
</div>
|
||||
<div style="clear: both"></div>
|
||||
</div>
|
||||
%else:
|
||||
<input type="hidden" name="shed_tool_conf" value="${shed_tool_conf|h}"/>
|
||||
%endif
|
||||
%if includes_tools_for_display_in_tool_panel:
|
||||
<div class="form-row">
|
||||
<label>Add new tool panel section:</label>
|
||||
<input name="new_tool_panel_section_label" type="textfield" value="${new_tool_panel_section_label|h}" size="40"/>
|
||||
<div class="toolParamHelp" style="clear: both;">
|
||||
Add a new tool panel section to contain the installed tools (optional).
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-row">
|
||||
<label>Select existing tool panel section:</label>
|
||||
${render_select(tool_panel_section_select_field)}
|
||||
<div class="toolParamHelp" style="clear: both;">
|
||||
Choose an existing section in your tool panel to contain the installed tools (optional).
|
||||
</div>
|
||||
</div>
|
||||
%endif
|
||||
<div class="form-row">
|
||||
<input type="submit" name="select_tool_panel_section_button" value="Install"/>
|
||||
<div class="toolParamHelp" style="clear: both;">
|
||||
%if includes_tools_for_display_in_tool_panel:
|
||||
Clicking <b>Install</b> without selecting a tool panel section will load the installed tools into the tool panel outside of any sections.
|
||||
%endif
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
Reference in New Issue
Block a user