mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-10-13 08:51:38 +08:00
Merge pull request #13975 from jmchilton/remove_tool_dependencies_packages_5
Remove can_eliminate_tool_dependency from tool shed code.
This commit is contained in:
Björn Grüning
@@ -1,11 +1,7 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
|
||||
from fabric.api import lcd
|
||||
from sqlalchemy import or_
|
||||
|
||||
from galaxy import (
|
||||
@@ -17,498 +13,26 @@ from galaxy.tool_shed.galaxy_install.metadata.installed_repository_metadata_mana
|
||||
InstalledRepositoryMetadataManager,
|
||||
)
|
||||
from galaxy.tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager
|
||||
from galaxy.tool_shed.galaxy_install.tool_dependencies.recipe.env_file_builder import EnvFileBuilder
|
||||
from galaxy.tool_shed.galaxy_install.tool_dependencies.recipe.install_environment import InstallEnvironment
|
||||
from galaxy.tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import (
|
||||
StepManager,
|
||||
TagManager,
|
||||
)
|
||||
from galaxy.tool_shed.galaxy_install.tools import (
|
||||
data_manager,
|
||||
tool_panel_manager,
|
||||
)
|
||||
from galaxy.tool_shed.tools.data_table_manager import ShedToolDataTableManager
|
||||
from galaxy.tool_shed.util import (
|
||||
basic_util,
|
||||
hg_util,
|
||||
repository_util,
|
||||
)
|
||||
from galaxy.tool_shed.util import shed_util_common as suc
|
||||
from galaxy.tool_shed.util import (
|
||||
tool_dependency_util,
|
||||
tool_util,
|
||||
)
|
||||
from galaxy.tool_shed.util import tool_util
|
||||
from galaxy.tool_util.deps import views
|
||||
from galaxy.util.tool_shed import (
|
||||
common_util,
|
||||
encoding_util,
|
||||
xml_util,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class InstallToolDependencyManager:
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
self.install_model = self.app.install_model
|
||||
self.INSTALL_ACTIONS = [
|
||||
"download_binary",
|
||||
"download_by_url",
|
||||
"download_file",
|
||||
"setup_perl_environment",
|
||||
"setup_python_environment",
|
||||
"setup_r_environment",
|
||||
"setup_ruby_environment",
|
||||
"shell_command",
|
||||
]
|
||||
|
||||
def format_traceback(self):
|
||||
ex_type, ex, tb = sys.exc_info()
|
||||
return "".join(traceback.format_tb(tb))
|
||||
|
||||
def get_tool_shed_repository_install_dir(self, tool_shed_repository):
|
||||
return os.path.abspath(tool_shed_repository.repo_files_directory(self.app))
|
||||
|
||||
def install_and_build_package(self, install_environment, tool_dependency, actions_dict):
|
||||
"""Install a Galaxy tool dependency package either via a url or a mercurial or git clone command."""
|
||||
install_dir = actions_dict["install_dir"]
|
||||
package_name = actions_dict["package_name"]
|
||||
actions = actions_dict.get("actions", None)
|
||||
filtered_actions = []
|
||||
env_file_builder = EnvFileBuilder(install_dir)
|
||||
step_manager = StepManager(self.app)
|
||||
if actions:
|
||||
with install_environment.use_tmp_dir() as work_dir:
|
||||
with lcd(work_dir):
|
||||
# The first action in the list of actions will be the one that defines the initial download process.
|
||||
# There are currently three supported actions; download_binary, download_by_url and clone via a
|
||||
# shell_command action type. The recipe steps will be filtered at this stage in the process, with
|
||||
# the filtered actions being used in the next stage below. The installation directory (i.e., dir)
|
||||
# is also defined in this stage and is used in the next stage below when defining current_dir.
|
||||
action_type, action_dict = actions[0]
|
||||
if action_type in self.INSTALL_ACTIONS:
|
||||
# Some of the parameters passed here are needed only by a subset of the step handler classes,
|
||||
# but to allow for a standard method signature we'll pass them along. We don't check the
|
||||
# tool_dependency status in this stage because it should not have been changed based on a
|
||||
# download.
|
||||
tool_dependency, filtered_actions, dir = step_manager.execute_step(
|
||||
tool_dependency=tool_dependency,
|
||||
package_name=package_name,
|
||||
actions=actions,
|
||||
action_type=action_type,
|
||||
action_dict=action_dict,
|
||||
filtered_actions=filtered_actions,
|
||||
env_file_builder=env_file_builder,
|
||||
install_environment=install_environment,
|
||||
work_dir=work_dir,
|
||||
current_dir=None,
|
||||
initial_download=True,
|
||||
)
|
||||
else:
|
||||
# We're handling a complex repository dependency where we only have a set_environment tag set.
|
||||
# <action type="set_environment">
|
||||
# <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR/bin</environment_variable>
|
||||
# </action>
|
||||
filtered_actions = [a for a in actions]
|
||||
dir = install_dir
|
||||
# We're in stage 2 of the installation process. The package has been down-loaded, so we can
|
||||
# now perform all of the actions defined for building it.
|
||||
for action_tup in filtered_actions:
|
||||
if dir is None:
|
||||
dir = ""
|
||||
current_dir = os.path.abspath(os.path.join(work_dir, dir))
|
||||
with lcd(current_dir):
|
||||
action_type, action_dict = action_tup
|
||||
tool_dependency, tmp_filtered_actions, tmp_dir = step_manager.execute_step(
|
||||
tool_dependency=tool_dependency,
|
||||
package_name=package_name,
|
||||
actions=actions,
|
||||
action_type=action_type,
|
||||
action_dict=action_dict,
|
||||
filtered_actions=filtered_actions,
|
||||
env_file_builder=env_file_builder,
|
||||
install_environment=install_environment,
|
||||
work_dir=work_dir,
|
||||
current_dir=current_dir,
|
||||
initial_download=False,
|
||||
)
|
||||
if tool_dependency.status in [self.install_model.ToolDependency.installation_status.ERROR]:
|
||||
# If the tool_dependency status is in an error state, return it with no additional
|
||||
# processing.
|
||||
return tool_dependency
|
||||
# Make sure to handle the special case where the value of dir is reset (this happens when
|
||||
# the action_type is change_directiory). In all other action types, dir will be returned as
|
||||
# None.
|
||||
if tmp_dir is not None:
|
||||
dir = tmp_dir
|
||||
return tool_dependency
|
||||
|
||||
def install_and_build_package_via_fabric(
|
||||
self, install_environment, tool_shed_repository, tool_dependency, actions_dict
|
||||
):
|
||||
try:
|
||||
# There is currently only one fabric method.
|
||||
tool_dependency = self.install_and_build_package(install_environment, tool_dependency, actions_dict)
|
||||
except Exception as e:
|
||||
log.exception(
|
||||
"Error installing tool dependency %s version %s.", tool_dependency.name, tool_dependency.version
|
||||
)
|
||||
# Since there was an installation error, update the tool dependency status to Error. The remove_installation_path option must
|
||||
# be left False here.
|
||||
error_message = f"{self.format_traceback()}\n{util.unicodify(e)}"
|
||||
tool_dependency = tool_dependency_util.set_tool_dependency_attributes(
|
||||
self.app,
|
||||
tool_dependency=tool_dependency,
|
||||
status=self.app.install_model.ToolDependency.installation_status.ERROR,
|
||||
error_message=error_message,
|
||||
)
|
||||
tool_dependency = self.mark_tool_dependency_installed(tool_dependency)
|
||||
return tool_dependency
|
||||
|
||||
def install_specified_tool_dependencies(self, tool_shed_repository, tool_dependencies_config, tool_dependencies):
|
||||
"""
|
||||
Follow the recipe in the received tool_dependencies_config to install specified packages for
|
||||
repository tools. The received list of tool_dependencies are the database records for those
|
||||
dependencies defined in the tool_dependencies_config that are to be installed. This list may
|
||||
be a subset of the set of dependencies defined in the tool_dependencies_config. This allows
|
||||
for filtering out dependencies that have not been checked for installation on the 'Manage tool
|
||||
dependencies' page for an installed Tool Shed repository.
|
||||
"""
|
||||
attr_tups_of_dependencies_for_install = [(td.name, td.version, td.type) for td in tool_dependencies]
|
||||
installed_packages = []
|
||||
tag_manager = TagManager(self.app)
|
||||
# Parse the tool_dependencies.xml config.
|
||||
tree, error_message = xml_util.parse_xml(tool_dependencies_config)
|
||||
if tree is None:
|
||||
log.debug(f"The received tool_dependencies.xml file is likely invalid: {error_message}")
|
||||
return installed_packages
|
||||
root = tree.getroot()
|
||||
elems = []
|
||||
for elem in root:
|
||||
if elem.tag == "set_environment":
|
||||
version = elem.get("version", "1.0")
|
||||
if version != "1.0":
|
||||
raise Exception("The <set_environment> tag must have a version attribute with value 1.0")
|
||||
for sub_elem in elem:
|
||||
elems.append(sub_elem)
|
||||
else:
|
||||
elems.append(elem)
|
||||
for elem in elems:
|
||||
name = elem.get("name", None)
|
||||
version = elem.get("version", None)
|
||||
type = elem.get("type", None)
|
||||
if type is None:
|
||||
if elem.tag in ["environment_variable", "set_environment"]:
|
||||
type = "set_environment"
|
||||
else:
|
||||
type = "package"
|
||||
if (name and type == "set_environment") or (name and version):
|
||||
# elem is a package set_environment tag set.
|
||||
attr_tup = (name, version, type)
|
||||
try:
|
||||
index = attr_tups_of_dependencies_for_install.index(attr_tup)
|
||||
except ValueError:
|
||||
index = None
|
||||
if index is not None:
|
||||
tool_dependency = tool_dependencies[index]
|
||||
# If the tool_dependency.type is 'set_environment', then the call to process_tag_set() will
|
||||
# handle everything - no additional installation is necessary.
|
||||
tool_dependency, proceed_with_install, action_elem_tuples = tag_manager.process_tag_set(
|
||||
tool_shed_repository,
|
||||
tool_dependency,
|
||||
elem,
|
||||
name,
|
||||
version,
|
||||
tool_dependency_db_records=tool_dependencies,
|
||||
)
|
||||
if tool_dependency.type == "package" and proceed_with_install:
|
||||
try:
|
||||
tool_dependency = self.install_package(
|
||||
elem, tool_shed_repository, tool_dependencies=tool_dependencies
|
||||
)
|
||||
except Exception as e:
|
||||
error_message = f"Error installing tool dependency {name} version {version}: {e}"
|
||||
log.exception(error_message)
|
||||
if tool_dependency:
|
||||
# Since there was an installation error, update the tool dependency status to Error. The
|
||||
# remove_installation_path option must be left False here.
|
||||
tool_dependency = tool_dependency_util.set_tool_dependency_attributes(
|
||||
self.app,
|
||||
tool_dependency=tool_dependency,
|
||||
status=self.app.install_model.ToolDependency.installation_status.ERROR,
|
||||
error_message=error_message,
|
||||
)
|
||||
if tool_dependency and tool_dependency.status in [
|
||||
self.install_model.ToolDependency.installation_status.INSTALLED,
|
||||
self.install_model.ToolDependency.installation_status.ERROR,
|
||||
]:
|
||||
installed_packages.append(tool_dependency)
|
||||
return installed_packages
|
||||
|
||||
def install_via_fabric(
|
||||
self,
|
||||
tool_shed_repository,
|
||||
tool_dependency,
|
||||
install_dir,
|
||||
package_name=None,
|
||||
custom_fabfile_path=None,
|
||||
actions_elem=None,
|
||||
action_elem=None,
|
||||
**kwd,
|
||||
):
|
||||
"""
|
||||
Parse a tool_dependency.xml file's <actions> tag set to gather information for installation using
|
||||
self.install_and_build_package(). The use of fabric is being eliminated, so some of these functions
|
||||
may need to be renamed at some point.
|
||||
"""
|
||||
if not os.path.exists(install_dir):
|
||||
os.makedirs(install_dir)
|
||||
actions_dict = dict(install_dir=install_dir)
|
||||
if package_name:
|
||||
actions_dict["package_name"] = package_name
|
||||
actions = []
|
||||
is_binary_download = False
|
||||
if actions_elem is not None:
|
||||
elems = actions_elem
|
||||
if elems.get("os") is not None and elems.get("architecture") is not None:
|
||||
is_binary_download = True
|
||||
elif action_elem is not None:
|
||||
# We were provided with a single <action> element to perform certain actions after a platform-specific tarball was downloaded.
|
||||
elems = [action_elem]
|
||||
else:
|
||||
elems = []
|
||||
step_manager = StepManager(self.app)
|
||||
tool_shed_repository_install_dir = self.get_tool_shed_repository_install_dir(tool_shed_repository)
|
||||
install_environment = InstallEnvironment(self.app, tool_shed_repository_install_dir, install_dir)
|
||||
for action_elem in elems:
|
||||
# Make sure to skip all comments, since they are now included in the XML tree.
|
||||
if action_elem.tag != "action":
|
||||
continue
|
||||
action_dict = {}
|
||||
action_type = action_elem.get("type", None)
|
||||
if action_type is not None:
|
||||
action_dict = step_manager.prepare_step(
|
||||
tool_dependency=tool_dependency,
|
||||
action_type=action_type,
|
||||
action_elem=action_elem,
|
||||
action_dict=action_dict,
|
||||
install_environment=install_environment,
|
||||
is_binary_download=is_binary_download,
|
||||
)
|
||||
action_tuple = (action_type, action_dict)
|
||||
if action_type == "set_environment":
|
||||
if action_tuple not in actions:
|
||||
actions.append(action_tuple)
|
||||
else:
|
||||
actions.append(action_tuple)
|
||||
if actions:
|
||||
actions_dict["actions"] = actions
|
||||
if custom_fabfile_path is not None:
|
||||
# TODO: this is not yet supported or functional, but when it is handle it using the fabric api.
|
||||
raise Exception("Tool dependency installation using proprietary fabric scripts is not yet supported.")
|
||||
else:
|
||||
tool_dependency = self.install_and_build_package_via_fabric(
|
||||
install_environment, tool_shed_repository, tool_dependency, actions_dict
|
||||
)
|
||||
return tool_dependency
|
||||
|
||||
def install_package(self, elem, tool_shed_repository, tool_dependencies=None):
|
||||
"""
|
||||
Install a tool dependency package defined by the XML element elem. The value of tool_dependencies is
|
||||
a partial or full list of ToolDependency records associated with the tool_shed_repository.
|
||||
"""
|
||||
tag_manager = TagManager(self.app)
|
||||
# The value of package_name should match the value of the "package" type in the tool config's
|
||||
# <requirements> tag set, but it's not required.
|
||||
package_name = elem.get("name", None)
|
||||
package_version = elem.get("version", None)
|
||||
if tool_dependencies and package_name and package_version:
|
||||
tool_dependency = None
|
||||
for tool_dependency in tool_dependencies:
|
||||
if package_name == str(tool_dependency.name) and package_version == str(tool_dependency.version):
|
||||
break
|
||||
if tool_dependency is not None:
|
||||
for package_elem in elem:
|
||||
tool_dependency, proceed_with_install, actions_elem_tuples = tag_manager.process_tag_set(
|
||||
tool_shed_repository,
|
||||
tool_dependency,
|
||||
package_elem,
|
||||
package_name,
|
||||
package_version,
|
||||
tool_dependency_db_records=None,
|
||||
)
|
||||
if proceed_with_install and actions_elem_tuples:
|
||||
# Get the installation directory for tool dependencies that will be installed for the received
|
||||
# tool_shed_repository.
|
||||
install_dir = tool_dependency_util.get_tool_dependency_install_dir(
|
||||
app=self.app,
|
||||
repository_name=tool_shed_repository.name,
|
||||
repository_owner=tool_shed_repository.owner,
|
||||
repository_changeset_revision=tool_shed_repository.installed_changeset_revision,
|
||||
tool_dependency_type="package",
|
||||
tool_dependency_name=package_name,
|
||||
tool_dependency_version=package_version,
|
||||
)
|
||||
# At this point we have a list of <actions> elems that are either defined within an <actions_group>
|
||||
# tag set with <actions> sub-elements that contains os and architecture attributes filtered by the
|
||||
# platform into which the appropriate compiled binary will be installed, or not defined within an
|
||||
# <actions_group> tag set and not filtered. Here is an example actions_elem_tuple.
|
||||
# [(True, [<Element 'actions' at 0x109293d10>)]
|
||||
binary_installed = False
|
||||
for actions_elem_tuple in actions_elem_tuples:
|
||||
in_actions_group, actions_elems = actions_elem_tuple
|
||||
if in_actions_group:
|
||||
# Platform matching is only performed inside <actions_group> tag sets, os and architecture
|
||||
# attributes are otherwise ignored.
|
||||
can_install_from_source = False
|
||||
for actions_elem in actions_elems:
|
||||
system = actions_elem.get("os")
|
||||
architecture = actions_elem.get("architecture")
|
||||
# If this <actions> element has the os and architecture attributes defined, then we only
|
||||
# want to process until a successful installation is achieved.
|
||||
if system and architecture:
|
||||
# If an <actions> tag has been defined that matches our current platform, and the
|
||||
# recipe specified within that <actions> tag has been successfully processed, skip
|
||||
# any remaining platform-specific <actions> tags. We cannot break out of the loop
|
||||
# here because there may be <action> tags at the end of the <actions_group> tag set
|
||||
# that must be processed.
|
||||
if binary_installed:
|
||||
continue
|
||||
# No platform-specific <actions> recipe has yet resulted in a successful installation.
|
||||
tool_dependency = self.install_via_fabric(
|
||||
tool_shed_repository,
|
||||
tool_dependency,
|
||||
install_dir,
|
||||
package_name=package_name,
|
||||
actions_elem=actions_elem,
|
||||
action_elem=None,
|
||||
)
|
||||
if (
|
||||
tool_dependency.status
|
||||
== self.install_model.ToolDependency.installation_status.INSTALLED
|
||||
):
|
||||
# If an <actions> tag was found that matches the current platform, and
|
||||
# self.install_via_fabric() did not result in an error state, set binary_installed
|
||||
# to True in order to skip any remaining platform-specific <actions> tags.
|
||||
binary_installed = True
|
||||
else:
|
||||
# Process the next matching <actions> tag, or any defined <actions> tags that do not
|
||||
# contain platform dependent recipes.
|
||||
log.debug(
|
||||
"Error downloading binary for tool dependency %s version %s: %s"
|
||||
% (
|
||||
str(package_name),
|
||||
str(package_version),
|
||||
str(tool_dependency.error_message),
|
||||
)
|
||||
)
|
||||
else:
|
||||
if actions_elem.tag == "actions":
|
||||
# We've reached an <actions> tag that defines the recipe for installing and compiling from
|
||||
# source. If binary installation failed, we proceed with the recipe.
|
||||
if not binary_installed:
|
||||
installation_directory = tool_dependency.installation_directory(
|
||||
self.app
|
||||
)
|
||||
if os.path.exists(installation_directory):
|
||||
# Delete contents of installation directory if attempt at binary installation failed.
|
||||
installation_directory_contents = os.listdir(installation_directory)
|
||||
if installation_directory_contents:
|
||||
(
|
||||
removed,
|
||||
error_message,
|
||||
) = tool_dependency_util.remove_tool_dependency(
|
||||
self.app, tool_dependency
|
||||
)
|
||||
if removed:
|
||||
can_install_from_source = True
|
||||
else:
|
||||
log.debug(
|
||||
"Error removing old files from installation directory %s: %s"
|
||||
% (str(installation_directory, str(error_message)))
|
||||
)
|
||||
else:
|
||||
can_install_from_source = True
|
||||
else:
|
||||
can_install_from_source = True
|
||||
if can_install_from_source:
|
||||
# We now know that binary installation was not successful, so proceed with the <actions>
|
||||
# tag set that defines the recipe to install and compile from source.
|
||||
log.debug(
|
||||
"Proceeding with install and compile recipe for tool dependency %s."
|
||||
% str(tool_dependency.name)
|
||||
)
|
||||
# Reset above error to installing
|
||||
tool_dependency.status = (
|
||||
self.install_model.ToolDependency.installation_status.INSTALLING
|
||||
)
|
||||
tool_dependency = self.install_via_fabric(
|
||||
tool_shed_repository,
|
||||
tool_dependency,
|
||||
install_dir,
|
||||
package_name=package_name,
|
||||
actions_elem=actions_elem,
|
||||
action_elem=None,
|
||||
)
|
||||
if (
|
||||
actions_elem.tag == "action"
|
||||
and tool_dependency.status
|
||||
!= self.install_model.ToolDependency.installation_status.ERROR
|
||||
):
|
||||
# If the tool dependency is not in an error state, perform any final actions that have been
|
||||
# defined within the actions_group tag set, but outside of an <actions> tag, which defines
|
||||
# the recipe for installing and compiling from source.
|
||||
tool_dependency = self.install_via_fabric(
|
||||
tool_shed_repository,
|
||||
tool_dependency,
|
||||
install_dir,
|
||||
package_name=package_name,
|
||||
actions_elem=None,
|
||||
action_elem=actions_elem,
|
||||
)
|
||||
else:
|
||||
# Checks for "os" and "architecture" attributes are not made for any <actions> tag sets outside of
|
||||
# an <actions_group> tag set. If the attributes are defined, they will be ignored. All <actions> tags
|
||||
# outside of an <actions_group> tag set will always be processed.
|
||||
tool_dependency = self.install_via_fabric(
|
||||
tool_shed_repository,
|
||||
tool_dependency,
|
||||
install_dir,
|
||||
package_name=package_name,
|
||||
actions_elem=actions_elems,
|
||||
action_elem=None,
|
||||
)
|
||||
if (
|
||||
tool_dependency.status
|
||||
!= self.install_model.ToolDependency.installation_status.ERROR
|
||||
):
|
||||
log.debug(
|
||||
"Tool dependency %s version %s has been installed in %s."
|
||||
% (str(package_name), str(package_version), str(install_dir))
|
||||
)
|
||||
return tool_dependency
|
||||
|
||||
def mark_tool_dependency_installed(self, tool_dependency):
|
||||
if tool_dependency.status not in [
|
||||
self.install_model.ToolDependency.installation_status.ERROR,
|
||||
self.install_model.ToolDependency.installation_status.INSTALLED,
|
||||
]:
|
||||
log.debug(
|
||||
"Changing status for tool dependency %s from %s to %s."
|
||||
% (
|
||||
str(tool_dependency.name),
|
||||
str(tool_dependency.status),
|
||||
str(self.install_model.ToolDependency.installation_status.INSTALLED),
|
||||
)
|
||||
)
|
||||
status = self.install_model.ToolDependency.installation_status.INSTALLED
|
||||
tool_dependency = tool_dependency_util.set_tool_dependency_attributes(
|
||||
self.app, tool_dependency=tool_dependency, status=status
|
||||
)
|
||||
return tool_dependency
|
||||
|
||||
|
||||
class InstallRepositoryManager:
|
||||
def __init__(self, app, tpm=None):
|
||||
self.app = app
|
||||
@@ -623,10 +147,6 @@ class InstallRepositoryManager:
|
||||
tool_shed_repository.tool_shed_status = tool_shed_status_dict
|
||||
self.install_model.context.add(tool_shed_repository)
|
||||
self.install_model.context.flush()
|
||||
if "tool_dependencies" in irmm_metadata_dict and not reinstalling:
|
||||
tool_dependency_util.create_tool_dependency_objects(
|
||||
self.app, tool_shed_repository, relative_install_dir, set_status=True
|
||||
)
|
||||
if "sample_files" in irmm_metadata_dict:
|
||||
sample_files = irmm_metadata_dict.get("sample_files", [])
|
||||
tool_index_sample_files = stdtm.get_tool_index_sample_files(sample_files)
|
||||
@@ -1130,22 +650,6 @@ class InstallRepositoryManager:
|
||||
if dependency_manager.cached:
|
||||
[dependency_manager.build_cache(r) for r in new_requirements]
|
||||
|
||||
if install_tool_dependencies and tool_shed_repository.tool_dependencies and "tool_dependencies" in metadata:
|
||||
work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-itsr")
|
||||
# Install tool dependencies.
|
||||
self.update_tool_shed_repository_status(
|
||||
tool_shed_repository,
|
||||
self.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES,
|
||||
)
|
||||
# Get the tool_dependencies.xml file from the repository.
|
||||
tool_dependencies_config = hg_util.get_config_from_disk("tool_dependencies.xml", install_dir)
|
||||
itdm = InstallToolDependencyManager(self.app)
|
||||
itdm.install_specified_tool_dependencies(
|
||||
tool_shed_repository=tool_shed_repository,
|
||||
tool_dependencies_config=tool_dependencies_config,
|
||||
tool_dependencies=tool_shed_repository.tool_dependencies,
|
||||
)
|
||||
basic_util.remove_dir(work_dir)
|
||||
self.update_tool_shed_repository_status(
|
||||
tool_shed_repository, self.install_model.ToolShedRepository.installation_status.INSTALLED
|
||||
)
|
||||
@@ -1163,7 +667,6 @@ class InstallRepositoryManager:
|
||||
original_metadata_dict = repository.metadata_
|
||||
original_repository_dependencies_dict = original_metadata_dict.get("repository_dependencies", {})
|
||||
original_repository_dependencies = original_repository_dependencies_dict.get("repository_dependencies", [])
|
||||
original_tool_dependencies_dict = original_metadata_dict.get("tool_dependencies", {})
|
||||
shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir(
|
||||
self.app, repository
|
||||
)
|
||||
@@ -1236,7 +739,6 @@ class InstallRepositoryManager:
|
||||
if "repository_dependencies" in irmm_metadata_dict or "tool_dependencies" in irmm_metadata_dict:
|
||||
new_repository_dependencies_dict = irmm_metadata_dict.get("repository_dependencies", {})
|
||||
new_repository_dependencies = new_repository_dependencies_dict.get("repository_dependencies", [])
|
||||
new_tool_dependencies_dict = irmm_metadata_dict.get("tool_dependencies", {})
|
||||
if new_repository_dependencies:
|
||||
# [[http://localhost:9009', package_picard_1_56_0', devteam', 910b0b056666', False', False']]
|
||||
if new_repository_dependencies == original_repository_dependencies:
|
||||
@@ -1288,16 +790,6 @@ class InstallRepositoryManager:
|
||||
self.install(
|
||||
tool_shed_url, new_name, new_owner, new_changeset_revision, install_options
|
||||
)
|
||||
# Updates received did not include any newly defined repository dependencies but did include
|
||||
# newly defined tool dependencies. If the newly defined tool dependencies are not the same
|
||||
# as the originally defined tool dependencies, we need to install them.
|
||||
if not install_new_dependencies:
|
||||
for new_key, new_val in new_tool_dependencies_dict.items():
|
||||
if new_key not in original_tool_dependencies_dict:
|
||||
return ("tool", irmm_metadata_dict)
|
||||
original_val = original_tool_dependencies_dict[new_key]
|
||||
if new_val != original_val:
|
||||
return ("tool", irmm_metadata_dict)
|
||||
# Updates received did not include any newly defined repository dependencies or newly defined
|
||||
# tool dependencies that need to be installed.
|
||||
repository = self.app.update_repository_manager.update_repository_record(
|
||||
|
||||
@@ -1,147 +0,0 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
from galaxy.tool_shed.util.repository_util import get_repository_for_dependency_relationship
|
||||
from galaxy.util.tool_shed.common_util import remove_protocol_from_tool_shed_url
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EnvManager:
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
|
||||
def create_env_var_dict(self, elem, install_environment):
|
||||
env_var_name = elem.get("name", "PATH")
|
||||
env_var_action = elem.get("action", "prepend_to")
|
||||
env_var_text = None
|
||||
tool_dependency_install_dir = install_environment.install_dir
|
||||
tool_shed_repository_install_dir = install_environment.tool_shed_repository_install_dir
|
||||
if elem.text and elem.text.find("REPOSITORY_INSTALL_DIR") >= 0:
|
||||
if tool_shed_repository_install_dir and elem.text.find("$REPOSITORY_INSTALL_DIR") != -1:
|
||||
env_var_text = elem.text.replace("$REPOSITORY_INSTALL_DIR", tool_shed_repository_install_dir)
|
||||
return dict(name=env_var_name, action=env_var_action, value=env_var_text)
|
||||
else:
|
||||
env_var_text = elem.text.replace("$REPOSITORY_INSTALL_DIR", tool_dependency_install_dir)
|
||||
return dict(name=env_var_name, action=env_var_action, value=env_var_text)
|
||||
if elem.text and elem.text.find("INSTALL_DIR") >= 0:
|
||||
if tool_dependency_install_dir:
|
||||
env_var_text = elem.text.replace("$INSTALL_DIR", tool_dependency_install_dir)
|
||||
return dict(name=env_var_name, action=env_var_action, value=env_var_text)
|
||||
else:
|
||||
env_var_text = elem.text.replace("$INSTALL_DIR", tool_shed_repository_install_dir)
|
||||
return dict(name=env_var_name, action=env_var_action, value=env_var_text)
|
||||
# Allow for environment variables that contain neither REPOSITORY_INSTALL_DIR nor INSTALL_DIR
|
||||
# since there may be command line parameters that are tuned for a Galaxy instance. Allowing them
|
||||
# to be set in one location rather than being hard coded into each tool config is the best approach.
|
||||
# For example:
|
||||
# <environment_variable name="GATK2_SITE_OPTIONS" action="set_to">
|
||||
# "--num_threads 4 --num_cpu_threads_per_data_thread 3 --phone_home STANDARD"
|
||||
# </environment_variable>
|
||||
if elem.text is None:
|
||||
# Also allow for an empty value for the environment variable, which
|
||||
# may makes sense especially for the 'set_to' action
|
||||
elem.text = ""
|
||||
return dict(name=env_var_name, action=env_var_action, value=elem.text)
|
||||
|
||||
def get_env_shell_file_path(self, installation_directory):
|
||||
env_shell_file_name = "env.sh"
|
||||
default_location = os.path.abspath(os.path.join(installation_directory, env_shell_file_name))
|
||||
if os.path.exists(default_location):
|
||||
return default_location
|
||||
for root, _dirs, files in os.walk(installation_directory):
|
||||
for name in files:
|
||||
if name == env_shell_file_name:
|
||||
return os.path.abspath(os.path.join(root, name))
|
||||
return None
|
||||
|
||||
def get_env_shell_file_paths(self, elem):
|
||||
# Currently only the following tag set is supported.
|
||||
# <repository toolshed="http://localhost:9009/" name="package_numpy_1_7" owner="test" changeset_revision="c84c6a8be056">
|
||||
# <package name="numpy" version="1.7.1" />
|
||||
# </repository>
|
||||
env_shell_file_paths = []
|
||||
toolshed = elem.get("toolshed", None)
|
||||
repository_name = elem.get("name", None)
|
||||
repository_owner = elem.get("owner", None)
|
||||
changeset_revision = elem.get("changeset_revision", None)
|
||||
if toolshed and repository_name and repository_owner and changeset_revision:
|
||||
# The protocol is not stored, but the port is if it exists.
|
||||
toolshed = remove_protocol_from_tool_shed_url(toolshed)
|
||||
repository = get_repository_for_dependency_relationship(
|
||||
self.app, toolshed, repository_name, repository_owner, changeset_revision
|
||||
)
|
||||
if repository:
|
||||
for sub_elem in elem:
|
||||
tool_dependency_type = sub_elem.tag
|
||||
tool_dependency_name = sub_elem.get("name")
|
||||
tool_dependency_version = sub_elem.get("version")
|
||||
if tool_dependency_type and tool_dependency_name and tool_dependency_version:
|
||||
# Get the tool_dependency so we can get its installation directory.
|
||||
tool_dependency = None
|
||||
for tool_dependency in repository.tool_dependencies:
|
||||
if (
|
||||
tool_dependency.type == tool_dependency_type
|
||||
and tool_dependency.name == tool_dependency_name
|
||||
and tool_dependency.version == tool_dependency_version
|
||||
):
|
||||
break
|
||||
if tool_dependency:
|
||||
installation_directory = tool_dependency.installation_directory(self.app)
|
||||
env_shell_file_path = self.get_env_shell_file_path(installation_directory)
|
||||
if env_shell_file_path:
|
||||
env_shell_file_paths.append(env_shell_file_path)
|
||||
else:
|
||||
error_message = "Skipping tool dependency definition because unable to locate env.sh file for tool dependency "
|
||||
error_message += f"type {tool_dependency_type}, name {tool_dependency_name}, version {tool_dependency_version} for repository {repository.name}"
|
||||
log.debug(error_message)
|
||||
continue
|
||||
else:
|
||||
error_message = (
|
||||
"Skipping tool dependency definition because unable to locate tool dependency "
|
||||
)
|
||||
error_message += f"type {tool_dependency_type}, name {tool_dependency_name}, version {tool_dependency_version} for repository {repository.name}"
|
||||
log.debug(error_message)
|
||||
continue
|
||||
else:
|
||||
error_message = f"Skipping invalid tool dependency definition: type {tool_dependency_type}, name {tool_dependency_name}, version {tool_dependency_version}."
|
||||
log.debug(error_message)
|
||||
continue
|
||||
else:
|
||||
error_message = "Skipping set_environment_for_install definition because unable to locate required installed tool shed repository: "
|
||||
error_message += f"toolshed {toolshed}, name {repository_name}, owner {repository_owner}, changeset_revision {changeset_revision}."
|
||||
log.debug(error_message)
|
||||
else:
|
||||
error_message = (
|
||||
"Skipping invalid set_environment_for_install definition: toolshed %s, name %s, owner %s, changeset_revision %s."
|
||||
% (str(toolshed), str(repository_name), str(repository_owner), str(changeset_revision))
|
||||
)
|
||||
log.debug(error_message)
|
||||
return env_shell_file_paths
|
||||
|
||||
def get_env_shell_file_paths_from_setup_environment_elem(self, all_env_shell_file_paths, elem, action_dict):
|
||||
"""
|
||||
Parse an XML tag set to discover all child repository dependency tags and define the path to an env.sh file associated
|
||||
with the repository (this requires the repository dependency to be in an installed state). The received action_dict
|
||||
will be updated with these discovered paths and returned to the caller. This method handles tool dependency definition
|
||||
tag sets <setup_r_environment>, <setup_ruby_environment>, <setup_python_environment> and <setup_perl_environment>.
|
||||
"""
|
||||
# An example elem is:
|
||||
# <action type="setup_perl_environment">
|
||||
# <repository name="package_perl_5_18" owner="iuc">
|
||||
# <package name="perl" version="5.18.1" />
|
||||
# </repository>
|
||||
# <repository name="package_expat_2_1" owner="iuc" prior_installation_required="True">
|
||||
# <package name="expat" version="2.1.0" />
|
||||
# </repository>
|
||||
# <package>http://search.cpan.org/CPAN/authors/id/T/TO/TODDR/XML-Parser-2.41.tar.gz</package>
|
||||
# <package>http://search.cpan.org/CPAN/authors/id/L/LD/LDS/CGI.pm-3.43.tar.gz</package>
|
||||
# </action>
|
||||
for action_elem in elem:
|
||||
if action_elem.tag == "repository":
|
||||
env_shell_file_paths = self.get_env_shell_file_paths(action_elem)
|
||||
all_env_shell_file_paths.extend(env_shell_file_paths)
|
||||
if all_env_shell_file_paths:
|
||||
action_dict["env_shell_file_paths"] = all_env_shell_file_paths
|
||||
action_dict["action_shell_file_paths"] = all_env_shell_file_paths
|
||||
return action_dict
|
||||
@@ -1,33 +0,0 @@
|
||||
import logging
|
||||
import threading
|
||||
|
||||
from galaxy.util import unicodify
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AsynchronousReader(threading.Thread):
|
||||
"""
|
||||
A helper class to implement asynchronous reading of a stream in a separate thread. Read lines are pushed
|
||||
onto a queue to be consumed in another thread.
|
||||
"""
|
||||
|
||||
def __init__(self, fd, queue):
|
||||
threading.Thread.__init__(self)
|
||||
self._fd = fd
|
||||
self._queue = queue
|
||||
self.lines = []
|
||||
|
||||
def run(self):
|
||||
"""Read lines and put them on the queue."""
|
||||
thread_lock = threading.Lock()
|
||||
thread_lock.acquire()
|
||||
for line in iter(self._fd.readline, b""):
|
||||
stripped_line = unicodify(line).rstrip()
|
||||
self.lines.append(stripped_line)
|
||||
self._queue.put(stripped_line)
|
||||
thread_lock.release()
|
||||
|
||||
def installation_complete(self):
|
||||
"""Make sure there is more installation and compilation logging content expected."""
|
||||
return not self.is_alive() and self._queue.empty()
|
||||
@@ -1,94 +0,0 @@
|
||||
import logging
|
||||
import os
|
||||
import stat
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EnvFileBuilder:
|
||||
def __init__(self, install_dir):
|
||||
self.install_dir = install_dir
|
||||
self.return_code = 0
|
||||
|
||||
def append_line(self, make_executable=True, **kwd):
|
||||
env_var_dict = dict(**kwd)
|
||||
env_entry, env_file = self.create_or_update_env_shell_file(self.install_dir, env_var_dict)
|
||||
return_code = self.file_append(env_entry, env_file, make_executable=make_executable)
|
||||
self.return_code = self.return_code or return_code
|
||||
return self.return_code
|
||||
|
||||
@staticmethod
|
||||
def create_or_update_env_shell_file(install_dir, env_var_dict):
|
||||
env_var_action = env_var_dict["action"]
|
||||
env_var_value = env_var_dict["value"]
|
||||
if env_var_action in ["prepend_to", "set_to", "append_to"]:
|
||||
env_var_name = env_var_dict["name"]
|
||||
if env_var_action == "prepend_to":
|
||||
changed_value = f"{env_var_value}:${env_var_name}"
|
||||
elif env_var_action == "set_to":
|
||||
changed_value = f"{env_var_value}"
|
||||
elif env_var_action == "append_to":
|
||||
changed_value = f"${env_var_name}:{env_var_value}"
|
||||
line = f"{env_var_name}={changed_value}; export {env_var_name}"
|
||||
elif env_var_action == "source":
|
||||
line = f"if [ -f {env_var_value} ] ; then . {env_var_value} ; fi"
|
||||
else:
|
||||
raise Exception(f"Unknown shell file action {env_var_action}")
|
||||
env_shell_file_path = os.path.join(install_dir, "env.sh")
|
||||
return line, env_shell_file_path
|
||||
|
||||
def file_append(self, text, file_path, make_executable=True):
|
||||
"""
|
||||
Append a line to a file unless the line already exists in the file. This method creates the file if
|
||||
it doesn't exist. If make_executable is True, the permissions on the file are set to executable by
|
||||
the owner.
|
||||
"""
|
||||
file_dir = os.path.dirname(file_path)
|
||||
if not os.path.exists(file_dir):
|
||||
try:
|
||||
os.makedirs(file_dir)
|
||||
except Exception as e:
|
||||
log.exception(str(e))
|
||||
return 1
|
||||
if os.path.exists(file_path):
|
||||
try:
|
||||
new_env_file_contents = []
|
||||
env_file_contents = open(file_path).readlines()
|
||||
# Clean out blank lines from the env.sh file.
|
||||
for line in env_file_contents:
|
||||
line = line.rstrip()
|
||||
if line:
|
||||
new_env_file_contents.append(line)
|
||||
env_file_contents = new_env_file_contents
|
||||
except Exception as e:
|
||||
log.exception(str(e))
|
||||
return 1
|
||||
else:
|
||||
env_file_handle = open(file_path, "w")
|
||||
env_file_handle.close()
|
||||
env_file_contents = []
|
||||
if make_executable:
|
||||
# Explicitly set the file's executable bits.
|
||||
try:
|
||||
os.chmod(file_path, os.stat(file_path).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
except Exception as e:
|
||||
log.exception(str(e))
|
||||
return 1
|
||||
# Convert the received text to a list, in order to support adding one or more lines to the file.
|
||||
if isinstance(text, str):
|
||||
text = [text]
|
||||
for line in text:
|
||||
line = line.rstrip()
|
||||
if line and line not in env_file_contents:
|
||||
env_file_contents.append(line)
|
||||
try:
|
||||
open(file_path, "w").write("\n".join(env_file_contents))
|
||||
except Exception as e:
|
||||
log.exception(str(e))
|
||||
return 1
|
||||
return 0
|
||||
|
||||
def handle_action_shell_file_paths(self, action_dict):
|
||||
shell_file_paths = action_dict.get("action_shell_file_paths", [])
|
||||
for shell_file_path in shell_file_paths:
|
||||
self.append_line(action="source", value=shell_file_path)
|
||||
@@ -1,288 +0,0 @@
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
|
||||
# TODO: eliminate the use of fabric here.
|
||||
from fabric import state
|
||||
from fabric.operations import _AttributeString
|
||||
|
||||
from galaxy.tool_shed.galaxy_install.tool_dependencies.recipe import asynchronous_reader
|
||||
from galaxy.tool_shed.util.basic_util import (
|
||||
INSTALLATION_LOG,
|
||||
NO_OUTPUT_TIMEOUT,
|
||||
)
|
||||
from galaxy.tool_shed.util.tool_dependency_util import set_tool_dependency_attributes
|
||||
from galaxy.util import (
|
||||
DATABASE_MAX_STRING_SIZE,
|
||||
DATABASE_MAX_STRING_SIZE_PRETTY,
|
||||
shrink_string_by_size,
|
||||
unicodify,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class InstallEnvironment:
|
||||
"""Object describing the environment built up as part of the process of building and installing a package."""
|
||||
|
||||
def __init__(self, app, tool_shed_repository_install_dir, install_dir):
|
||||
"""
|
||||
The value of the received tool_shed_repository_install_dir is the root installation directory
|
||||
of the repository containing the tool dependency, and the value of the received install_dir is
|
||||
the root installation directory of the tool dependency.
|
||||
"""
|
||||
self.app = app
|
||||
self.env_shell_file_paths = []
|
||||
self.install_dir = install_dir
|
||||
self.tool_shed_repository_install_dir = tool_shed_repository_install_dir
|
||||
self.tmp_work_dir = os.path.abspath(tempfile.mkdtemp(prefix="tmp-toolshed-mtd"))
|
||||
|
||||
def add_env_shell_file_paths(self, paths):
|
||||
for path in paths:
|
||||
self.env_shell_file_paths.append(str(path))
|
||||
|
||||
def build_command(self, command, action_type="shell_command"):
|
||||
"""
|
||||
Build command line for execution from simple command, but
|
||||
configuring environment described by this object.
|
||||
"""
|
||||
env_cmds = self.environment_commands(action_type)
|
||||
return "\n".join(env_cmds + [command])
|
||||
|
||||
def close_file_descriptor(self, fd):
|
||||
"""Attempt to close a file descriptor."""
|
||||
start_timer = time.time()
|
||||
error = ""
|
||||
while True:
|
||||
try:
|
||||
fd.close()
|
||||
break
|
||||
except OSError as e:
|
||||
# Undoubtedly close() was called during a concurrent operation on the same file object.
|
||||
log.debug(f"Error closing file descriptor: {e}")
|
||||
time.sleep(0.5)
|
||||
current_wait_time = time.time() - start_timer
|
||||
if current_wait_time >= 600:
|
||||
error = f"Error closing file descriptor: {e}"
|
||||
break
|
||||
return error
|
||||
|
||||
def enqueue_output(self, stdout, stdout_queue, stderr, stderr_queue):
|
||||
"""
|
||||
This method places streamed stdout and stderr into a threaded IPC queue target. Received data
|
||||
is printed and saved to that thread's queue. The calling thread can then retrieve the data using
|
||||
thread.stdout and thread.stderr.
|
||||
"""
|
||||
stdout_logger = logging.getLogger("install_environment.STDOUT")
|
||||
stderr_logger = logging.getLogger("install_environment.STDERR")
|
||||
for line in iter(stdout.readline, b""):
|
||||
output = unicodify(line).rstrip()
|
||||
stdout_logger.debug(output)
|
||||
stdout_queue.put(output)
|
||||
stdout_queue.put(None)
|
||||
for line in iter(stderr.readline, b""):
|
||||
output = unicodify(line).rstrip()
|
||||
stderr_logger.debug(output)
|
||||
stderr_queue.put(output)
|
||||
stderr_queue.put(None)
|
||||
|
||||
def environment_commands(self, action_type):
|
||||
"""Build a list of commands used to construct the environment described by this object."""
|
||||
cmds = []
|
||||
for env_shell_file_path in self.env_shell_file_paths:
|
||||
if os.path.exists(env_shell_file_path):
|
||||
for env_setting in open(env_shell_file_path):
|
||||
cmds.append(env_setting.strip("\n"))
|
||||
else:
|
||||
log.debug(f"Invalid file {env_shell_file_path} specified, ignoring {action_type} action.")
|
||||
return cmds
|
||||
|
||||
def environment_dict(self, action_type="template_command"):
|
||||
env_vars = dict()
|
||||
for env_shell_file_path in self.env_shell_file_paths:
|
||||
if os.path.exists(env_shell_file_path):
|
||||
for env_setting in open(env_shell_file_path):
|
||||
env_string = env_setting.split(";")[0]
|
||||
env_name, env_path = env_string.split("=")
|
||||
env_vars[env_name] = env_path
|
||||
else:
|
||||
log.debug(f"Invalid file {env_shell_file_path} specified, ignoring template_command action.")
|
||||
return env_vars
|
||||
|
||||
def handle_command(self, tool_dependency, cmd, return_output=False, job_name=""):
|
||||
"""Handle a command and log the results."""
|
||||
command = str(cmd)
|
||||
output = self.handle_complex_command(command, job_name=job_name)
|
||||
self.log_results(cmd, output, os.path.join(self.install_dir, INSTALLATION_LOG))
|
||||
stdout = output.stdout
|
||||
stderr = output.stderr
|
||||
if len(stdout) > DATABASE_MAX_STRING_SIZE:
|
||||
log.warning(
|
||||
f"Length of stdout > {DATABASE_MAX_STRING_SIZE_PRETTY}, so only a portion will be saved in the database."
|
||||
)
|
||||
stdout = shrink_string_by_size(
|
||||
stdout, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True
|
||||
)
|
||||
if len(stderr) > DATABASE_MAX_STRING_SIZE:
|
||||
log.warning(
|
||||
f"Length of stderr > {DATABASE_MAX_STRING_SIZE_PRETTY}, so only a portion will be saved in the database."
|
||||
)
|
||||
stderr = shrink_string_by_size(
|
||||
stderr, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True
|
||||
)
|
||||
if output.return_code not in [0]:
|
||||
status = self.app.install_model.ToolDependency.installation_status.ERROR
|
||||
if stderr:
|
||||
error_message = unicodify(stderr)
|
||||
elif stdout:
|
||||
error_message = unicodify(stdout)
|
||||
else:
|
||||
# We have a problem if there was no stdout and no stderr.
|
||||
error_message = (
|
||||
f"Unknown error occurred executing shell command {cmd}, return_code: {output.return_code}"
|
||||
)
|
||||
set_tool_dependency_attributes(
|
||||
self.app, tool_dependency=tool_dependency, status=status, error_message=error_message
|
||||
)
|
||||
if return_output:
|
||||
return output
|
||||
return output.return_code
|
||||
|
||||
def handle_complex_command(self, command, job_name=""):
|
||||
"""
|
||||
Wrap subprocess.Popen in such a way that the stderr and stdout from running a shell command will
|
||||
be captured and logged in nearly real time. This is similar to fabric.local, but allows us to
|
||||
retain control over the process. This method is named "complex" because it uses queues and
|
||||
threads to execute a command while capturing and displaying the output.
|
||||
"""
|
||||
# We define a "local logger" here such that we can give it a slightly
|
||||
# different name. We use the package name as part of the logger to
|
||||
# allow admins to easily distinguish between which package is currently
|
||||
# being installed.
|
||||
llog_name = __name__
|
||||
if len(job_name) > 0:
|
||||
llog_name += f":{job_name}"
|
||||
llog = logging.getLogger(llog_name)
|
||||
# Print the command we're about to execute, ``set -x`` style.
|
||||
llog.debug(f"+ {command}")
|
||||
# Launch the command as subprocess. A bufsize of 1 means line buffered.
|
||||
process_handle = subprocess.Popen(
|
||||
str(command),
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
bufsize=1,
|
||||
close_fds=False,
|
||||
shell=True,
|
||||
cwd=state.env["lcwd"],
|
||||
)
|
||||
pid = process_handle.pid
|
||||
# Launch the asynchronous readers of the process' stdout and stderr.
|
||||
stdout_queue = queue.Queue()
|
||||
stdout_reader = asynchronous_reader.AsynchronousReader(process_handle.stdout, stdout_queue)
|
||||
stdout_reader.start()
|
||||
stderr_queue = queue.Queue()
|
||||
stderr_reader = asynchronous_reader.AsynchronousReader(process_handle.stderr, stderr_queue)
|
||||
stderr_reader.start()
|
||||
# Place streamed stdout and stderr into a threaded IPC queue target so it can
|
||||
# be printed and stored for later retrieval when generating the INSTALLATION.log.
|
||||
stdio_thread = threading.Thread(
|
||||
target=self.enqueue_output, args=(process_handle.stdout, stdout_queue, process_handle.stderr, stderr_queue)
|
||||
)
|
||||
thread_lock = threading.Lock()
|
||||
thread_lock.acquire()
|
||||
stdio_thread.start()
|
||||
# Check the queues for output until there is nothing more to get.
|
||||
start_timer = time.time()
|
||||
while not stdout_reader.installation_complete() or not stderr_reader.installation_complete():
|
||||
# Show what we received from standard output.
|
||||
while not stdout_queue.empty():
|
||||
try:
|
||||
line = stdout_queue.get()
|
||||
except queue.Empty:
|
||||
line = None
|
||||
break
|
||||
if line:
|
||||
llog.debug(line)
|
||||
start_timer = time.time()
|
||||
else:
|
||||
break
|
||||
# Show what we received from standard error.
|
||||
while not stderr_queue.empty():
|
||||
try:
|
||||
line = stderr_queue.get()
|
||||
except queue.Empty:
|
||||
line = None
|
||||
break
|
||||
if line:
|
||||
llog.debug(line)
|
||||
start_timer = time.time()
|
||||
else:
|
||||
stderr_queue.task_done()
|
||||
break
|
||||
# Sleep a bit before asking the readers again.
|
||||
time.sleep(0.1)
|
||||
current_wait_time = time.time() - start_timer
|
||||
if stdout_queue.empty() and stderr_queue.empty() and current_wait_time > NO_OUTPUT_TIMEOUT:
|
||||
err_msg = (
|
||||
"\nShutting down process id %s because it generated no output for the defined timeout period of %.1f seconds.\n"
|
||||
% (pid, NO_OUTPUT_TIMEOUT)
|
||||
)
|
||||
stderr_reader.lines.append(err_msg)
|
||||
process_handle.kill()
|
||||
break
|
||||
thread_lock.release()
|
||||
# Wait until each of the threads we've started terminate. The following calls will block each thread
|
||||
# until it terminates either normally, through an unhandled exception, or until the timeout occurs.
|
||||
stdio_thread.join(NO_OUTPUT_TIMEOUT)
|
||||
stdout_reader.join(NO_OUTPUT_TIMEOUT)
|
||||
stderr_reader.join(NO_OUTPUT_TIMEOUT)
|
||||
# Close subprocess' file descriptors.
|
||||
self.close_file_descriptor(process_handle.stdout)
|
||||
self.close_file_descriptor(process_handle.stderr)
|
||||
stdout = "\n".join(stdout_reader.lines)
|
||||
stderr = "\n".join(stderr_reader.lines)
|
||||
# Handle error condition (deal with stdout being None, too)
|
||||
output = _AttributeString(stdout.strip() if stdout else "")
|
||||
errors = _AttributeString(stderr.strip() if stderr else "")
|
||||
# Make sure the process has finished.
|
||||
process_handle.poll()
|
||||
output.return_code = process_handle.returncode
|
||||
output.stderr = errors
|
||||
return output
|
||||
|
||||
def log_results(self, command, fabric_AttributeString, file_path):
|
||||
"""Write attributes of fabric.operations._AttributeString to a specified log file."""
|
||||
mode = "a" if os.path.exists(file_path) else "w"
|
||||
with open(file_path, mode) as logfile:
|
||||
logfile.write("\n#############################################\n")
|
||||
logfile.write(command)
|
||||
logfile.write("\nSTDOUT\n")
|
||||
logfile.write(fabric_AttributeString.stdout)
|
||||
logfile.write("\n#############################################\n")
|
||||
logfile.write("\n#############################################\n")
|
||||
logfile.write(command)
|
||||
logfile.write("\nSTDERR\n")
|
||||
logfile.write(fabric_AttributeString.stderr)
|
||||
logfile.write("\n#############################################\n")
|
||||
|
||||
@contextmanager
|
||||
def use_tmp_dir(self):
|
||||
work_dir = self.tmp_work_dir
|
||||
yield work_dir
|
||||
if os.path.exists(work_dir):
|
||||
try:
|
||||
shutil.rmtree(work_dir)
|
||||
except Exception as e:
|
||||
log.exception(str(e))
|
||||
|
||||
def __setup_environment(self):
|
||||
return "&&".join(f". {_}" for _ in self.__valid_env_shell_file_paths())
|
||||
|
||||
def __valid_env_shell_file_paths(self):
|
||||
return [_ for _ in self.env_shell_file_paths if os.path.exists(_)]
|
||||
@@ -1,136 +0,0 @@
|
||||
import logging
|
||||
|
||||
from galaxy.tool_shed.galaxy_install.tool_dependencies.recipe import (
|
||||
step_handler,
|
||||
tag_handler,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StepManager:
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
self.step_handlers_by_type = self.load_step_handlers()
|
||||
|
||||
def get_step_handler_by_type(self, type):
|
||||
return self.step_handlers_by_type.get(type, None)
|
||||
|
||||
def execute_step(
|
||||
self,
|
||||
tool_dependency,
|
||||
package_name,
|
||||
actions,
|
||||
action_type,
|
||||
action_dict,
|
||||
filtered_actions,
|
||||
env_file_builder,
|
||||
install_environment,
|
||||
work_dir,
|
||||
current_dir=None,
|
||||
initial_download=False,
|
||||
):
|
||||
if actions:
|
||||
step_handler = self.get_step_handler_by_type(action_type)
|
||||
tool_dependency, filtered_actions, dir = step_handler.execute_step(
|
||||
tool_dependency=tool_dependency,
|
||||
package_name=package_name,
|
||||
actions=actions,
|
||||
action_dict=action_dict,
|
||||
filtered_actions=filtered_actions,
|
||||
env_file_builder=env_file_builder,
|
||||
install_environment=install_environment,
|
||||
work_dir=work_dir,
|
||||
current_dir=current_dir,
|
||||
initial_download=initial_download,
|
||||
)
|
||||
else:
|
||||
dir = None
|
||||
return tool_dependency, filtered_actions, dir
|
||||
|
||||
def load_step_handlers(self):
|
||||
step_handlers_by_type = dict(
|
||||
assert_directory_executable=step_handler.AssertDirectoryExecutable(self.app),
|
||||
assert_directory_exists=step_handler.AssertDirectoryExists(self.app),
|
||||
assert_file_executable=step_handler.AssertFileExecutable(self.app),
|
||||
assert_file_exists=step_handler.AssertFileExists(self.app),
|
||||
autoconf=step_handler.Autoconf(self.app),
|
||||
change_directory=step_handler.ChangeDirectory(self.app),
|
||||
chmod=step_handler.Chmod(self.app),
|
||||
download_binary=step_handler.DownloadBinary(self.app),
|
||||
download_by_url=step_handler.DownloadByUrl(self.app),
|
||||
download_file=step_handler.DownloadFile(self.app),
|
||||
make_directory=step_handler.MakeDirectory(self.app),
|
||||
make_install=step_handler.MakeInstall(self.app),
|
||||
move_directory_files=step_handler.MoveDirectoryFiles(self.app),
|
||||
move_file=step_handler.MoveFile(self.app),
|
||||
regex_replace=step_handler.RegexReplace(self.app),
|
||||
set_environment=step_handler.SetEnvironment(self.app),
|
||||
set_environment_for_install=step_handler.SetEnvironmentForInstall(self.app),
|
||||
setup_perl_environment=step_handler.SetupPerlEnvironment(self.app),
|
||||
setup_r_environment=step_handler.SetupREnvironment(self.app),
|
||||
setup_ruby_environment=step_handler.SetupRubyEnvironment(self.app),
|
||||
setup_python_environment=step_handler.SetupPythonEnvironment(self.app),
|
||||
setup_virtualenv=step_handler.SetupVirtualEnv(self.app),
|
||||
shell_command=step_handler.ShellCommand(self.app),
|
||||
template_command=step_handler.TemplateCommand(self.app),
|
||||
)
|
||||
return step_handlers_by_type
|
||||
|
||||
def prepare_step(
|
||||
self, tool_dependency, action_type, action_elem, action_dict, install_environment, is_binary_download
|
||||
):
|
||||
"""
|
||||
Prepare the recipe step for later execution. This generally alters the received action_dict
|
||||
with new information needed during this step's execution.
|
||||
"""
|
||||
if action_elem is not None:
|
||||
step_handler = self.get_step_handler_by_type(action_type)
|
||||
action_dict = step_handler.prepare_step(
|
||||
tool_dependency=tool_dependency,
|
||||
action_elem=action_elem,
|
||||
action_dict=action_dict,
|
||||
install_environment=install_environment,
|
||||
is_binary_download=is_binary_download,
|
||||
)
|
||||
return action_dict
|
||||
|
||||
|
||||
class TagManager:
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
self.tag_handlers = self.load_tag_handlers()
|
||||
|
||||
def get_tag_handler_by_tag(self, tag):
|
||||
return self.tag_handlers.get(tag, None)
|
||||
|
||||
def process_tag_set(
|
||||
self,
|
||||
tool_shed_repository,
|
||||
tool_dependency,
|
||||
package_elem,
|
||||
package_name,
|
||||
package_version,
|
||||
tool_dependency_db_records=None,
|
||||
):
|
||||
tag_handler = self.get_tag_handler_by_tag(package_elem.tag)
|
||||
tool_dependency, proceed_with_install, action_elem_tuples = tag_handler.process_tag_set(
|
||||
tool_shed_repository,
|
||||
tool_dependency,
|
||||
package_elem,
|
||||
package_name,
|
||||
package_version,
|
||||
tool_dependency_db_records=tool_dependency_db_records,
|
||||
)
|
||||
return tool_dependency, proceed_with_install, action_elem_tuples
|
||||
|
||||
def load_tag_handlers(self):
|
||||
tag_handlers = dict(
|
||||
environment_variable=tag_handler.SetEnvironment(self.app),
|
||||
install=tag_handler.Install(self.app),
|
||||
package=tag_handler.Package(self.app),
|
||||
readme=tag_handler.ReadMe(self.app),
|
||||
repository=tag_handler.Repository(self.app),
|
||||
set_environment=tag_handler.SetEnvironment(self.app),
|
||||
)
|
||||
return tag_handlers
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,757 +0,0 @@
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
from galaxy.tool_shed.galaxy_install.tool_dependencies.env_manager import EnvManager
|
||||
from galaxy.tool_shed.galaxy_install.tool_dependencies.recipe.env_file_builder import EnvFileBuilder
|
||||
from galaxy.tool_shed.galaxy_install.tool_dependencies.recipe.install_environment import InstallEnvironment
|
||||
from galaxy.tool_shed.util import tool_dependency_util
|
||||
from galaxy.tool_shed.util.basic_util import (
|
||||
INSTALLATION_LOG,
|
||||
remove_dir,
|
||||
)
|
||||
from galaxy.tool_shed.util.metadata_util import get_updated_changeset_revisions_from_tool_shed
|
||||
from galaxy.tool_shed.util.repository_util import (
|
||||
get_absolute_path_to_file_in_repository,
|
||||
get_repository_for_dependency_relationship,
|
||||
)
|
||||
from galaxy.tool_util.deps.resolvers import NullDependency
|
||||
from galaxy.util import (
|
||||
listify,
|
||||
url_get,
|
||||
)
|
||||
from galaxy.util.tool_shed.common_util import (
|
||||
get_tool_shed_url_from_tool_shed_registry,
|
||||
remove_protocol_from_tool_shed_url,
|
||||
)
|
||||
from galaxy.util.tool_shed.xml_util import parse_xml
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RecipeTag:
|
||||
"""Abstract class that defines a standard format for handling recipe tags when installing packages."""
|
||||
|
||||
def process_tag_set(
|
||||
self,
|
||||
tool_shed_repository,
|
||||
tool_dependency,
|
||||
package_elem,
|
||||
package_name,
|
||||
package_version,
|
||||
tool_dependency_db_records=None,
|
||||
):
|
||||
raise Exception("Unimplemented Method")
|
||||
|
||||
|
||||
class SyncDatabase:
|
||||
def sync_database_with_file_system(
|
||||
self,
|
||||
app,
|
||||
tool_shed_repository,
|
||||
tool_dependency_name,
|
||||
tool_dependency_version,
|
||||
tool_dependency_install_dir,
|
||||
tool_dependency_type="package",
|
||||
):
|
||||
"""
|
||||
The installation directory defined by the received tool_dependency_install_dir exists, so check for
|
||||
the presence of INSTALLATION_LOG. If the files exists, we'll assume the tool dependency is installed,
|
||||
but not necessarily successfully (it could be in an error state on disk. However, we can justifiably
|
||||
assume here that no matter the state, an associated database record will exist.
|
||||
"""
|
||||
# This method should be reached very rarely. It implies that either the Galaxy environment
|
||||
# became corrupted (i.e., the database records for installed tool dependencies is not synchronized
|
||||
# with tool dependencies on disk) or the Tool Shed's install and test framework is running. The Tool
|
||||
# Shed's install and test framework installs repositories in 2 stages, those of type tool_dependency_definition
|
||||
# followed by those containing valid tools and tool functional test components.
|
||||
log.debug("Synchronizing the database with the file system...")
|
||||
try:
|
||||
log.debug(
|
||||
"The value of app.config.running_functional_tests is: %s" % str(app.config.running_functional_tests)
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
sa_session = app.install_model.context
|
||||
can_install_tool_dependency = False
|
||||
tool_dependency = tool_dependency_util.get_tool_dependency_by_name_version_type_repository(
|
||||
app, tool_shed_repository, tool_dependency_name, tool_dependency_version, tool_dependency_type
|
||||
)
|
||||
if tool_dependency.status == app.install_model.ToolDependency.installation_status.INSTALLING:
|
||||
# The tool dependency is in an Installing state, so we don't want to do anything to it. If the tool
|
||||
# dependency is being installed by someone else, we don't want to interfere with that. This assumes
|
||||
# the installation by "someone else" is not hung in an Installing state, which is a weakness if that
|
||||
# "someone else" never repaired it.
|
||||
log.debug(
|
||||
"Skipping installation of tool dependency %s version %s because it has a status of %s"
|
||||
% (str(tool_dependency.name), str(tool_dependency.version), str(tool_dependency.status))
|
||||
)
|
||||
else:
|
||||
# We have a pre-existing installation directory on the file system, but our associated database record is
|
||||
# in a state that allowed us to arrive here. At this point, we'll inspect the installation directory to
|
||||
# see if we have a "proper installation" and if so, synchronize the database record rather than reinstalling
|
||||
# the dependency if we're "running_functional_tests". If we're not "running_functional_tests, we'll set
|
||||
# the tool dependency's installation status to ERROR.
|
||||
tool_dependency_installation_directory_contents = os.listdir(tool_dependency_install_dir)
|
||||
if INSTALLATION_LOG in tool_dependency_installation_directory_contents:
|
||||
# Since this tool dependency's installation directory contains an installation log, we consider it to be
|
||||
# installed. In some cases the record may be missing from the database due to some activity outside of
|
||||
# the control of the Tool Shed. Since a new record was created for it and we don't know the state of the
|
||||
# files on disk, we will set it to an error state (unless we are running Tool Shed functional tests - see
|
||||
# below).
|
||||
log.debug(
|
||||
"Skipping installation of tool dependency %s version %s because it is installed in %s"
|
||||
% (str(tool_dependency.name), str(tool_dependency.version), str(tool_dependency_install_dir))
|
||||
)
|
||||
if app.config.running_functional_tests:
|
||||
# If we are running functional tests, the state will be set to Installed because previously compiled
|
||||
# tool dependencies are not deleted by default, from the "install and test" framework..
|
||||
tool_dependency.status = app.install_model.ToolDependency.installation_status.INSTALLED
|
||||
else:
|
||||
error_message = "The installation directory for this tool dependency had contents but the database had no record. "
|
||||
error_message += (
|
||||
"The installation log may show this tool dependency to be correctly installed, but due to the "
|
||||
)
|
||||
error_message += "missing database record it is now being set to Error."
|
||||
tool_dependency.status = app.install_model.ToolDependency.installation_status.ERROR
|
||||
tool_dependency.error_message = error_message
|
||||
else:
|
||||
error_message = (
|
||||
"\nInstallation path %s for tool dependency %s version %s exists, but the expected file %s"
|
||||
% (
|
||||
str(tool_dependency_install_dir),
|
||||
str(tool_dependency_name),
|
||||
str(tool_dependency_version),
|
||||
str(INSTALLATION_LOG),
|
||||
)
|
||||
)
|
||||
error_message += " is missing. This indicates an installation error so the tool dependency is being"
|
||||
error_message += " prepared for re-installation."
|
||||
log.error(error_message)
|
||||
tool_dependency.status = app.install_model.ToolDependency.installation_status.NEVER_INSTALLED
|
||||
remove_dir(tool_dependency_install_dir)
|
||||
can_install_tool_dependency = True
|
||||
sa_session.add(tool_dependency)
|
||||
sa_session.flush()
|
||||
try:
|
||||
log.debug(
|
||||
"Returning from sync_database_with_file_system with tool_dependency %s, can_install_tool_dependency %s."
|
||||
% (str(tool_dependency.name), str(can_install_tool_dependency))
|
||||
)
|
||||
except Exception as e:
|
||||
log.debug(str(e))
|
||||
return tool_dependency, can_install_tool_dependency
|
||||
|
||||
|
||||
class Install(RecipeTag, SyncDatabase):
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
self.tag = "install"
|
||||
|
||||
def process_tag_set(
|
||||
self,
|
||||
tool_shed_repository,
|
||||
tool_dependency,
|
||||
package_elem,
|
||||
package_name,
|
||||
package_version,
|
||||
tool_dependency_db_records=None,
|
||||
):
|
||||
# <install version="1.0">
|
||||
# Get the installation directory for tool dependencies that will be installed for the received tool_shed_repository.
|
||||
actions_elem_tuples = []
|
||||
proceed_with_install = False
|
||||
install_dir = tool_dependency_util.get_tool_dependency_install_dir(
|
||||
app=self.app,
|
||||
repository_name=tool_shed_repository.name,
|
||||
repository_owner=tool_shed_repository.owner,
|
||||
repository_changeset_revision=tool_shed_repository.installed_changeset_revision,
|
||||
tool_dependency_type="package",
|
||||
tool_dependency_name=package_name,
|
||||
tool_dependency_version=package_version,
|
||||
)
|
||||
if os.path.exists(install_dir):
|
||||
# Notice that we'll throw away the following tool_dependency if it can be installed.
|
||||
tool_dependency, proceed_with_install = self.sync_database_with_file_system(
|
||||
self.app,
|
||||
tool_shed_repository,
|
||||
package_name,
|
||||
package_version,
|
||||
install_dir,
|
||||
tool_dependency_type="package",
|
||||
)
|
||||
if not proceed_with_install:
|
||||
log.debug(
|
||||
"Tool dependency %s version %s cannot be installed (it was probably previously installed), so returning it."
|
||||
% (str(tool_dependency.name), str(tool_dependency.version))
|
||||
)
|
||||
return tool_dependency, proceed_with_install, actions_elem_tuples
|
||||
else:
|
||||
proceed_with_install = True
|
||||
if proceed_with_install:
|
||||
package_install_version = package_elem.get("version", "1.0")
|
||||
status = self.app.install_model.ToolDependency.installation_status.INSTALLING
|
||||
tool_dependency = tool_dependency_util.create_or_update_tool_dependency(
|
||||
app=self.app,
|
||||
tool_shed_repository=tool_shed_repository,
|
||||
name=package_name,
|
||||
version=package_version,
|
||||
type="package",
|
||||
status=status,
|
||||
set_status=True,
|
||||
)
|
||||
# Get the information about the current platform in case the tool dependency definition includes tag sets
|
||||
# for installing compiled binaries.
|
||||
platform_info_dict = tool_dependency_util.get_platform_info_dict()
|
||||
if package_install_version == "1.0":
|
||||
# Handle tool dependency installation using a fabric method included in the Galaxy framework.
|
||||
actions_elem_tuples = tool_dependency_util.parse_package_elem(
|
||||
package_elem, platform_info_dict=platform_info_dict, include_after_install_actions=True
|
||||
)
|
||||
if not actions_elem_tuples:
|
||||
proceed_with_install = False
|
||||
error_message = (
|
||||
f"Version {package_version} of the {package_name} package cannot be installed because "
|
||||
)
|
||||
error_message += "the recipe for installing the package is missing either an <actions> tag set or an <actions_group> "
|
||||
error_message += "tag set."
|
||||
# Since there was an installation error, update the tool dependency status to Error.
|
||||
# The remove_installation_path option must be left False here.
|
||||
tool_dependency = tool_dependency_util.set_tool_dependency_attributes(
|
||||
self.app,
|
||||
tool_dependency=tool_dependency,
|
||||
status=self.app.install_model.ToolDependency.installation_status.ERROR,
|
||||
error_message=error_message,
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
'Only install version 1.0 is currently supported (i.e., change your tag to be <install version="1.0">).'
|
||||
)
|
||||
return tool_dependency, proceed_with_install, actions_elem_tuples
|
||||
|
||||
|
||||
class Package(RecipeTag):
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
self.tag = "package"
|
||||
|
||||
def process_tag_set(
|
||||
self,
|
||||
tool_shed_repository,
|
||||
tool_dependency,
|
||||
package_elem,
|
||||
package_name,
|
||||
package_version,
|
||||
tool_dependency_db_records=None,
|
||||
):
|
||||
action_elem_tuples = []
|
||||
proceed_with_install = False
|
||||
# Only install the tool_dependency if it is not already installed and it is associated with a database
|
||||
# record in the received tool_dependencies.
|
||||
if package_name and package_version:
|
||||
dependencies_ignored = not self.app.toolbox.dependency_manager.uses_tool_shed_dependencies()
|
||||
if dependencies_ignored:
|
||||
log.debug(
|
||||
"Skipping installation of tool dependency package %s because tool shed dependency resolver not enabled."
|
||||
% str(package_name)
|
||||
)
|
||||
# Tool dependency resolves have been configured and they do not include the tool shed. Do not install package.
|
||||
dep = self.app.toolbox.dependency_manager.find_dep(package_name, package_version, type="package")
|
||||
if not isinstance(dep, NullDependency):
|
||||
# TODO: Do something here such as marking it installed or configured externally.
|
||||
pass
|
||||
tool_dependency = tool_dependency_util.set_tool_dependency_attributes(
|
||||
self.app,
|
||||
tool_dependency=tool_dependency,
|
||||
status=self.app.install_model.ToolDependency.installation_status.ERROR,
|
||||
)
|
||||
else:
|
||||
proceed_with_install = True
|
||||
return tool_dependency, proceed_with_install, action_elem_tuples
|
||||
|
||||
|
||||
class ReadMe(RecipeTag):
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
self.tag = "readme"
|
||||
|
||||
def process_tag_set(
|
||||
self,
|
||||
tool_shed_repository,
|
||||
tool_dependency,
|
||||
package_elem,
|
||||
package_name,
|
||||
package_version,
|
||||
tool_dependency_db_records=None,
|
||||
):
|
||||
# Nothing to be done.
|
||||
action_elem_tuples = []
|
||||
proceed_with_install = False
|
||||
return tool_dependency, proceed_with_install, action_elem_tuples
|
||||
|
||||
|
||||
class Repository(RecipeTag, SyncDatabase):
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
self.tag = "repository"
|
||||
|
||||
def create_temporary_tool_dependencies_config(self, tool_shed_url, name, owner, changeset_revision):
|
||||
"""Make a call to the tool shed to get the required repository's tool_dependencies.xml file."""
|
||||
tool_shed_url = get_tool_shed_url_from_tool_shed_registry(self.app, tool_shed_url)
|
||||
if tool_shed_url is None or name is None or owner is None or changeset_revision is None:
|
||||
message = (
|
||||
"Unable to retrieve required tool_dependencies.xml file from the Tool Shed because one or more of the "
|
||||
)
|
||||
message += (
|
||||
"following required parameters is None: tool_shed_url: %s, name: %s, owner: %s, changeset_revision: %s "
|
||||
% (str(tool_shed_url), str(name), str(owner), str(changeset_revision))
|
||||
)
|
||||
raise Exception(message)
|
||||
params = dict(name=name, owner=owner, changeset_revision=changeset_revision)
|
||||
pathspec = ["repository", "get_tool_dependencies_config_contents"]
|
||||
text = url_get(
|
||||
tool_shed_url, auth=self.app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params
|
||||
)
|
||||
if text:
|
||||
# Write the contents to a temporary file on disk so it can be reloaded and parsed.
|
||||
fh = tempfile.NamedTemporaryFile("w", prefix="tmp-toolshed-cttdc")
|
||||
tmp_filename = fh.name
|
||||
fh.close()
|
||||
fh = open(tmp_filename, "w")
|
||||
fh.write(text)
|
||||
fh.close()
|
||||
return tmp_filename
|
||||
else:
|
||||
message = "Unable to retrieve required tool_dependencies.xml file from the Tool Shed for revision "
|
||||
message += f"{changeset_revision} of installed repository {name} owned by {owner}."
|
||||
raise Exception(message)
|
||||
|
||||
def create_tool_dependency_with_initialized_env_sh_file(
|
||||
self,
|
||||
dependent_install_dir,
|
||||
tool_shed_repository,
|
||||
required_repository,
|
||||
package_name,
|
||||
package_version,
|
||||
tool_dependencies_config,
|
||||
):
|
||||
"""
|
||||
Create or get a tool_dependency record that is defined by the received package_name and package_version.
|
||||
An env.sh file will be created for the tool_dependency in the received dependent_install_dir.
|
||||
"""
|
||||
# The received required_repository refers to a tool_shed_repository record that is defined as a complex
|
||||
# repository dependency for this tool_dependency. The required_repository may or may not be currently
|
||||
# installed (it doesn't matter). If it is installed, it is associated with a tool_dependency that has
|
||||
# an env.sh file that this new tool_dependency must be able to locate and "source". If it is not installed,
|
||||
# we can still determine where that env.sh file will be, so we'll initialize this new tool_dependency's env.sh
|
||||
# file in either case. If the required repository ends up with an installation error, this new tool
|
||||
# dependency will still be fine because its containing repository will be defined as missing dependencies.
|
||||
tool_dependencies = []
|
||||
if not os.path.exists(dependent_install_dir):
|
||||
os.makedirs(dependent_install_dir)
|
||||
required_tool_dependency_env_file_path = None
|
||||
if tool_dependencies_config:
|
||||
required_td_tree, error_message = parse_xml(tool_dependencies_config)
|
||||
if required_td_tree:
|
||||
required_td_root = required_td_tree.getroot()
|
||||
for required_td_elem in required_td_root:
|
||||
# Find the appropriate package name and version.
|
||||
if required_td_elem.tag == "package":
|
||||
# <package name="bwa" version="0.5.9">
|
||||
required_td_package_name = required_td_elem.get("name", None)
|
||||
required_td_package_version = required_td_elem.get("version", None)
|
||||
# Check the database to see if we have a record for the required tool dependency (we may not which is ok). If we
|
||||
# find a record, we need to see if it is in an error state and if so handle it appropriately.
|
||||
required_tool_dependency = (
|
||||
tool_dependency_util.get_tool_dependency_by_name_version_type_repository(
|
||||
self.app,
|
||||
required_repository,
|
||||
required_td_package_name,
|
||||
required_td_package_version,
|
||||
"package",
|
||||
)
|
||||
)
|
||||
if required_td_package_name == package_name and required_td_package_version == package_version:
|
||||
# Get or create a database tool_dependency record with which the installed package on disk will be associated.
|
||||
tool_dependency = tool_dependency_util.create_or_update_tool_dependency(
|
||||
app=self.app,
|
||||
tool_shed_repository=tool_shed_repository,
|
||||
name=package_name,
|
||||
version=package_version,
|
||||
type="package",
|
||||
status=self.app.install_model.ToolDependency.installation_status.NEVER_INSTALLED,
|
||||
set_status=True,
|
||||
)
|
||||
# Create an env.sh file for the tool_dependency whose first line will source the env.sh file located in
|
||||
# the path defined by required_tool_dependency_env_file_path. It doesn't matter if the required env.sh
|
||||
# file currently exists..
|
||||
required_tool_dependency_env_file_path = self.get_required_repository_package_env_sh_path(
|
||||
package_name, package_version, required_repository
|
||||
)
|
||||
env_file_builder = EnvFileBuilder(tool_dependency.installation_directory(self.app))
|
||||
env_file_builder.append_line(action="source", value=required_tool_dependency_env_file_path)
|
||||
return_code = env_file_builder.return_code
|
||||
if return_code:
|
||||
error_message = (
|
||||
f"Error defining env.sh file for package {package_name}, return_code: {return_code}"
|
||||
)
|
||||
tool_dependency = tool_dependency_util.set_tool_dependency_attributes(
|
||||
self.app,
|
||||
tool_dependency=tool_dependency,
|
||||
status=self.app.install_model.ToolDependency.installation_status.ERROR,
|
||||
error_message=error_message,
|
||||
)
|
||||
elif required_tool_dependency is not None and required_tool_dependency.in_error_state:
|
||||
error_message = (
|
||||
"This tool dependency's required tool dependency %s version %s has status %s."
|
||||
% (
|
||||
str(required_tool_dependency.name),
|
||||
str(required_tool_dependency.version),
|
||||
str(required_tool_dependency.status),
|
||||
)
|
||||
)
|
||||
tool_dependency = tool_dependency_util.set_tool_dependency_attributes(
|
||||
self.app,
|
||||
tool_dependency=tool_dependency,
|
||||
status=self.app.install_model.ToolDependency.installation_status.ERROR,
|
||||
error_message=error_message,
|
||||
)
|
||||
else:
|
||||
tool_dependency = tool_dependency_util.set_tool_dependency_attributes(
|
||||
self.app,
|
||||
tool_dependency=tool_dependency,
|
||||
status=self.app.install_model.ToolDependency.installation_status.INSTALLED,
|
||||
)
|
||||
tool_dependencies.append(tool_dependency)
|
||||
return tool_dependencies
|
||||
|
||||
def get_required_repository_package_env_sh_path(self, package_name, package_version, required_repository):
|
||||
"""Return path to env.sh file in required repository if the required repository has been installed."""
|
||||
env_sh_file_dir = tool_dependency_util.get_tool_dependency_install_dir(
|
||||
app=self.app,
|
||||
repository_name=required_repository.name,
|
||||
repository_owner=required_repository.owner,
|
||||
repository_changeset_revision=required_repository.installed_changeset_revision,
|
||||
tool_dependency_type="package",
|
||||
tool_dependency_name=package_name,
|
||||
tool_dependency_version=package_version,
|
||||
)
|
||||
env_sh_file_path = os.path.join(env_sh_file_dir, "env.sh")
|
||||
return env_sh_file_path
|
||||
|
||||
def handle_complex_repository_dependency_for_package(
|
||||
self, elem, package_name, package_version, tool_shed_repository
|
||||
):
|
||||
"""
|
||||
Inspect the repository defined by a complex repository dependency definition and take certain steps to
|
||||
enable installation of the received package name and version to proceed. The received elem is the
|
||||
<repository> tag set which defines the complex repository dependency. The received tool_shed_repository
|
||||
is the installed tool shed repository for which the tool dependency defined by the received package_name
|
||||
and package_version is being installed.
|
||||
"""
|
||||
handled_tool_dependencies = []
|
||||
tool_shed_url = elem.attrib["toolshed"]
|
||||
required_repository_name = elem.attrib["name"]
|
||||
required_repository_owner = elem.attrib["owner"]
|
||||
default_required_repository_changeset_revision = elem.attrib["changeset_revision"]
|
||||
required_repository = get_repository_for_dependency_relationship(
|
||||
self.app,
|
||||
tool_shed_url,
|
||||
required_repository_name,
|
||||
required_repository_owner,
|
||||
default_required_repository_changeset_revision,
|
||||
)
|
||||
tool_shed = remove_protocol_from_tool_shed_url(tool_shed_url)
|
||||
tmp_filename = None
|
||||
if required_repository:
|
||||
required_repository_changeset_revision = required_repository.installed_changeset_revision
|
||||
# Define the installation directory for the required tool dependency package in the required repository.
|
||||
required_repository_package_install_dir = tool_dependency_util.get_tool_dependency_install_dir(
|
||||
app=self.app,
|
||||
repository_name=required_repository_name,
|
||||
repository_owner=required_repository_owner,
|
||||
repository_changeset_revision=required_repository_changeset_revision,
|
||||
tool_dependency_type="package",
|
||||
tool_dependency_name=package_name,
|
||||
tool_dependency_version=package_version,
|
||||
)
|
||||
# Define this dependent repository's tool dependency installation directory that will contain
|
||||
# the env.sh file with a path to the required repository's installed tool dependency package.
|
||||
dependent_install_dir = tool_dependency_util.get_tool_dependency_install_dir(
|
||||
app=self.app,
|
||||
repository_name=tool_shed_repository.name,
|
||||
repository_owner=tool_shed_repository.owner,
|
||||
repository_changeset_revision=tool_shed_repository.installed_changeset_revision,
|
||||
tool_dependency_type="package",
|
||||
tool_dependency_name=package_name,
|
||||
tool_dependency_version=package_version,
|
||||
)
|
||||
if os.path.exists(dependent_install_dir):
|
||||
# Notice that we'll throw away the following tool_dependency if it can be installed.
|
||||
tool_dependency, can_install_tool_dependency = self.sync_database_with_file_system(
|
||||
self.app,
|
||||
tool_shed_repository,
|
||||
package_name,
|
||||
package_version,
|
||||
dependent_install_dir,
|
||||
tool_dependency_type="package",
|
||||
)
|
||||
if not can_install_tool_dependency:
|
||||
log.debug(
|
||||
"Tool dependency %s version %s cannot be installed (it was probably previously installed), "
|
||||
"so appending it to the list of handled tool dependencies.",
|
||||
str(tool_dependency.name),
|
||||
str(tool_dependency.version),
|
||||
)
|
||||
handled_tool_dependencies.append(tool_dependency)
|
||||
else:
|
||||
can_install_tool_dependency = True
|
||||
if can_install_tool_dependency:
|
||||
# Set this dependent repository's tool dependency env.sh file with a path to the required repository's
|
||||
# installed tool dependency package. We can get everything we need from the discovered installed
|
||||
# required_repository.
|
||||
if required_repository.is_deactivated_or_installed:
|
||||
if not os.path.exists(required_repository_package_install_dir):
|
||||
log.error(
|
||||
f"Missing required tool dependency directory {required_repository_package_install_dir}"
|
||||
)
|
||||
repo_files_dir = required_repository.repo_files_directory(self.app)
|
||||
if not repo_files_dir:
|
||||
message = (
|
||||
"Unable to locate the repository directory for revision %s of installed repository %s owned by %s."
|
||||
% (
|
||||
str(required_repository.changeset_revision),
|
||||
str(required_repository.name),
|
||||
str(required_repository.owner),
|
||||
)
|
||||
)
|
||||
raise Exception(message)
|
||||
tool_dependencies_config = get_absolute_path_to_file_in_repository(
|
||||
repo_files_dir, "tool_dependencies.xml"
|
||||
)
|
||||
if tool_dependencies_config:
|
||||
config_to_use = tool_dependencies_config
|
||||
else:
|
||||
message = (
|
||||
"Unable to locate required tool_dependencies.xml file for revision %s of installed repository %s owned by %s."
|
||||
% (
|
||||
str(required_repository.changeset_revision),
|
||||
str(required_repository.name),
|
||||
str(required_repository.owner),
|
||||
)
|
||||
)
|
||||
raise Exception(message)
|
||||
else:
|
||||
# Make a call to the tool shed to get the changeset revision to which the current value of required_repository_changeset_revision
|
||||
# should be updated if it's not current.
|
||||
text = get_updated_changeset_revisions_from_tool_shed(
|
||||
app=self.app,
|
||||
tool_shed_url=tool_shed,
|
||||
name=required_repository_name,
|
||||
owner=required_repository_owner,
|
||||
changeset_revision=required_repository_changeset_revision,
|
||||
)
|
||||
if text:
|
||||
updated_changeset_revisions = listify(text)
|
||||
# The list of changeset revisions is in reverse order, so the newest will be first.
|
||||
required_repository_changeset_revision = updated_changeset_revisions[0]
|
||||
# Make a call to the tool shed to get the required repository's tool_dependencies.xml file.
|
||||
tmp_filename = self.create_temporary_tool_dependencies_config(
|
||||
tool_shed,
|
||||
required_repository_name,
|
||||
required_repository_owner,
|
||||
required_repository_changeset_revision,
|
||||
)
|
||||
config_to_use = tmp_filename
|
||||
handled_tool_dependencies = self.create_tool_dependency_with_initialized_env_sh_file(
|
||||
dependent_install_dir=dependent_install_dir,
|
||||
tool_shed_repository=tool_shed_repository,
|
||||
required_repository=required_repository,
|
||||
package_name=package_name,
|
||||
package_version=package_version,
|
||||
tool_dependencies_config=config_to_use,
|
||||
)
|
||||
self.remove_file(tmp_filename)
|
||||
else:
|
||||
message = f"Unable to locate required tool shed repository named {required_repository_name} owned by {required_repository_owner} with revision {default_required_repository_changeset_revision}."
|
||||
raise Exception(message)
|
||||
return handled_tool_dependencies
|
||||
|
||||
def process_tag_set(
|
||||
self,
|
||||
tool_shed_repository,
|
||||
tool_dependency,
|
||||
package_elem,
|
||||
package_name,
|
||||
package_version,
|
||||
tool_dependency_db_records=None,
|
||||
):
|
||||
# We have a complex repository dependency definition.
|
||||
action_elem_tuples = []
|
||||
proceed_with_install = False
|
||||
rd_tool_dependencies = self.handle_complex_repository_dependency_for_package(
|
||||
package_elem, package_name, package_version, tool_shed_repository
|
||||
)
|
||||
for rd_tool_dependency in rd_tool_dependencies:
|
||||
if rd_tool_dependency.status == self.app.install_model.ToolDependency.installation_status.ERROR:
|
||||
# We'll log the error here, but continue installing packages since some may not require this dependency.
|
||||
log.error(
|
||||
f"Error installing tool dependency for required repository: {rd_tool_dependency.error_message}"
|
||||
)
|
||||
return tool_dependency, proceed_with_install, action_elem_tuples
|
||||
|
||||
def remove_file(self, file_name):
|
||||
"""Attempt to remove a file from disk."""
|
||||
if file_name:
|
||||
if os.path.exists(file_name):
|
||||
try:
|
||||
os.remove(file_name)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
class SetEnvironment(RecipeTag):
|
||||
def __init__(self, app):
|
||||
self.app = app
|
||||
self.tag = "set_environment"
|
||||
|
||||
def process_tag_set(
|
||||
self,
|
||||
tool_shed_repository,
|
||||
tool_dependency,
|
||||
package_elem,
|
||||
package_name,
|
||||
package_version,
|
||||
tool_dependency_db_records=None,
|
||||
):
|
||||
# We need to handle two tag sets for package_elem here, this:
|
||||
# <set_environment version="1.0">
|
||||
# <environment_variable name="R_SCRIPT_PATH"action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable>
|
||||
# </set_environment>
|
||||
# or this:
|
||||
# <environment_variable name="R_SCRIPT_PATH"action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable>
|
||||
action_elem_tuples = []
|
||||
proceed_with_install = False
|
||||
if tool_dependency_db_records is None:
|
||||
attr_tups_of_dependencies_for_install = []
|
||||
else:
|
||||
attr_tups_of_dependencies_for_install = [
|
||||
(td.name, td.version, td.type) for td in tool_dependency_db_records
|
||||
]
|
||||
try:
|
||||
self.set_environment(package_elem, tool_shed_repository, attr_tups_of_dependencies_for_install)
|
||||
except Exception as e:
|
||||
error_message = f"Error setting environment for tool dependency: {e}"
|
||||
log.debug(error_message)
|
||||
return tool_dependency, proceed_with_install, action_elem_tuples
|
||||
|
||||
def set_environment(self, elem, tool_shed_repository, attr_tups_of_dependencies_for_install):
|
||||
"""
|
||||
Create a ToolDependency to set an environment variable. This is different from the process used to
|
||||
set an environment variable that is associated with a package. An example entry in a tool_dependencies.xml
|
||||
file is::
|
||||
|
||||
<set_environment version="1.0">
|
||||
<environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable>
|
||||
</set_environment>
|
||||
|
||||
This method must also handle the sub-element tag::
|
||||
<environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable>
|
||||
"""
|
||||
# TODO: Add support for a repository dependency definition within this tool dependency type's tag set. This should look something like
|
||||
# the following. See the implementation of support for this in the tool dependency package type's method above.
|
||||
# This function is only called for set environment actions as defined below, not within an <install version="1.0"> tool
|
||||
# dependency type. Here is an example of the tag set this function does handle:
|
||||
# <action type="set_environment">
|
||||
# <environment_variable name="PATH" action="prepend_to">$INSTALL_DIR</environment_variable>
|
||||
# </action>
|
||||
# Here is an example of the tag set this function does not handle:
|
||||
# <set_environment version="1.0">
|
||||
# <repository toolshed="<tool shed>" name="<repository name>" owner="<repository owner>" changeset_revision="<changeset revision>" />
|
||||
# </set_environment>
|
||||
env_manager = EnvManager(self.app)
|
||||
tool_dependencies = []
|
||||
env_var_version = elem.get("version", "1.0")
|
||||
tool_shed_repository_install_dir = os.path.abspath(tool_shed_repository.repo_files_directory(self.app))
|
||||
if elem.tag == "environment_variable":
|
||||
# <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable>
|
||||
elems = [elem]
|
||||
else:
|
||||
# <set_environment version="1.0">
|
||||
# <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable>
|
||||
# </set_environment>
|
||||
elems = [env_var_elem for env_var_elem in elem]
|
||||
for env_var_elem in elems:
|
||||
env_var_name = env_var_elem.get("name")
|
||||
if not env_var_name:
|
||||
raise Exception("The <environment_variable> tag must have a name attribute")
|
||||
# The value of env_var_name must match the text value of at least 1 <requirement> tag in the
|
||||
# tool config's <requirements> tag set whose "type" attribute is "set_environment" (e.g.,
|
||||
# <requirement type="set_environment">R_SCRIPT_PATH</requirement>).
|
||||
env_var_action = env_var_elem.get("action")
|
||||
if not env_var_action:
|
||||
raise Exception("The <environment_variable> tag must have an action attribute")
|
||||
# Tool dependencies of type "set_environment" always have the version attribute set to None.
|
||||
attr_tup = (env_var_name, None, "set_environment")
|
||||
if attr_tup in attr_tups_of_dependencies_for_install:
|
||||
install_dir = tool_dependency_util.get_tool_dependency_install_dir(
|
||||
app=self.app,
|
||||
repository_name=tool_shed_repository.name,
|
||||
repository_owner=tool_shed_repository.owner,
|
||||
repository_changeset_revision=tool_shed_repository.installed_changeset_revision,
|
||||
tool_dependency_type="set_environment",
|
||||
tool_dependency_name=env_var_name,
|
||||
tool_dependency_version=None,
|
||||
)
|
||||
install_environment = InstallEnvironment(
|
||||
app=self.app,
|
||||
tool_shed_repository_install_dir=tool_shed_repository_install_dir,
|
||||
install_dir=install_dir,
|
||||
)
|
||||
env_var_dict = env_manager.create_env_var_dict(
|
||||
elem=env_var_elem, install_environment=install_environment
|
||||
)
|
||||
if not os.path.exists(install_dir):
|
||||
os.makedirs(install_dir)
|
||||
status = self.app.install_model.ToolDependency.installation_status.INSTALLING
|
||||
tool_dependency = tool_dependency_util.create_or_update_tool_dependency(
|
||||
app=self.app,
|
||||
tool_shed_repository=tool_shed_repository,
|
||||
name=env_var_name,
|
||||
version=None,
|
||||
type="set_environment",
|
||||
status=status,
|
||||
set_status=True,
|
||||
)
|
||||
if env_var_version == "1.0":
|
||||
# Create this tool dependency's env.sh file.
|
||||
env_file_builder = EnvFileBuilder(install_dir)
|
||||
return_code = env_file_builder.append_line(make_executable=True, **env_var_dict)
|
||||
if return_code:
|
||||
error_message = f"Error creating env.sh file for tool dependency {tool_dependency.name}, return_code: {return_code}"
|
||||
log.debug(error_message)
|
||||
status = self.app.install_model.ToolDependency.installation_status.ERROR
|
||||
tool_dependency = tool_dependency_util.set_tool_dependency_attributes(
|
||||
self.app, tool_dependency=tool_dependency, status=status, error_message=error_message
|
||||
)
|
||||
else:
|
||||
if tool_dependency.status not in [
|
||||
self.app.install_model.ToolDependency.installation_status.ERROR,
|
||||
self.app.install_model.ToolDependency.installation_status.INSTALLED,
|
||||
]:
|
||||
status = self.app.install_model.ToolDependency.installation_status.INSTALLED
|
||||
tool_dependency = tool_dependency_util.set_tool_dependency_attributes(
|
||||
self.app, tool_dependency=tool_dependency, status=status
|
||||
)
|
||||
log.debug(
|
||||
"Environment variable %s set in %s for tool dependency %s."
|
||||
% (str(env_var_name), str(install_dir), str(tool_dependency.name))
|
||||
)
|
||||
else:
|
||||
error_message = 'Only set_environment version 1.0 is currently supported (i.e., change your tag to be <set_environment version="1.0">).'
|
||||
status = self.app.install_model.ToolDependency.installation_status.ERROR
|
||||
tool_dependency = tool_dependency_util.set_tool_dependency_attributes(
|
||||
self.app, tool_dependency=tool_dependency, status=status, error_message=error_message
|
||||
)
|
||||
tool_dependencies.append(tool_dependency)
|
||||
return tool_dependencies
|
||||
@@ -29,31 +29,6 @@ def can_eliminate_repository_dependency(metadata_dict, tool_shed_url, name, owne
|
||||
return True
|
||||
|
||||
|
||||
def can_eliminate_tool_dependency(metadata_dict, name, dependency_type, version):
|
||||
"""
|
||||
Determine if the relationship between a tool_dependency record
|
||||
associated with a tool_shed_repository record on the Galaxy side
|
||||
can be eliminated.
|
||||
"""
|
||||
td_dict = metadata_dict.get("tool_dependencies", {})
|
||||
for td_key, td_val in td_dict.items():
|
||||
if td_key == "set_environment":
|
||||
for td in td_val:
|
||||
n = td.get("name", None)
|
||||
t = td.get("type", None)
|
||||
if n == name and t == dependency_type:
|
||||
# The tool dependency is current, so keep it.
|
||||
return False
|
||||
else:
|
||||
n = td_val.get("name", None)
|
||||
t = td_val.get("type", None)
|
||||
v = td_val.get("version", None)
|
||||
if n == name and t == dependency_type and v == version:
|
||||
# The tool dependency is current, so keep it.
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def clean_dependency_relationships(trans, metadata_dict, tool_shed_repository, tool_shed_url):
|
||||
"""
|
||||
Repositories of type tool_dependency_definition allow for defining a
|
||||
@@ -71,13 +46,6 @@ def clean_dependency_relationships(trans, metadata_dict, tool_shed_repository, t
|
||||
log.debug(message % (r.name, r.owner, tool_shed_repository.name, tool_shed_repository.owner))
|
||||
trans.install_model.context.delete(rrda)
|
||||
trans.install_model.context.flush()
|
||||
for td in tool_shed_repository.tool_dependencies:
|
||||
if can_eliminate_tool_dependency(metadata_dict, td.name, td.type, td.version):
|
||||
message = "Tool dependency %s, version %s is not required by repository %s, owner %s, "
|
||||
message += "removing from list of tool dependencies."
|
||||
log.debug(message % (td.name, td.version, tool_shed_repository.name, tool_shed_repository.owner))
|
||||
trans.install_model.context.delete(td)
|
||||
trans.install_model.context.flush()
|
||||
|
||||
|
||||
def generate_tool_guid(repository_clone_url, tool):
|
||||
@@ -212,7 +180,6 @@ def tool_shed_is_this_tool_shed(toolshed_base_url):
|
||||
|
||||
__all__ = (
|
||||
"can_eliminate_repository_dependency",
|
||||
"can_eliminate_tool_dependency",
|
||||
"clean_dependency_relationships",
|
||||
"DATATYPES_CONFIG_FILENAME",
|
||||
"generate_tool_guid",
|
||||
|
||||
@@ -5,161 +5,10 @@ import shutil
|
||||
from sqlalchemy import and_
|
||||
|
||||
from galaxy import util
|
||||
from galaxy.tool_shed.util.hg_util import get_config_from_disk
|
||||
from galaxy.util.tool_shed.xml_util import parse_xml
|
||||
from galaxy.web.form_builder import SelectField
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def build_tool_dependencies_select_field(
|
||||
app, tool_shed_repository, name, multiple=True, display="checkboxes", uninstalled_only=False
|
||||
):
|
||||
"""
|
||||
Generate a SelectField consisting of the current list of tool dependency ids
|
||||
for an installed tool shed repository.
|
||||
"""
|
||||
tool_dependencies_select_field = SelectField(name=name, multiple=multiple, display=display)
|
||||
for tool_dependency in tool_shed_repository.tool_dependencies:
|
||||
if uninstalled_only:
|
||||
if tool_dependency.status not in [
|
||||
app.install_model.ToolDependency.installation_status.NEVER_INSTALLED,
|
||||
app.install_model.ToolDependency.installation_status.UNINSTALLED,
|
||||
]:
|
||||
continue
|
||||
else:
|
||||
if tool_dependency.status in [
|
||||
app.install_model.ToolDependency.installation_status.NEVER_INSTALLED,
|
||||
app.install_model.ToolDependency.installation_status.UNINSTALLED,
|
||||
]:
|
||||
continue
|
||||
option_label = f"{tool_dependency.name} version {tool_dependency.version}"
|
||||
option_value = app.security.encode_id(tool_dependency.id)
|
||||
tool_dependencies_select_field.add_option(option_label, option_value)
|
||||
return tool_dependencies_select_field
|
||||
|
||||
|
||||
def create_or_update_tool_dependency(app, tool_shed_repository, name, version, type, status, set_status=True):
|
||||
"""Create or update a tool_dependency record in the Galaxy database."""
|
||||
# Called from Galaxy (never the tool shed) when a new repository is being installed or when an uninstalled
|
||||
# repository is being reinstalled.
|
||||
context = app.install_model.context
|
||||
# First see if an appropriate tool_dependency record exists for the received tool_shed_repository.
|
||||
if version:
|
||||
tool_dependency = get_tool_dependency_by_name_version_type_repository(
|
||||
app, tool_shed_repository, name, version, type
|
||||
)
|
||||
else:
|
||||
tool_dependency = get_tool_dependency_by_name_type_repository(app, tool_shed_repository, name, type)
|
||||
if tool_dependency:
|
||||
# In some cases we should not override the current status of an existing tool_dependency, so do so only
|
||||
# if set_status is True.
|
||||
if set_status:
|
||||
set_tool_dependency_attributes(app, tool_dependency=tool_dependency, status=status)
|
||||
else:
|
||||
# Create a new tool_dependency record for the tool_shed_repository.
|
||||
debug_msg = f"Creating a new record for version {version} of tool dependency {name} for revision {tool_shed_repository.changeset_revision} of repository {tool_shed_repository.name}. "
|
||||
debug_msg += f"The status is being set to {status}."
|
||||
log.debug(debug_msg)
|
||||
tool_dependency = app.install_model.ToolDependency(tool_shed_repository.id, name, version, type, status)
|
||||
context.add(tool_dependency)
|
||||
context.flush()
|
||||
return tool_dependency
|
||||
|
||||
|
||||
def create_tool_dependency_objects(app, tool_shed_repository, relative_install_dir, set_status=True):
|
||||
"""
|
||||
Create or update a ToolDependency for each entry in tool_dependencies_config. This method is called when
|
||||
installing a new tool_shed_repository.
|
||||
"""
|
||||
tool_dependency_objects = []
|
||||
shed_config_dict = tool_shed_repository.get_shed_config_dict(app)
|
||||
if shed_config_dict.get("tool_path"):
|
||||
relative_install_dir = os.path.join(shed_config_dict.get("tool_path"), relative_install_dir)
|
||||
# Get the tool_dependencies.xml file from the repository.
|
||||
tool_dependencies_config = get_config_from_disk("tool_dependencies.xml", relative_install_dir)
|
||||
tree, error_message = parse_xml(tool_dependencies_config)
|
||||
if tree is None:
|
||||
return tool_dependency_objects
|
||||
root = tree.getroot()
|
||||
for elem in root:
|
||||
tool_dependency_type = elem.tag
|
||||
if tool_dependency_type == "package":
|
||||
name = elem.get("name", None)
|
||||
version = elem.get("version", None)
|
||||
if name and version:
|
||||
status = app.install_model.ToolDependency.installation_status.NEVER_INSTALLED
|
||||
tool_dependency = create_or_update_tool_dependency(
|
||||
app,
|
||||
tool_shed_repository,
|
||||
name=name,
|
||||
version=version,
|
||||
type=tool_dependency_type,
|
||||
status=status,
|
||||
set_status=set_status,
|
||||
)
|
||||
tool_dependency_objects.append(tool_dependency)
|
||||
elif tool_dependency_type == "set_environment":
|
||||
for env_elem in elem:
|
||||
# <environment_variable name="R_SCRIPT_PATH" action="set_to">$REPOSITORY_INSTALL_DIR</environment_variable>
|
||||
name = env_elem.get("name", None)
|
||||
action = env_elem.get("action", None)
|
||||
if name and action:
|
||||
status = app.install_model.ToolDependency.installation_status.NEVER_INSTALLED
|
||||
tool_dependency = create_or_update_tool_dependency(
|
||||
app,
|
||||
tool_shed_repository,
|
||||
name=name,
|
||||
version=None,
|
||||
type=tool_dependency_type,
|
||||
status=status,
|
||||
set_status=set_status,
|
||||
)
|
||||
tool_dependency_objects.append(tool_dependency)
|
||||
return tool_dependency_objects
|
||||
|
||||
|
||||
def get_download_url_for_platform(url_templates, platform_info_dict):
|
||||
"""
|
||||
Compare the dict returned by get_platform_info() with the values specified in the url_template element. Return
|
||||
true if and only if all defined attributes match the corresponding dict entries. If an entry is not
|
||||
defined in the url_template element, it is assumed to be irrelevant at this stage. For example,
|
||||
<url_template os="darwin">http://hgdownload.cse.ucsc.edu/admin/exe/macOSX.${architecture}/faToTwoBit</url_template>
|
||||
where the OS must be 'darwin', but the architecture is filled in later using string.Template.
|
||||
"""
|
||||
os_ok = False
|
||||
architecture_ok = False
|
||||
for url_template in url_templates:
|
||||
os_name = url_template.get("os", None)
|
||||
architecture = url_template.get("architecture", None)
|
||||
if os_name:
|
||||
if os_name.lower() == platform_info_dict["os"]:
|
||||
os_ok = True
|
||||
else:
|
||||
os_ok = False
|
||||
else:
|
||||
os_ok = True
|
||||
if architecture:
|
||||
if architecture.lower() == platform_info_dict["architecture"]:
|
||||
architecture_ok = True
|
||||
else:
|
||||
architecture_ok = False
|
||||
else:
|
||||
architecture_ok = True
|
||||
if os_ok and architecture_ok:
|
||||
return url_template
|
||||
return None
|
||||
|
||||
|
||||
def get_platform_info_dict():
|
||||
"""Return a dict with information about the current platform."""
|
||||
platform_dict = {}
|
||||
sysname, nodename, release, version, machine = os.uname()
|
||||
platform_dict["os"] = sysname.lower()
|
||||
platform_dict["architecture"] = machine.lower()
|
||||
return platform_dict
|
||||
|
||||
|
||||
def get_tool_dependency(app, id):
|
||||
"""Get a tool_dependency from the database via id"""
|
||||
return app.install_model.context.query(app.install_model.ToolDependency).get(app.security.decode_id(id))
|
||||
@@ -230,39 +79,6 @@ def get_tool_dependency_ids(as_string=False, **kwd):
|
||||
return tool_dependency_ids
|
||||
|
||||
|
||||
def get_tool_dependency_install_dir(
|
||||
app,
|
||||
repository_name,
|
||||
repository_owner,
|
||||
repository_changeset_revision,
|
||||
tool_dependency_type,
|
||||
tool_dependency_name,
|
||||
tool_dependency_version,
|
||||
):
|
||||
if tool_dependency_type == "package":
|
||||
return os.path.abspath(
|
||||
os.path.join(
|
||||
app.tool_dependency_dir,
|
||||
tool_dependency_name,
|
||||
tool_dependency_version,
|
||||
repository_owner,
|
||||
repository_name,
|
||||
repository_changeset_revision,
|
||||
)
|
||||
)
|
||||
if tool_dependency_type == "set_environment":
|
||||
return os.path.abspath(
|
||||
os.path.join(
|
||||
app.tool_dependency_dir,
|
||||
"environment_settings",
|
||||
tool_dependency_name,
|
||||
repository_owner,
|
||||
repository_name,
|
||||
repository_changeset_revision,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def parse_package_elem(package_elem, platform_info_dict=None, include_after_install_actions=True):
|
||||
"""
|
||||
Parse a <package> element within a tool dependency definition and return a list of action tuples.
|
||||
@@ -421,28 +237,3 @@ def remove_tool_dependency_installation_directory(dependency_install_dir):
|
||||
removed = True
|
||||
error_message = ""
|
||||
return removed, error_message
|
||||
|
||||
|
||||
def set_tool_dependency_attributes(app, tool_dependency, status, error_message=None):
|
||||
sa_session = app.install_model.context
|
||||
if status == app.install_model.ToolDependency.installation_status.UNINSTALLED:
|
||||
installation_directory = tool_dependency.installation_directory(app)
|
||||
remove_tool_dependency_installation_directory(installation_directory)
|
||||
tool_dependency.error_message = error_message
|
||||
if str(tool_dependency.status) != str(status):
|
||||
tool_shed_repository = tool_dependency.tool_shed_repository
|
||||
debug_msg = (
|
||||
"Updating an existing record for version %s of tool dependency %s for revision %s of repository %s "
|
||||
% (
|
||||
str(tool_dependency.version),
|
||||
str(tool_dependency.name),
|
||||
str(tool_shed_repository.changeset_revision),
|
||||
str(tool_shed_repository.name),
|
||||
)
|
||||
)
|
||||
debug_msg += f"by updating the status from {tool_dependency.status} to {status}."
|
||||
log.debug(debug_msg)
|
||||
tool_dependency.status = status
|
||||
sa_session.add(tool_dependency)
|
||||
sa_session.flush()
|
||||
return tool_dependency
|
||||
|
||||
@@ -302,52 +302,6 @@ class AdminToolshed(AdminGalaxy):
|
||||
repo_information_dict = json.loads(raw_text)
|
||||
return repo_information_dict
|
||||
|
||||
@web.expose
|
||||
@web.require_admin
|
||||
@legacy_tool_shed_endpoint
|
||||
def initiate_tool_dependency_installation(self, trans, tool_dependencies, **kwd):
|
||||
"""
|
||||
Install specified dependencies for repository tools. The received list of tool_dependencies
|
||||
are the database records for those dependencies defined in the tool_dependencies.xml file
|
||||
(contained in the repository) that should be installed. This allows for filtering out dependencies
|
||||
that have not been checked for installation on the 'Manage tool dependencies' page for an installed
|
||||
tool shed repository.
|
||||
"""
|
||||
# Get the tool_shed_repository from one of the tool_dependencies.
|
||||
message = str(escape(kwd.get("message", "")))
|
||||
status = kwd.get("status", "done")
|
||||
err_msg = ""
|
||||
tool_shed_repository = tool_dependencies[0].tool_shed_repository
|
||||
# Get the tool_dependencies.xml file from the repository.
|
||||
tool_dependencies_config = hg_util.get_config_from_disk(
|
||||
rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME, tool_shed_repository.repo_path(trans.app)
|
||||
)
|
||||
itdm = install_manager.InstallToolDependencyManager(trans.app)
|
||||
installed_tool_dependencies = itdm.install_specified_tool_dependencies(
|
||||
tool_shed_repository=tool_shed_repository,
|
||||
tool_dependencies_config=tool_dependencies_config,
|
||||
tool_dependencies=tool_dependencies,
|
||||
)
|
||||
for installed_tool_dependency in installed_tool_dependencies:
|
||||
if installed_tool_dependency.status == trans.app.install_model.ToolDependency.installation_status.ERROR:
|
||||
text = unicodify(installed_tool_dependency.error_message)
|
||||
if text is not None:
|
||||
err_msg += f" {text}"
|
||||
if err_msg:
|
||||
message += err_msg
|
||||
status = "error"
|
||||
message += f"Installed tool dependencies: {', '.join(td.name for td in installed_tool_dependencies)}"
|
||||
td_ids = [trans.security.encode_id(td.id) for td in tool_shed_repository.tool_dependencies]
|
||||
return trans.response.send_redirect(
|
||||
web.url_for(
|
||||
controller="admin_toolshed",
|
||||
action="manage_tool_dependencies",
|
||||
tool_dependency_ids=td_ids,
|
||||
message=message,
|
||||
status=status,
|
||||
)
|
||||
)
|
||||
|
||||
@web.expose
|
||||
@web.require_admin
|
||||
@legacy_tool_shed_endpoint
|
||||
@@ -436,10 +390,6 @@ class AdminToolshed(AdminGalaxy):
|
||||
else:
|
||||
tool_dependencies_dict = {}
|
||||
encoded_relative_install_dir = kwd.get("encoded_relative_install_dir", None)
|
||||
if encoded_relative_install_dir is not None:
|
||||
relative_install_dir = encoding_util.tool_shed_decode(encoded_relative_install_dir)
|
||||
else:
|
||||
relative_install_dir = ""
|
||||
updating_to_changeset_revision = kwd.get("updating_to_changeset_revision", None)
|
||||
updating_to_ctx_rev = kwd.get("updating_to_ctx_rev", None)
|
||||
encoded_updated_metadata = kwd.get("encoded_updated_metadata", None)
|
||||
@@ -458,27 +408,6 @@ class AdminToolshed(AdminGalaxy):
|
||||
updated_changeset_revision=updating_to_changeset_revision,
|
||||
updated_ctx_rev=updating_to_ctx_rev,
|
||||
)
|
||||
if tool_dependencies_dict:
|
||||
tool_dependencies = tool_dependency_util.create_tool_dependency_objects(
|
||||
trans.app, repository, relative_install_dir, set_status=False
|
||||
)
|
||||
message = f"The installed repository named '{escape(str(repository.name))}' has been updated to change set revision '{updating_to_changeset_revision}'. "
|
||||
self.initiate_tool_dependency_installation(trans, tool_dependencies, message=message, status=status)
|
||||
# Handle tool dependencies check box.
|
||||
if not trans.app.toolbox.dependency_manager.uses_tool_shed_dependencies:
|
||||
if tool_dependencies_dict:
|
||||
message = (
|
||||
"Tool dependencies defined in this repository can be automatically installed if you set "
|
||||
"the value of your <b>tool_dependency_dir</b> setting in your Galaxy config file "
|
||||
"(galaxy.yml) and restart your Galaxy server."
|
||||
)
|
||||
status = "warning"
|
||||
install_tool_dependencies_check_box_checked = False
|
||||
else:
|
||||
install_tool_dependencies_check_box_checked = True
|
||||
install_tool_dependencies_check_box = CheckboxField(
|
||||
"install_tool_dependencies", value=install_tool_dependencies_check_box_checked
|
||||
)
|
||||
view = views.DependencyResolversView(self.app)
|
||||
if view.installable_resolvers:
|
||||
install_resolver_dependencies_check_box = CheckboxField("install_resolver_dependencies", value=True)
|
||||
@@ -494,7 +423,6 @@ class AdminToolshed(AdminGalaxy):
|
||||
encoded_relative_install_dir=encoded_relative_install_dir,
|
||||
encoded_tool_dependencies_dict=encoded_tool_dependencies_dict,
|
||||
install_resolver_dependencies_check_box=install_resolver_dependencies_check_box,
|
||||
install_tool_dependencies_check_box=install_tool_dependencies_check_box,
|
||||
tool_dependencies_dict=tool_dependencies_dict,
|
||||
message=message,
|
||||
status=status,
|
||||
@@ -928,19 +856,6 @@ class AdminToolshed(AdminGalaxy):
|
||||
containers_dicts.append(containers_dict)
|
||||
# Merge all containers into a single container.
|
||||
containers_dict = dd.merge_containers_dicts_for_new_install(containers_dicts)
|
||||
# Handle tool dependencies check box.
|
||||
if not trans.app.toolbox.dependency_manager.uses_tool_shed_dependencies:
|
||||
if includes_tool_dependencies:
|
||||
message = "Tool dependencies defined in this repository can be automatically installed if you set "
|
||||
message += "the value of your <b>tool_dependency_dir</b> setting in your Galaxy config file "
|
||||
message += "(galaxy.yml) and restart your Galaxy server before installing the repository."
|
||||
status = "warning"
|
||||
install_tool_dependencies_check_box_checked = False
|
||||
else:
|
||||
install_tool_dependencies_check_box_checked = True
|
||||
install_tool_dependencies_check_box = CheckboxField(
|
||||
"install_tool_dependencies", value=install_tool_dependencies_check_box_checked
|
||||
)
|
||||
# Handle repository dependencies check box.
|
||||
install_repository_dependencies_check_box = CheckboxField("install_repository_dependencies", value=True)
|
||||
view = views.DependencyResolversView(self.app)
|
||||
@@ -961,7 +876,6 @@ class AdminToolshed(AdminGalaxy):
|
||||
includes_tools=includes_tools,
|
||||
includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel,
|
||||
includes_tool_dependencies=includes_tool_dependencies,
|
||||
install_tool_dependencies_check_box=install_tool_dependencies_check_box,
|
||||
install_resolver_dependencies_check_box=install_resolver_dependencies_check_box,
|
||||
has_repository_dependencies=has_repository_dependencies,
|
||||
install_repository_dependencies_check_box=install_repository_dependencies_check_box,
|
||||
@@ -1324,18 +1238,6 @@ class AdminToolshed(AdminGalaxy):
|
||||
containers_dict = dd.merge_missing_repository_dependencies_to_installed_container(containers_dict)
|
||||
# Handle repository dependencies check box.
|
||||
install_repository_dependencies_check_box = CheckboxField("install_repository_dependencies", value=True)
|
||||
# Handle tool dependencies check box.
|
||||
if not trans.app.toolbox.dependency_manager.uses_tool_shed_dependencies:
|
||||
if includes_tool_dependencies:
|
||||
message += "Tool dependencies defined in this repository can be automatically installed if you set the value of your <b>tool_dependency_dir</b> "
|
||||
message += "setting in your Galaxy config file (galaxy.yml) and restart your Galaxy server before installing the repository. "
|
||||
status = "warning"
|
||||
install_tool_dependencies_check_box_checked = False
|
||||
else:
|
||||
install_tool_dependencies_check_box_checked = True
|
||||
install_tool_dependencies_check_box = CheckboxField(
|
||||
"install_tool_dependencies", value=install_tool_dependencies_check_box_checked
|
||||
)
|
||||
view = views.DependencyResolversView(self.app)
|
||||
if view.installable_resolvers:
|
||||
install_resolver_dependencies_check_box = CheckboxField("install_resolver_dependencies", value=True)
|
||||
@@ -1354,7 +1256,6 @@ class AdminToolshed(AdminGalaxy):
|
||||
includes_workflows=includes_workflows,
|
||||
has_repository_dependencies=has_repository_dependencies,
|
||||
install_repository_dependencies_check_box=install_repository_dependencies_check_box,
|
||||
install_tool_dependencies_check_box=install_tool_dependencies_check_box,
|
||||
install_resolver_dependencies_check_box=install_resolver_dependencies_check_box,
|
||||
containers_dict=containers_dict,
|
||||
tool_panel_section_select_field=tool_panel_section_select_field,
|
||||
@@ -1460,26 +1361,6 @@ class AdminToolshed(AdminGalaxy):
|
||||
updating=True,
|
||||
)
|
||||
return self.prepare_for_install(trans, **new_kwd)
|
||||
elif install_dependencies == "tool":
|
||||
# Updates received did not include any newly defined repository dependencies but did include
|
||||
# newly defined tool dependencies. If the newly defined tool dependencies are not the same
|
||||
# as the originally defined tool dependencies, we need to install them.
|
||||
relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir(trans.app, repository)[2]
|
||||
encoded_tool_dependencies_dict = encoding_util.tool_shed_encode(
|
||||
irmm_metadata_dict.get("tool_dependencies", {})
|
||||
)
|
||||
encoded_relative_install_dir = encoding_util.tool_shed_encode(relative_install_dir)
|
||||
new_kwd = dict(
|
||||
updating_repository_id=trans.security.encode_id(repository.id),
|
||||
updating_to_ctx_rev=latest_ctx_rev,
|
||||
updating_to_changeset_revision=latest_changeset_revision,
|
||||
encoded_updated_metadata=encoding_util.tool_shed_encode(irmm_metadata_dict),
|
||||
encoded_relative_install_dir=encoded_relative_install_dir,
|
||||
encoded_tool_dependencies_dict=encoded_tool_dependencies_dict,
|
||||
message=message,
|
||||
status=status,
|
||||
)
|
||||
return self.install_tool_dependencies_with_update(trans, **new_kwd)
|
||||
message = f"The installed repository named '{name}' has been updated to change set revision '{latest_changeset_revision}'. "
|
||||
else:
|
||||
message = (
|
||||
|
||||
@@ -71,8 +71,6 @@ class ShedTwillTestCase(DrivenFunctionalTestCase):
|
||||
self.tool_data_path = os.environ.get("GALAXY_TEST_TOOL_DATA_PATH")
|
||||
self.shed_tool_conf = os.environ.get("GALAXY_TEST_SHED_TOOL_CONF")
|
||||
self.test_db_util = test_db_util
|
||||
# TODO: Figure out a way to alter these attributes during tests.
|
||||
self.galaxy_tool_dependency_dir = os.environ.get("GALAXY_TEST_TOOL_DEPENDENCY_DIR")
|
||||
|
||||
def check_for_strings(self, strings_displayed=None, strings_not_displayed=None):
|
||||
strings_displayed = strings_displayed or []
|
||||
@@ -354,22 +352,6 @@ class ShedTwillTestCase(DrivenFunctionalTestCase):
|
||||
tool_panel_section == expected_tool_panel_section
|
||||
), f"Expected to find tool panel section *{expected_tool_panel_section}*, but instead found *{tool_panel_section}*\nMetadata: {metadata}\n"
|
||||
|
||||
def check_installed_repository_tool_dependencies(
|
||||
self, installed_repository, strings_displayed=None, strings_not_displayed=None, dependencies_installed=False
|
||||
):
|
||||
# Tool dependencies are not being installed in these functional tests. If this is changed, the test method will also need to be updated.
|
||||
if not dependencies_installed:
|
||||
strings_displayed.append("Missing tool dependencies")
|
||||
else:
|
||||
strings_displayed.append("Tool dependencies")
|
||||
if dependencies_installed:
|
||||
strings_displayed.append("Installed")
|
||||
else:
|
||||
strings_displayed.append("Never installed")
|
||||
params = {"id": self.security.encode_id(installed_repository.id)}
|
||||
self.visit_galaxy_url("/admin_toolshed/manage_repository", params=params)
|
||||
self.check_for_strings(strings_displayed, strings_not_displayed)
|
||||
|
||||
def check_repository_changelog(self, repository, strings_displayed=None, strings_not_displayed=None):
|
||||
params = {"id": self.security.encode_id(repository.id)}
|
||||
self.visit_url("/repository/view_changelog", params=params)
|
||||
@@ -898,13 +880,6 @@ class ShedTwillTestCase(DrivenFunctionalTestCase):
|
||||
datatypes = loads(html)
|
||||
return len(datatypes)
|
||||
|
||||
def get_env_sh_path(self, tool_dependency_name, tool_dependency_version, repository):
|
||||
"""Return the absolute path to an installed repository's env.sh file."""
|
||||
env_sh_path = os.path.join(
|
||||
self.get_tool_dependency_path(tool_dependency_name, tool_dependency_version, repository), "env.sh"
|
||||
)
|
||||
return env_sh_path
|
||||
|
||||
def get_filename(self, filename, filepath=None):
|
||||
if filepath is not None:
|
||||
return os.path.abspath(os.path.join(filepath, filename))
|
||||
@@ -931,17 +906,6 @@ class ShedTwillTestCase(DrivenFunctionalTestCase):
|
||||
self.visit_url(url)
|
||||
self.check_for_strings(strings_displayed, strings_not_displayed)
|
||||
|
||||
def get_tool_dependency_path(self, tool_dependency_name, tool_dependency_version, repository):
|
||||
"""Return the absolute path for an installed tool dependency."""
|
||||
return os.path.join(
|
||||
self.galaxy_tool_dependency_dir,
|
||||
tool_dependency_name,
|
||||
tool_dependency_version,
|
||||
repository.owner,
|
||||
repository.name,
|
||||
repository.installed_changeset_revision,
|
||||
)
|
||||
|
||||
def get_or_create_repository(self, owner=None, strings_displayed=None, strings_not_displayed=None, **kwd):
|
||||
# If not checking for a specific string, it should be safe to assume that
|
||||
# we expect repository creation to be successful.
|
||||
|
||||
@@ -136,8 +136,6 @@ class ToolWithToolDependencies(ShedTwillTestCase):
|
||||
strings_displayed = [
|
||||
"Never installed",
|
||||
"install all needed dependencies",
|
||||
"install Tool Shed managed",
|
||||
"tool dependencies",
|
||||
]
|
||||
strings_displayed.extend(["freebayes", "0.9.4_9696d0ce8a9", "samtools", "0.1.18"])
|
||||
self.install_repository(
|
||||
@@ -161,10 +159,6 @@ class ToolWithToolDependencies(ShedTwillTestCase):
|
||||
self.display_galaxy_browse_repositories_page(strings_displayed=strings_displayed)
|
||||
strings_displayed.extend(["Installed tool shed repository", "Valid tools", "FreeBayes"])
|
||||
self.display_installed_repository_manage_page(installed_repository, strings_displayed=strings_displayed)
|
||||
strings_displayed = ["freebayes", "0.9.4_9696d0ce8a9", "samtools", "0.1.18"]
|
||||
self.check_installed_repository_tool_dependencies(
|
||||
installed_repository, strings_displayed=strings_displayed, dependencies_installed=False
|
||||
)
|
||||
self.verify_tool_metadata_for_installed_repository(installed_repository)
|
||||
|
||||
def test_0020_verify_installed_repository_metadata(self):
|
||||
|
||||
-4
@@ -137,10 +137,6 @@ class ToolWithRepositoryDependencies(ShedTwillTestCase):
|
||||
self.display_galaxy_browse_repositories_page(strings_displayed=strings_displayed)
|
||||
strings_displayed.extend(["Installed tool shed repository", "Valid tools", "antigenic"])
|
||||
self.display_installed_repository_manage_page(installed_repository, strings_displayed=strings_displayed)
|
||||
strings_displayed = ["emboss", "5.0.0", "package"]
|
||||
self.check_installed_repository_tool_dependencies(
|
||||
installed_repository, strings_displayed=strings_displayed, dependencies_installed=False
|
||||
)
|
||||
self.verify_tool_metadata_for_installed_repository(installed_repository)
|
||||
current_datatypes = int(self.get_datatypes_count())
|
||||
assert (
|
||||
|
||||
-4
@@ -215,10 +215,6 @@ class RepositoryWithDependencyRevisions(ShedTwillTestCase):
|
||||
self.display_galaxy_browse_repositories_page(strings_displayed=strings_displayed)
|
||||
strings_displayed.extend(["Installed tool shed repository", "Valid tools", "antigenic"])
|
||||
self.display_installed_repository_manage_page(installed_repository, strings_displayed=strings_displayed)
|
||||
strings_displayed = ["emboss", "5.0.0", "package"]
|
||||
self.check_installed_repository_tool_dependencies(
|
||||
installed_repository, strings_displayed=strings_displayed, dependencies_installed=False
|
||||
)
|
||||
self.verify_tool_metadata_for_installed_repository(installed_repository)
|
||||
self.update_installed_repository(installed_repository, strings_displayed=["there are no updates available"])
|
||||
current_datatypes = int(self.get_datatypes_count())
|
||||
|
||||
-2
@@ -153,7 +153,6 @@ class TestInstallingCircularDependencies(ShedTwillTestCase):
|
||||
self.display_installed_repository_manage_page(installed_filtering_repository)
|
||||
self.deactivate_repository(installed_filtering_repository)
|
||||
self.test_db_util.ga_refresh(installed_filtering_repository)
|
||||
self.check_galaxy_repository_tool_panel_section(installed_filtering_repository, "Get Data")
|
||||
strings_displayed = [
|
||||
"Missing repository",
|
||||
"filtering",
|
||||
@@ -191,7 +190,6 @@ class TestInstallingCircularDependencies(ShedTwillTestCase):
|
||||
self.display_installed_repository_manage_page(installed_filtering_repository)
|
||||
self.deactivate_repository(installed_freebayes_repository)
|
||||
self.test_db_util.ga_refresh(installed_freebayes_repository)
|
||||
self.check_galaxy_repository_tool_panel_section(installed_freebayes_repository, "Get Data")
|
||||
strings_displayed = [
|
||||
"Missing repository",
|
||||
"freebayes",
|
||||
|
||||
-46
@@ -321,49 +321,3 @@ class TestInstallingComplexRepositoryDependencies(ShedTwillTestCase):
|
||||
self.display_installed_repository_manage_page(
|
||||
base_repository, strings_displayed=strings_displayed, strings_not_displayed=strings_not_displayed
|
||||
)
|
||||
|
||||
def test_0055_verify_complex_tool_dependency(self):
|
||||
"""Verify that the generated env.sh contains the right data."""
|
||||
base_repository = self.test_db_util.get_installed_repository_by_name_owner(
|
||||
bwa_base_repository_name, common.test_user_1_name
|
||||
)
|
||||
tool_repository = self.test_db_util.get_installed_repository_by_name_owner(
|
||||
bwa_package_repository_name, common.test_user_1_name
|
||||
)
|
||||
env_sh_path = self.get_env_sh_path(
|
||||
tool_dependency_name="bwa", tool_dependency_version="0.5.9", repository=base_repository
|
||||
)
|
||||
assert os.path.exists(env_sh_path), f"env.sh was not generated in {env_sh_path} for this dependency."
|
||||
contents = open(env_sh_path).read()
|
||||
if tool_repository.installed_changeset_revision not in contents:
|
||||
raise AssertionError(
|
||||
"Installed changeset revision %s not found in env.sh.\nContents of env.sh: %s"
|
||||
% (tool_repository.installed_changeset_revision, contents)
|
||||
)
|
||||
if "package_bwa_0_5_9_0100" not in contents:
|
||||
raise AssertionError(
|
||||
f"Repository name package_bwa_0_5_9_0100 not found in env.sh.\nContents of env.sh: {contents}"
|
||||
)
|
||||
|
||||
def test_0060_verify_tool_dependency_uninstallation(self):
|
||||
"""Uninstall the package_bwa_0_5_9_0100 repository.
|
||||
|
||||
Uninstall the repository that defines a tool dependency relationship on BWA 0.5.9, and verify
|
||||
that this results in the compiled binary package also being removed.
|
||||
"""
|
||||
tool_repository = self.test_db_util.get_installed_repository_by_name_owner(
|
||||
bwa_package_repository_name, common.test_user_1_name
|
||||
)
|
||||
self.deactivate_repository(tool_repository)
|
||||
env_sh_path = os.path.join(
|
||||
self.galaxy_tool_dependency_dir,
|
||||
"bwa",
|
||||
"0.5.9",
|
||||
tool_repository.owner,
|
||||
tool_repository.name,
|
||||
tool_repository.installed_changeset_revision,
|
||||
"env.sh",
|
||||
)
|
||||
assert os.path.exists(
|
||||
env_sh_path
|
||||
), f"Path {env_sh_path} does not exist after deactivating the repository that generated it."
|
||||
|
||||
+1
-8
@@ -117,14 +117,7 @@ class UninstallingAndReinstallingRepositories(ShedTwillTestCase):
|
||||
def test_0010_install_freebayes_repository(self):
|
||||
"""Install the freebayes repository into the Galaxy instance."""
|
||||
self.galaxy_login(email=common.admin_email, username=common.admin_username)
|
||||
strings_displayed = [
|
||||
"install Tool Shed managed",
|
||||
"tool dependencies",
|
||||
"freebayes",
|
||||
"0.9.4_9696d0ce8a9",
|
||||
"samtools",
|
||||
"0.1.18",
|
||||
]
|
||||
strings_displayed = []
|
||||
self.install_repository(
|
||||
"freebayes_0010",
|
||||
common.test_user_1_name,
|
||||
|
||||
@@ -1,411 +0,0 @@
|
||||
import logging
|
||||
|
||||
from ..base.twilltestcase import (
|
||||
common,
|
||||
ShedTwillTestCase,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
category_name = "Test 1420 Tool dependency environment variable inheritance"
|
||||
category_description = "Test script 1420 for interpolation of inherited environment variables."
|
||||
package_atlas_repository_name = "package_atlas_3_10_1420"
|
||||
package_bzlib_repository_name = "package_bzlib_1_0_1420"
|
||||
package_boost_repository_name = "package_boost_1_53_1420"
|
||||
package_numpy_repository_name = "package_numpy_1_7_1420"
|
||||
package_rdkit_repository_name = "package_rdkit_2012_12_1420"
|
||||
package_lapack_repository_name = "package_lapack_3_4_1420"
|
||||
package_atlas_repository_description = "Automatically Tuned Linear Algebra Software"
|
||||
package_bzlib_repository_description = (
|
||||
"Contains a tool dependency definition that downloads and compiles version 1.0 of the bzlib library."
|
||||
)
|
||||
package_boost_repository_description = (
|
||||
"Contains a tool dependency definition that downloads and compiles version 1.53 of the boost C++ libraries"
|
||||
)
|
||||
package_numpy_repository_description = (
|
||||
"Contains a tool dependency definition that downloads and compiles version 1.7 of the the python numpy package"
|
||||
)
|
||||
package_rdkit_repository_description = "Contains a tool dependency definition that downloads and compiles version 2012-12 of the RDKit cheminformatics and machine-learning package."
|
||||
package_lapack_repository_description = "Linear Algebra PACKage"
|
||||
package_atlas_repository_long_description = f"{package_atlas_repository_name}: {package_atlas_repository_description}"
|
||||
package_bzlib_repository_long_description = f"{package_bzlib_repository_name}: {package_bzlib_repository_description}"
|
||||
package_boost_repository_long_description = f"{package_boost_repository_name}: {package_boost_repository_description}"
|
||||
package_numpy_repository_long_description = f"{package_numpy_repository_name}: {package_numpy_repository_description}"
|
||||
package_rdkit_repository_long_description = f"{package_rdkit_repository_name}: {package_rdkit_repository_description}"
|
||||
package_lapack_repository_long_description = (
|
||||
f"{package_lapack_repository_name}: {package_lapack_repository_description}"
|
||||
)
|
||||
|
||||
"""
|
||||
1. Create repository package_lapack_3_4_1420
|
||||
|
||||
2. Create repository package_atlas_3_10_1420
|
||||
|
||||
3. Create repository package_bzlib_1_0_1420
|
||||
|
||||
4. Create repository package_boost_1_53_1420
|
||||
|
||||
5. Create repository package_numpy_1_7_1420
|
||||
|
||||
6. Create repository package_rdkit_2012_12_1420
|
||||
|
||||
Repository dependency structure should be as follows:
|
||||
Repository package_rdkit_2012_12_1420
|
||||
Repository package_boost_1_53_1420 (prior install required)
|
||||
Repository package_bzlib_1_0_1420 (prior install required)
|
||||
Repository package_numpy_1_7_1420 (prior install required)
|
||||
Repository package_lapack_3_4_1420 (prior install required)
|
||||
Repository package_atlas_3_10_1420 (prior install required)
|
||||
|
||||
8. Install package_rdkit_2012_12 into Galaxy.
|
||||
|
||||
9. Verify that the env.sh file for package_rdkit_2012_12_1420 also defines the variables inherited from package_numpy_1_7_1420 and package_boost_1_53_1420.
|
||||
"""
|
||||
|
||||
|
||||
class TestEnvironmentInheritance(ShedTwillTestCase):
|
||||
"""Test referencing environment variables that were defined in a separate tool dependency."""
|
||||
|
||||
def test_0000_initiate_users_and_category(self):
|
||||
"""Create necessary user accounts and login as an admin user."""
|
||||
self.login(email=common.admin_email, username=common.admin_username)
|
||||
admin_user = self.test_db_util.get_user(common.admin_email)
|
||||
assert admin_user is not None, f"Problem retrieving user with email {common.admin_email} from the database"
|
||||
self.test_db_util.get_private_role(admin_user)
|
||||
self.create_category(name=category_name, description=category_description)
|
||||
self.login(email=common.test_user_2_email, username=common.test_user_2_name)
|
||||
test_user_2 = self.test_db_util.get_user(common.test_user_2_email)
|
||||
assert (
|
||||
test_user_2 is not None
|
||||
), f"Problem retrieving user with email {common.test_user_2_email} from the database"
|
||||
self.test_db_util.get_private_role(test_user_2)
|
||||
self.login(email=common.test_user_1_email, username=common.test_user_1_name)
|
||||
test_user_1 = self.test_db_util.get_user(common.test_user_1_email)
|
||||
assert (
|
||||
test_user_1 is not None
|
||||
), f"Problem retrieving user with email {common.test_user_1_email} from the database"
|
||||
self.test_db_util.get_private_role(test_user_1)
|
||||
|
||||
def test_0005_create_lapack_repository(self):
|
||||
"""Create and populate package_lapack_3_4_1420.
|
||||
|
||||
This is step 1 - Create repository package_lapack_3_4_1420.
|
||||
|
||||
All tool dependency definitions should download and extract a tarball containing precompiled binaries from the local
|
||||
filesystem and install them into the path specified by $INSTALL_DIR.
|
||||
"""
|
||||
category = self.test_db_util.get_category_by_name(category_name)
|
||||
repository = self.get_or_create_repository(
|
||||
name=package_lapack_repository_name,
|
||||
description=package_lapack_repository_description,
|
||||
long_description=package_lapack_repository_long_description,
|
||||
owner=common.test_user_1_name,
|
||||
category_id=self.security.encode_id(category.id),
|
||||
strings_displayed=[],
|
||||
)
|
||||
# Load the original tool dependency definition into memory, then fill in the __PATH__ placeholder with the
|
||||
# actual system path where the binary tarball is found.
|
||||
tool_dependency_path = self.generate_temp_path(
|
||||
"1420_tool_dependency", additional_paths=["package_lapack_3_4_1420"]
|
||||
)
|
||||
precompiled_binary_tarball = self.get_filename("1420_files/binary_tarballs/lapack.tar")
|
||||
edited_tool_dependency_filename = self.get_filename(
|
||||
filepath=tool_dependency_path, filename="tool_dependencies.xml"
|
||||
)
|
||||
original_tool_dependency = self.get_filename("1420_files/package_lapack_3_4_1420/tool_dependencies.xml")
|
||||
tool_dependency_definition = (
|
||||
open(original_tool_dependency).read().replace("__PATH__", precompiled_binary_tarball)
|
||||
)
|
||||
open(edited_tool_dependency_filename, "w").write(tool_dependency_definition)
|
||||
# Upload the edited tool dependency definition to the package_lapack_3_4_1420 repository.
|
||||
self.upload_file(
|
||||
repository,
|
||||
filename="tool_dependencies.xml",
|
||||
filepath=tool_dependency_path,
|
||||
valid_tools_only=True,
|
||||
uncompress_file=False,
|
||||
remove_repo_files_not_in_tar=False,
|
||||
commit_message="Populate package_lapack_3_4_1420 with tool dependency definitions.",
|
||||
strings_displayed=[],
|
||||
strings_not_displayed=[],
|
||||
)
|
||||
|
||||
def test_0010_create_atlas_repository(self):
|
||||
"""Create and populate package_atlas_3_10_1420.
|
||||
|
||||
This is step 1 - Create repository package_atlas_3_10_1420.
|
||||
|
||||
All tool dependency definitions should download and extract a tarball containing precompiled binaries from the local
|
||||
filesystem and install them into the path specified by $INSTALL_DIR.
|
||||
"""
|
||||
category = self.test_db_util.get_category_by_name(category_name)
|
||||
repository = self.get_or_create_repository(
|
||||
name=package_atlas_repository_name,
|
||||
description=package_atlas_repository_description,
|
||||
long_description=package_atlas_repository_long_description,
|
||||
owner=common.test_user_1_name,
|
||||
category_id=self.security.encode_id(category.id),
|
||||
strings_displayed=[],
|
||||
)
|
||||
# Load the original tool dependency definition into memory, then fill in the __PATH__ placeholder with the
|
||||
# actual system path where the binary tarball is found.
|
||||
tool_dependency_path = self.generate_temp_path(
|
||||
"1420_tool_dependency", additional_paths=["package_atlas_3_10_1420"]
|
||||
)
|
||||
precompiled_binary_tarball = self.get_filename("1420_files/binary_tarballs/atlas.tar")
|
||||
edited_tool_dependency_filename = self.get_filename(
|
||||
filepath=tool_dependency_path, filename="tool_dependencies.xml"
|
||||
)
|
||||
original_tool_dependency = self.get_filename("1420_files/package_atlas_3_10_1420/tool_dependencies.xml")
|
||||
tool_dependency_definition = (
|
||||
open(original_tool_dependency).read().replace("__PATH__", precompiled_binary_tarball)
|
||||
)
|
||||
open(edited_tool_dependency_filename, "w").write(tool_dependency_definition)
|
||||
# Upload the edited tool dependency definition to the package_atlas_3_10_1420 repository.
|
||||
self.upload_file(
|
||||
repository,
|
||||
filename="tool_dependencies.xml",
|
||||
filepath=tool_dependency_path,
|
||||
valid_tools_only=True,
|
||||
uncompress_file=False,
|
||||
remove_repo_files_not_in_tar=False,
|
||||
commit_message="Populate package_atlas_3_10_1420 with tool dependency definitions.",
|
||||
strings_displayed=[],
|
||||
strings_not_displayed=[],
|
||||
)
|
||||
|
||||
def test_0015_create_bzlib_repository(self):
|
||||
"""Create and populate package_bzlib_1_0_1420.
|
||||
|
||||
This is step 1 - Create repository package_bzlib_1_0_1420.
|
||||
|
||||
All tool dependency definitions should download and extract a tarball containing precompiled binaries from the local
|
||||
filesystem and install them into the path specified by $INSTALL_DIR.
|
||||
"""
|
||||
category = self.test_db_util.get_category_by_name(category_name)
|
||||
repository = self.get_or_create_repository(
|
||||
name=package_bzlib_repository_name,
|
||||
description=package_bzlib_repository_description,
|
||||
long_description=package_bzlib_repository_long_description,
|
||||
owner=common.test_user_1_name,
|
||||
category_id=self.security.encode_id(category.id),
|
||||
strings_displayed=[],
|
||||
)
|
||||
# Load the original tool dependency definition into memory, then fill in the __PATH__ placeholder with the
|
||||
# actual system path where the binary tarball is found.
|
||||
tool_dependency_path = self.generate_temp_path(
|
||||
"1420_tool_dependency", additional_paths=["package_bzlib_1_0_1420"]
|
||||
)
|
||||
precompiled_binary_tarball = self.get_filename("1420_files/binary_tarballs/bzlib.tar")
|
||||
edited_tool_dependency_filename = self.get_filename(
|
||||
filepath=tool_dependency_path, filename="tool_dependencies.xml"
|
||||
)
|
||||
original_tool_dependency = self.get_filename("1420_files/package_bzlib_1_0_1420/tool_dependencies.xml")
|
||||
tool_dependency_definition = (
|
||||
open(original_tool_dependency).read().replace("__PATH__", precompiled_binary_tarball)
|
||||
)
|
||||
open(edited_tool_dependency_filename, "w").write(tool_dependency_definition)
|
||||
# Upload the edited tool dependency definition to the package_bzlib_1_0_1420 repository.
|
||||
self.upload_file(
|
||||
repository,
|
||||
filename="tool_dependencies.xml",
|
||||
filepath=tool_dependency_path,
|
||||
valid_tools_only=True,
|
||||
uncompress_file=False,
|
||||
remove_repo_files_not_in_tar=False,
|
||||
commit_message="Populate package_bzlib_1_0_1420 with tool dependency definitions.",
|
||||
strings_displayed=[],
|
||||
strings_not_displayed=[],
|
||||
)
|
||||
|
||||
def test_0020_create_boost_repository(self):
|
||||
"""Create and populate package_boost_1_53_1420.
|
||||
|
||||
This is step 1 - Create repository package_boost_1_53_1420.
|
||||
|
||||
All tool dependency definitions should download and extract a tarball containing precompiled binaries from the local
|
||||
filesystem and install them into the path specified by $INSTALL_DIR.
|
||||
"""
|
||||
category = self.test_db_util.get_category_by_name(category_name)
|
||||
repository = self.get_or_create_repository(
|
||||
name=package_boost_repository_name,
|
||||
description=package_boost_repository_description,
|
||||
long_description=package_boost_repository_long_description,
|
||||
owner=common.test_user_1_name,
|
||||
category_id=self.security.encode_id(category.id),
|
||||
strings_displayed=[],
|
||||
)
|
||||
# Load the original tool dependency definition into memory, then fill in the __PATH__ placeholder with the
|
||||
# actual system path where the binary tarball is found.
|
||||
tool_dependency_path = self.generate_temp_path(
|
||||
"1420_tool_dependency", additional_paths=["package_boost_1_53_1420"]
|
||||
)
|
||||
precompiled_binary_tarball = self.get_filename("1420_files/binary_tarballs/boost.tar")
|
||||
edited_tool_dependency_filename = self.get_filename(
|
||||
filepath=tool_dependency_path, filename="tool_dependencies.xml"
|
||||
)
|
||||
original_tool_dependency = self.get_filename("1420_files/package_boost_1_53_1420/tool_dependencies.xml")
|
||||
tool_dependency_definition = (
|
||||
open(original_tool_dependency).read().replace("__PATH__", precompiled_binary_tarball)
|
||||
)
|
||||
open(edited_tool_dependency_filename, "w").write(tool_dependency_definition)
|
||||
# Upload the edited tool dependency definition to the package_boost_1_53_1420 repository.
|
||||
self.upload_file(
|
||||
repository,
|
||||
filename="tool_dependencies.xml",
|
||||
filepath=tool_dependency_path,
|
||||
valid_tools_only=True,
|
||||
uncompress_file=False,
|
||||
remove_repo_files_not_in_tar=False,
|
||||
commit_message="Populate package_boost_1_53_1420 with tool dependency definitions.",
|
||||
strings_displayed=[],
|
||||
strings_not_displayed=[],
|
||||
)
|
||||
|
||||
def test_0025_create_numpy_repository(self):
|
||||
"""Create and populate package_numpy_1_7_1420.
|
||||
|
||||
This is step 1 - Create repository package_numpy_1_7_1420.
|
||||
|
||||
All tool dependency definitions should download and extract a tarball containing precompiled binaries from the local
|
||||
filesystem and install them into the path specified by $INSTALL_DIR.
|
||||
"""
|
||||
category = self.test_db_util.get_category_by_name(category_name)
|
||||
repository = self.get_or_create_repository(
|
||||
name=package_numpy_repository_name,
|
||||
description=package_numpy_repository_description,
|
||||
long_description=package_numpy_repository_long_description,
|
||||
owner=common.test_user_1_name,
|
||||
category_id=self.security.encode_id(category.id),
|
||||
strings_displayed=[],
|
||||
)
|
||||
# Load the original tool dependency definition into memory, then fill in the __PATH__ placeholder with the
|
||||
# actual system path where the binary tarball is found.
|
||||
tool_dependency_path = self.generate_temp_path(
|
||||
"1420_tool_dependency", additional_paths=["package_numpy_1_7_1420"]
|
||||
)
|
||||
precompiled_binary_tarball = self.get_filename("1420_files/binary_tarballs/numpy.tar")
|
||||
edited_tool_dependency_filename = self.get_filename(
|
||||
filepath=tool_dependency_path, filename="tool_dependencies.xml"
|
||||
)
|
||||
original_tool_dependency = self.get_filename("1420_files/package_numpy_1_7_1420/tool_dependencies.xml")
|
||||
tool_dependency_definition = (
|
||||
open(original_tool_dependency).read().replace("__PATH__", precompiled_binary_tarball)
|
||||
)
|
||||
open(edited_tool_dependency_filename, "w").write(tool_dependency_definition)
|
||||
# Upload the edited tool dependency definition to the package_numpy_1_7_1420 repository.
|
||||
self.upload_file(
|
||||
repository,
|
||||
filename="tool_dependencies.xml",
|
||||
filepath=tool_dependency_path,
|
||||
valid_tools_only=True,
|
||||
uncompress_file=False,
|
||||
remove_repo_files_not_in_tar=False,
|
||||
commit_message="Populate package_numpy_1_7_1420 with tool dependency definitions.",
|
||||
strings_displayed=[],
|
||||
strings_not_displayed=[],
|
||||
)
|
||||
|
||||
def test_0030_create_rdkit_repository(self):
|
||||
"""Create and populate package_rdkit_2012_12_1420.
|
||||
|
||||
This is step 1 - Create repository package_rdkit_2012_12_1420.
|
||||
|
||||
All tool dependency definitions should download and extract a tarball containing precompiled binaries from the local
|
||||
filesystem and install them into the path specified by $INSTALL_DIR.
|
||||
"""
|
||||
category = self.test_db_util.get_category_by_name(category_name)
|
||||
repository = self.get_or_create_repository(
|
||||
name=package_rdkit_repository_name,
|
||||
description=package_rdkit_repository_description,
|
||||
long_description=package_rdkit_repository_long_description,
|
||||
owner=common.test_user_1_name,
|
||||
category_id=self.security.encode_id(category.id),
|
||||
strings_displayed=[],
|
||||
)
|
||||
# Load the original tool dependency definition into memory, then fill in the __PATH__ placeholder with the
|
||||
# actual system path where the binary tarball is found.
|
||||
tool_dependency_path = self.generate_temp_path(
|
||||
"1420_tool_dependency", additional_paths=["package_rdkit_2012_12_1420"]
|
||||
)
|
||||
precompiled_binary_tarball = self.get_filename("1420_files/binary_tarballs/rdkit.tar")
|
||||
edited_tool_dependency_filename = self.get_filename(
|
||||
filepath=tool_dependency_path, filename="tool_dependencies.xml"
|
||||
)
|
||||
original_tool_dependency = self.get_filename("1420_files/package_rdkit_2012_12_1420/tool_dependencies.xml")
|
||||
tool_dependency_definition = (
|
||||
open(original_tool_dependency).read().replace("__PATH__", precompiled_binary_tarball)
|
||||
)
|
||||
open(edited_tool_dependency_filename, "w").write(tool_dependency_definition)
|
||||
# Upload the edited tool dependency definition to the package_rdkit_2012_12_1420 repository.
|
||||
self.upload_file(
|
||||
repository,
|
||||
filename="tool_dependencies.xml",
|
||||
filepath=tool_dependency_path,
|
||||
valid_tools_only=True,
|
||||
uncompress_file=False,
|
||||
remove_repo_files_not_in_tar=False,
|
||||
commit_message="Populate package_rdkit_2012_12_1420 with tool dependency definitions.",
|
||||
strings_displayed=[],
|
||||
strings_not_displayed=[],
|
||||
)
|
||||
|
||||
def test_0035_install_rdkit_2012_12_repository(self):
|
||||
"""Install the package_rdkit_2012_12_1420 repository into Galaxy.
|
||||
|
||||
This is step 4 - Install package_rdkit_2012_12_1420 into Galaxy.
|
||||
|
||||
Install package_rdkit_2012_12_1420 with tool dependencies selected to be installed. The result of this should be
|
||||
package_atlas_3_10_1420, package_bzlib_1_0_1420, package_boost_1_53_1420, package_numpy_1_7_1420, package_rdkit_2012_12_1420,
|
||||
and package_lapack_3_4_1420 being installed, and an env.sh generated for package_rdkit_2012_12_1420 that
|
||||
contains environment variables defined in package_boost_1_53_1420 and package_numpy_1_7_1420.
|
||||
"""
|
||||
self.galaxy_login(email=common.admin_email, username=common.admin_username)
|
||||
post_submit_strings_displayed = [
|
||||
"package_rdkit_2012_12_1420",
|
||||
"package_atlas_3_10_1420",
|
||||
"package_bzlib_1_0_1420",
|
||||
"package_numpy_1_7_1420",
|
||||
"package_lapack_3_4_1420",
|
||||
"package_boost_1_53_1420",
|
||||
]
|
||||
self.install_repository(
|
||||
"package_rdkit_2012_12_1420",
|
||||
common.test_user_1_name,
|
||||
category_name,
|
||||
install_tool_dependencies=True,
|
||||
post_submit_strings_displayed=post_submit_strings_displayed,
|
||||
)
|
||||
|
||||
def test_0040_verify_env_sh_contents(self):
|
||||
"""Check the env.sh file for the appropriate contents.
|
||||
|
||||
This is step 5 - Verify that the env.sh file for package_rdkit_2012_12_1420 also defines the variables inherited from package_numpy_1_7_1420
|
||||
and package_boost_1_53_1420. Test for the numpy and boost tool dependency paths.
|
||||
"""
|
||||
package_rdkit_repository = self.test_db_util.get_installed_repository_by_name_owner(
|
||||
"package_rdkit_2012_12_1420", common.test_user_1_name
|
||||
)
|
||||
package_numpy_repository = self.test_db_util.get_installed_repository_by_name_owner(
|
||||
"package_numpy_1_7_1420", common.test_user_1_name
|
||||
)
|
||||
package_boost_repository = self.test_db_util.get_installed_repository_by_name_owner(
|
||||
"package_boost_1_53_1420", common.test_user_1_name
|
||||
)
|
||||
rdkit_env_sh = self.get_env_sh_path(
|
||||
tool_dependency_name="rdkit", tool_dependency_version="2012_12_1", repository=package_rdkit_repository
|
||||
)
|
||||
numpy_tool_dependency_path = self.get_tool_dependency_path(
|
||||
tool_dependency_name="numpy", tool_dependency_version="1.7.1", repository=package_numpy_repository
|
||||
)
|
||||
boost_tool_dependency_path = self.get_tool_dependency_path(
|
||||
tool_dependency_name="boost", tool_dependency_version="1.53.0", repository=package_boost_repository
|
||||
)
|
||||
rdkit_env_file_contents = open(rdkit_env_sh).read()
|
||||
if (
|
||||
numpy_tool_dependency_path not in rdkit_env_file_contents
|
||||
or boost_tool_dependency_path not in rdkit_env_file_contents
|
||||
):
|
||||
message = "Environment file for package_rdkit_2012_12_1420 does not contain expected path."
|
||||
message += f"\nExpected:\n{numpy_tool_dependency_path}\n{boost_tool_dependency_path}\nContents:\n{rdkit_env_file_contents}"
|
||||
raise AssertionError(message)
|
||||
@@ -1,146 +0,0 @@
|
||||
import logging
|
||||
|
||||
from ..base.twilltestcase import (
|
||||
common,
|
||||
ShedTwillTestCase,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
category_name = "Test 1440 Tool dependency missing env.sh"
|
||||
category_description = "Test script 1440 for detection of missing environment settings."
|
||||
package_repository_name = "package_env_sh_1_0_1440"
|
||||
tool_repository_name = "filter_1440"
|
||||
package_repository_description = "Repository that should result in an env.sh file, but does not."
|
||||
tool_repository_description = "Galaxy filtering tool."
|
||||
package_repository_long_description = f"{package_repository_name}: {package_repository_description}"
|
||||
tool_repository_long_description = f"{tool_repository_name}: {tool_repository_description}"
|
||||
|
||||
"""
|
||||
1. Create a tool dependency type repository that reliably fails to install successfully. This repository should define
|
||||
an action that would have created an env.sh file on success, resulting in an env.sh file that should exist, but is missing.
|
||||
|
||||
2. Create a repository that defines a complex repository dependency in the repository created in step 1, with prior_install_required
|
||||
and set_environment_for_install.
|
||||
|
||||
3. Attempt to install the second repository into a galaxy instance, verify that it is installed but missing tool dependencies.
|
||||
|
||||
"""
|
||||
|
||||
|
||||
class TestMissingEnvSh(ShedTwillTestCase):
|
||||
"""Test installing a repository that should create an env.sh file, but does not."""
|
||||
|
||||
def test_0000_initiate_users_and_category(self):
|
||||
"""Create necessary user accounts and login as an admin user."""
|
||||
self.login(email=common.admin_email, username=common.admin_username)
|
||||
admin_user = self.test_db_util.get_user(common.admin_email)
|
||||
assert admin_user is not None, f"Problem retrieving user with email {common.admin_email} from the database"
|
||||
self.test_db_util.get_private_role(admin_user)
|
||||
self.create_category(name=category_name, description=category_description)
|
||||
self.login(email=common.test_user_2_email, username=common.test_user_2_name)
|
||||
test_user_2 = self.test_db_util.get_user(common.test_user_2_email)
|
||||
assert (
|
||||
test_user_2 is not None
|
||||
), f"Problem retrieving user with email {common.test_user_2_email} from the database"
|
||||
self.test_db_util.get_private_role(test_user_2)
|
||||
self.login(email=common.test_user_1_email, username=common.test_user_1_name)
|
||||
test_user_1 = self.test_db_util.get_user(common.test_user_1_email)
|
||||
assert (
|
||||
test_user_1 is not None
|
||||
), f"Problem retrieving user with email {common.test_user_1_email} from the database"
|
||||
self.test_db_util.get_private_role(test_user_1)
|
||||
|
||||
def test_0005_create_package_repository(self):
|
||||
"""Create and populate package_env_sh_1_0_1440.
|
||||
|
||||
This is step 1 - Create repository package_env_sh_1_0_1440.
|
||||
|
||||
Create and populate a repository that is designed to fail a tool dependency installation. This tool dependency should
|
||||
also define one or more environment variables.
|
||||
"""
|
||||
category = self.test_db_util.get_category_by_name(category_name)
|
||||
repository = self.get_or_create_repository(
|
||||
name=package_repository_name,
|
||||
description=package_repository_description,
|
||||
long_description=package_repository_long_description,
|
||||
owner=common.test_user_1_name,
|
||||
category_id=self.security.encode_id(category.id),
|
||||
strings_displayed=[],
|
||||
)
|
||||
# Upload the edited tool dependency definition to the package_lapack_3_4_1440 repository.
|
||||
self.upload_file(
|
||||
repository,
|
||||
filename="1440_files/dependency_definition/tool_dependencies.xml",
|
||||
filepath=None,
|
||||
valid_tools_only=True,
|
||||
uncompress_file=False,
|
||||
remove_repo_files_not_in_tar=False,
|
||||
commit_message="Populate package_env_sh_1_0_1440 with a broken tool dependency definition.",
|
||||
strings_displayed=[],
|
||||
strings_not_displayed=[],
|
||||
)
|
||||
|
||||
def test_0010_create_filter_repository(self):
|
||||
"""Create and populate filter_1440.
|
||||
|
||||
This is step 2 - Create a repository that defines a complex repository dependency on the repository created in
|
||||
step 1, with prior_install_required and set_environment_for_install.
|
||||
"""
|
||||
category = self.test_db_util.get_category_by_name(category_name)
|
||||
repository = self.get_or_create_repository(
|
||||
name=tool_repository_name,
|
||||
description=tool_repository_description,
|
||||
long_description=tool_repository_long_description,
|
||||
owner=common.test_user_1_name,
|
||||
category_id=self.security.encode_id(category.id),
|
||||
strings_displayed=[],
|
||||
)
|
||||
# Upload the edited tool dependency definition to the package_lapack_3_4_1440 repository.
|
||||
self.upload_file(
|
||||
repository,
|
||||
filename="filtering/filtering_2.2.0.tar",
|
||||
filepath=None,
|
||||
valid_tools_only=True,
|
||||
uncompress_file=False,
|
||||
remove_repo_files_not_in_tar=False,
|
||||
commit_message="Populate filter_1440 with the filtering tool.",
|
||||
strings_displayed=[],
|
||||
strings_not_displayed=[],
|
||||
)
|
||||
self.upload_file(
|
||||
repository,
|
||||
filename="1440_files/complex_dependency/tool_dependencies.xml",
|
||||
filepath=None,
|
||||
valid_tools_only=True,
|
||||
uncompress_file=False,
|
||||
remove_repo_files_not_in_tar=False,
|
||||
commit_message="Populate filter_1440 with a dependency on package_env_sh_1_0_1440.",
|
||||
strings_displayed=[],
|
||||
strings_not_displayed=[],
|
||||
)
|
||||
|
||||
def test_0015_install_filter_repository(self):
|
||||
"""Install the filter_1440 repository to galaxy.
|
||||
|
||||
This is step 3 - Attempt to install the second repository into a galaxy instance, verify that it is installed but
|
||||
missing tool dependencies.
|
||||
"""
|
||||
self.galaxy_login(email=common.admin_email, username=common.admin_username)
|
||||
post_submit_strings_displayed = ["filter_1440", "package_env_sh_1_0_1440"]
|
||||
self.install_repository(
|
||||
"filter_1440",
|
||||
common.test_user_1_name,
|
||||
category_name,
|
||||
install_tool_dependencies=True,
|
||||
post_submit_strings_displayed=post_submit_strings_displayed,
|
||||
)
|
||||
|
||||
def test_0020_verify_missing_tool_dependency(self):
|
||||
"""Verify that the filter_1440 repository is installed and missing tool dependencies."""
|
||||
repository = self.test_db_util.get_installed_repository_by_name_owner("filter_1440", common.test_user_1_name)
|
||||
strings_displayed = ["Missing tool dependencies"]
|
||||
self.display_installed_repository_manage_page(repository, strings_displayed=strings_displayed)
|
||||
assert (
|
||||
len(repository.missing_tool_dependencies) == 1
|
||||
), "filter_1440 should have a missing tool dependency, but does not."
|
||||
@@ -15,7 +15,6 @@ import galaxy.tool_util.deps.requirements
|
||||
from galaxy import util
|
||||
from galaxy.tool_shed.util.shed_util_common import (
|
||||
can_eliminate_repository_dependency,
|
||||
can_eliminate_tool_dependency,
|
||||
clean_dependency_relationships,
|
||||
generate_tool_guid,
|
||||
get_ctx_rev,
|
||||
@@ -499,7 +498,6 @@ def open_repository_files_folder(app, folder_path, repository_id, is_admin=False
|
||||
|
||||
__all__ = (
|
||||
"can_eliminate_repository_dependency",
|
||||
"can_eliminate_tool_dependency",
|
||||
"clean_dependency_relationships",
|
||||
"count_repositories_in_category",
|
||||
"generate_tool_guid",
|
||||
|
||||
@@ -872,10 +872,6 @@ class RepositoriesController(BaseAPIController):
|
||||
)
|
||||
else:
|
||||
metadata_dict["repository_dependencies"] = []
|
||||
if metadata.includes_tool_dependencies and recursive:
|
||||
metadata_dict["tool_dependencies"] = repository.get_tool_dependencies(self.app, changehash)
|
||||
else:
|
||||
metadata_dict["tool_dependencies"] = {}
|
||||
if metadata.includes_tools:
|
||||
metadata_dict["tools"] = metadata.metadata["tools"]
|
||||
all_metadata[f"{int(changeset)}:{changehash}"] = metadata_dict
|
||||
|
||||
@@ -68,7 +68,7 @@
|
||||
</script>
|
||||
</%def>
|
||||
|
||||
<%def name="render_dependencies_section( install_resolver_dependencies_check_box, repository_dependencies_check_box, install_tool_dependencies_check_box, containers_dict, revision_label=None, export=False, requirements_status=None )">
|
||||
<%def name="render_dependencies_section( install_resolver_dependencies_check_box, repository_dependencies_check_box, containers_dict, revision_label=None, export=False, requirements_status=None )">
|
||||
<style type="text/css">
|
||||
#dependency_table{ table-layout:fixed;
|
||||
width:100%;
|
||||
@@ -186,20 +186,6 @@
|
||||
%endif
|
||||
%endif
|
||||
%if tool_dependencies_root_folder or missing_tool_dependencies_root_folder:
|
||||
%if install_tool_dependencies_check_box is not None:
|
||||
<div class="form-row">
|
||||
<label>When available, install Tool Shed managed tool dependencies?</label>
|
||||
<% disabled = trans.app.tool_dependency_dir is None %>
|
||||
${render_checkbox(install_tool_dependencies_check_box, disabled=disabled)}
|
||||
<div class="toolParamHelp" style="clear: both;">
|
||||
%if disabled:
|
||||
Set the tool_dependency_dir configuration value in your Galaxy config to automatically handle tool dependencies.
|
||||
%else:
|
||||
Select to automatically handle tool dependencies via Tool Shed.
|
||||
%endif
|
||||
</div>
|
||||
<div style="clear: both"></div>
|
||||
%endif
|
||||
%if tool_dependencies_root_folder:
|
||||
<div class="form-row">
|
||||
<p/>
|
||||
|
||||
+1
-1
@@ -127,7 +127,7 @@ or manually satisfy the dependencies listed below.</p>
|
||||
<th bgcolor="#EBD9B2">Confirm dependency installation</th>
|
||||
</table>
|
||||
</div>
|
||||
${render_dependencies_section( install_resolver_dependencies_check_box, install_repository_dependencies_check_box, install_tool_dependencies_check_box, containers_dict, revision_label=None, export=False, requirements_status=requirements_status )}
|
||||
${render_dependencies_section( install_resolver_dependencies_check_box, install_repository_dependencies_check_box, containers_dict, revision_label=None, export=False, requirements_status=requirements_status )}
|
||||
<div style="clear: both"></div>
|
||||
%endif
|
||||
%if shed_tool_conf_select_field:
|
||||
|
||||
@@ -68,8 +68,6 @@ PACKAGES = [
|
||||
"galaxy.tool_shed.galaxy_install.datatypes",
|
||||
"galaxy.tool_shed.galaxy_install.metadata",
|
||||
"galaxy.tool_shed.galaxy_install.repository_dependencies",
|
||||
"galaxy.tool_shed.galaxy_install.tool_dependencies",
|
||||
"galaxy.tool_shed.galaxy_install.tool_dependencies.recipe",
|
||||
"galaxy.tool_shed.galaxy_install.tools",
|
||||
"galaxy.tool_shed.util",
|
||||
"galaxy.tool_shed.metadata",
|
||||
|
||||
@@ -68,7 +68,7 @@
|
||||
</script>
|
||||
</%def>
|
||||
|
||||
<%def name="render_dependencies_section( install_resolver_dependencies_check_box, repository_dependencies_check_box, install_tool_dependencies_check_box, containers_dict, revision_label=None, export=False, requirements_status=None )">
|
||||
<%def name="render_dependencies_section( install_resolver_dependencies_check_box, repository_dependencies_check_box, containers_dict, revision_label=None, export=False, requirements_status=None )">
|
||||
<style type="text/css">
|
||||
#dependency_table{ table-layout:fixed;
|
||||
width:100%;
|
||||
@@ -186,20 +186,6 @@
|
||||
%endif
|
||||
%endif
|
||||
%if tool_dependencies_root_folder or missing_tool_dependencies_root_folder:
|
||||
%if install_tool_dependencies_check_box is not None:
|
||||
<div class="form-row">
|
||||
<label>When available, install Tool Shed managed tool dependencies?</label>
|
||||
<% disabled = trans.app.tool_dependency_dir is None %>
|
||||
${render_checkbox(install_tool_dependencies_check_box, disabled=disabled)}
|
||||
<div class="toolParamHelp" style="clear: both;">
|
||||
%if disabled:
|
||||
Set the tool_dependency_dir configuration value in your Galaxy config to automatically handle tool dependencies.
|
||||
%else:
|
||||
Select to automatically handle tool dependencies via Tool Shed.
|
||||
%endif
|
||||
</div>
|
||||
<div style="clear: both"></div>
|
||||
%endif
|
||||
%if tool_dependencies_root_folder:
|
||||
<div class="form-row">
|
||||
<p/>
|
||||
|
||||
@@ -28,21 +28,6 @@ ${render_galaxy_repository_actions( repository )}
|
||||
<input type="hidden" name="encoded_relative_install_dir" value="${encoded_relative_install_dir|h}"/>
|
||||
<input type="hidden" name="encoded_tool_dependencies_dict" value="${encoded_tool_dependencies_dict|h}"/>
|
||||
%if tool_dependencies_dict:
|
||||
%if install_tool_dependencies_check_box is not None:
|
||||
<div class="form-row">
|
||||
<label>Handle tool dependencies?</label>
|
||||
<% disabled = trans.app.tool_dependency_dir is None %>
|
||||
${render_checkbox(install_tool_dependencies_check_box, disabled=disabled)}
|
||||
<div class="toolParamHelp" style="clear: both;">
|
||||
%if disabled:
|
||||
Set the tool_dependency_dir configuration value in your Galaxy config to automatically handle tool dependencies.
|
||||
%else:
|
||||
Un-check to skip automatic handling of these tool dependencies.
|
||||
%endif
|
||||
</div>
|
||||
</div>
|
||||
<div style="clear: both"></div>
|
||||
%endif
|
||||
<div class="form-row">
|
||||
<table class="grid">
|
||||
<tr><td colspan="4" bgcolor="#D8D8D8"><b>New tool dependencies included in update</b></td></tr>
|
||||
|
||||
@@ -42,7 +42,7 @@
|
||||
<th bgcolor="#EBD9B2">Confirm dependency installation</th>
|
||||
</table>
|
||||
</div>
|
||||
${render_dependencies_section( install_resolver_dependencies_check_box, install_repository_dependencies_check_box, install_tool_dependencies_check_box, containers_dict, revision_label=None, export=False )}
|
||||
${render_dependencies_section( install_resolver_dependencies_check_box, install_repository_dependencies_check_box, containers_dict, revision_label=None, export=False )}
|
||||
%endif
|
||||
%if shed_tool_conf_select_field:
|
||||
<div class="form-row">
|
||||
|
||||
@@ -127,7 +127,7 @@ or manually satisfy the dependencies listed below.</p>
|
||||
<th bgcolor="#EBD9B2">Confirm dependency installation</th>
|
||||
</table>
|
||||
</div>
|
||||
${render_dependencies_section( install_resolver_dependencies_check_box, install_repository_dependencies_check_box, install_tool_dependencies_check_box, containers_dict, revision_label=None, export=False, requirements_status=requirements_status )}
|
||||
${render_dependencies_section( install_resolver_dependencies_check_box, install_repository_dependencies_check_box, containers_dict, revision_label=None, export=False, requirements_status=requirements_status )}
|
||||
<div style="clear: both"></div>
|
||||
%endif
|
||||
%if shed_tool_conf_select_field:
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
from contextlib import contextmanager
|
||||
|
||||
from galaxy.tool_shed.galaxy_install.tool_dependencies.recipe.env_file_builder import EnvFileBuilder
|
||||
|
||||
|
||||
def test_env_file_builder():
|
||||
install_dir = "/opt/galaxy/dependencies/foo/"
|
||||
env_file_builder = EnvFileBuilder(install_dir)
|
||||
added_lines = []
|
||||
mock_return = dict(value=0)
|
||||
|
||||
def mock_file_append(text, file_path, **kwds):
|
||||
added_lines.append(text)
|
||||
return mock_return["value"]
|
||||
|
||||
with __mock_env_file_builder_method(env_file_builder, "file_append", mock_file_append):
|
||||
env_file_builder.append_line(name="PATH", action="prepend_to", value="/usr/bin/local/R")
|
||||
assert added_lines == ["PATH=/usr/bin/local/R:$PATH; export PATH"]
|
||||
assert env_file_builder.return_code == 0
|
||||
|
||||
# Reset mock lines
|
||||
del added_lines[:]
|
||||
# Next time file_append will fail
|
||||
mock_return["value"] = 1
|
||||
|
||||
env_file_builder.append_line(action="source", value="/usr/bin/local/R/env.sh")
|
||||
assert added_lines == ["if [ -f /usr/bin/local/R/env.sh ] ; then . /usr/bin/local/R/env.sh ; fi"]
|
||||
# Check failure
|
||||
assert env_file_builder.return_code == 1
|
||||
|
||||
mock_return["value"] = 0
|
||||
env_file_builder.append_line(name="LD_LIBRARY_PATH", action="append_to", value="/usr/bin/local/R/lib")
|
||||
# Verify even though last append succeeded, previous failure still recorded.
|
||||
assert env_file_builder.return_code == 1
|
||||
|
||||
|
||||
# Poor man's mocking. Need to get a real mocking library as real Galaxy development
|
||||
# dependnecy.
|
||||
@contextmanager
|
||||
def __mock_env_file_builder_method(env_file_builder, name, mock_method):
|
||||
real_method = getattr(env_file_builder, name)
|
||||
try:
|
||||
setattr(env_file_builder, name, mock_method)
|
||||
yield
|
||||
finally:
|
||||
setattr(env_file_builder, name, real_method)
|
||||
@@ -1,99 +0,0 @@
|
||||
from contextlib import contextmanager
|
||||
from os.path import join
|
||||
from typing import (
|
||||
Any,
|
||||
Dict,
|
||||
)
|
||||
|
||||
from galaxy.tool_shed.galaxy_install.tool_dependencies.env_manager import EnvManager
|
||||
from galaxy.tool_shed.galaxy_install.tool_dependencies.recipe.env_file_builder import EnvFileBuilder
|
||||
from galaxy.util import parse_xml_string
|
||||
|
||||
TEST_DEPENDENCIES_DIR = "/opt/galaxy/dependencies"
|
||||
TEST_INSTALL_DIR = "%s/test_install_dir" % TEST_DEPENDENCIES_DIR
|
||||
|
||||
|
||||
class MockApp:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
|
||||
def test_create_or_update_env_shell_file():
|
||||
test_path = "/usr/share/R/libs"
|
||||
env_file_builder = EnvFileBuilder(test_path)
|
||||
line, path = env_file_builder.create_or_update_env_shell_file(
|
||||
TEST_INSTALL_DIR, dict(action="append_to", name="R_LIBS", value=test_path)
|
||||
)
|
||||
assert path == join(TEST_INSTALL_DIR, "env.sh")
|
||||
assert line == "R_LIBS=$R_LIBS:/usr/share/R/libs; export R_LIBS"
|
||||
|
||||
line, path = env_file_builder.create_or_update_env_shell_file(
|
||||
TEST_INSTALL_DIR, dict(action="prepend_to", name="R_LIBS", value=test_path)
|
||||
)
|
||||
assert path == join(TEST_INSTALL_DIR, "env.sh")
|
||||
assert line == "R_LIBS=/usr/share/R/libs:$R_LIBS; export R_LIBS"
|
||||
|
||||
line, path = env_file_builder.create_or_update_env_shell_file(
|
||||
TEST_INSTALL_DIR, dict(action="set_to", name="R_LIBS", value=test_path)
|
||||
)
|
||||
assert path == join(TEST_INSTALL_DIR, "env.sh")
|
||||
assert line == "R_LIBS=/usr/share/R/libs; export R_LIBS"
|
||||
|
||||
line, path = env_file_builder.create_or_update_env_shell_file(
|
||||
TEST_INSTALL_DIR, dict(action="source", value=test_path)
|
||||
)
|
||||
assert path == join(TEST_INSTALL_DIR, "env.sh")
|
||||
assert line == "if [ -f /usr/share/R/libs ] ; then . /usr/share/R/libs ; fi"
|
||||
|
||||
|
||||
def test_get_env_shell_file_paths_from_setup_environment_elem():
|
||||
xml = """<action name="setup_r_environment">
|
||||
<repository name="package_r_3_0_1" owner="bgruening" toolshed="toolshed.g2.bx.psu.edu" changeset_revision="1234567">
|
||||
<package name="R" version="3.0.1" />
|
||||
</repository>
|
||||
<repository name="package_zlib_1_2_8" owner="iuc" toolshed="toolshed.g2.bx.psu.edu" changeset_revision="7654321">
|
||||
<package name="zlib" version="1.2.8" />
|
||||
</repository>
|
||||
</action>
|
||||
"""
|
||||
mock_app = MockApp()
|
||||
action_elem = parse_xml_string(xml)
|
||||
required_for_install_env_sh = "/path/to/existing.sh"
|
||||
all_env_paths = [required_for_install_env_sh]
|
||||
action_dict: Dict[str, Any] = {}
|
||||
env_manager = EnvManager(mock_app)
|
||||
|
||||
r_env_sh = "/path/to/go/env.sh"
|
||||
|
||||
def mock_get_env_shell_file_paths(elem):
|
||||
assert elem.get("name") in ["package_r_3_0_1", "package_zlib_1_2_8"]
|
||||
return [r_env_sh]
|
||||
|
||||
with __mock_common_util_method(env_manager, "get_env_shell_file_paths", mock_get_env_shell_file_paths):
|
||||
env_manager.get_env_shell_file_paths_from_setup_environment_elem(all_env_paths, action_elem, action_dict)
|
||||
# Verify old env files weren't deleted.
|
||||
assert required_for_install_env_sh in all_env_paths
|
||||
# Verify new ones added.
|
||||
assert r_env_sh in all_env_paths
|
||||
# env_shell_file_paths includes everything
|
||||
assert all(env in action_dict["env_shell_file_paths"] for env in all_env_paths)
|
||||
# for every given repository there should be one env
|
||||
# file + the required_for_install_env_sh file
|
||||
assert len(action_dict["env_shell_file_paths"]) == 3
|
||||
|
||||
# action_shell_file_paths includes only env files defined
|
||||
# inside the setup_ action element.
|
||||
assert required_for_install_env_sh in action_dict["action_shell_file_paths"]
|
||||
assert r_env_sh in action_dict["action_shell_file_paths"]
|
||||
|
||||
|
||||
# Poor man's mocking. Need to get a real mocking library as real Galaxy development
|
||||
# dependnecy.
|
||||
@contextmanager
|
||||
def __mock_common_util_method(env_manager, name, mock_method):
|
||||
real_method = getattr(env_manager, name)
|
||||
try:
|
||||
setattr(env_manager, name, mock_method)
|
||||
yield
|
||||
finally:
|
||||
setattr(env_manager, name, real_method)
|
||||
Reference in New Issue
Block a user