From 5c807f6b6bcba23c7c7e7ecb0d5e5c81343e77ec Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 24 May 2022 14:47:30 -0400 Subject: [PATCH 1/5] Remove tool shed tool_dependencies.xml handling on the client side. --- .../galaxy_install/install_manager.py | 498 +--- .../tool_dependencies/__init__.py | 0 .../tool_dependencies/env_manager.py | 147 -- .../tool_dependencies/recipe/__init__.py | 0 .../recipe/asynchronous_reader.py | 33 - .../recipe/env_file_builder.py | 94 - .../recipe/install_environment.py | 288 --- .../recipe/recipe_manager.py | 136 -- .../tool_dependencies/recipe/step_handler.py | 2084 ----------------- .../tool_dependencies/recipe/tag_handler.py | 757 ------ .../tool_shed/util/tool_dependency_util.py | 54 - .../galaxy/controllers/admin_toolshed.py | 72 - lib/tool_shed/test/base/twilltestcase.py | 34 - ...stall_repository_with_tool_dependencies.py | 4 - ...repository_with_repository_dependencies.py | 4 - ...ll_repository_with_dependency_revisions.py | 4 - ...ll_repository_with_complex_dependencies.py | 46 - ...tool_dependency_environment_inheritance.py | 411 ---- .../test_1440_missing_env_sh_files.py | 146 -- lib/tool_shed/webapp/api/repositories.py | 4 - packages/app/setup.py | 2 - ...install_tool_dependencies_with_update.mako | 15 - test/unit/shed_unit/test_fabric_util.py | 46 - test/unit/shed_unit/test_td_common_util.py | 99 - 24 files changed, 1 insertion(+), 4977 deletions(-) delete mode 100644 lib/galaxy/tool_shed/galaxy_install/tool_dependencies/__init__.py delete mode 100644 lib/galaxy/tool_shed/galaxy_install/tool_dependencies/env_manager.py delete mode 100644 lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/__init__.py delete mode 100644 lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/asynchronous_reader.py delete mode 100644 lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/env_file_builder.py delete mode 100644 lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py delete mode 100644 lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/recipe_manager.py delete mode 100644 lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py delete mode 100644 lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py delete mode 100644 lib/tool_shed/test/functional/test_1420_tool_dependency_environment_inheritance.py delete mode 100644 lib/tool_shed/test/functional/test_1440_missing_env_sh_files.py delete mode 100644 test/unit/shed_unit/test_fabric_util.py delete mode 100644 test/unit/shed_unit/test_td_common_util.py diff --git a/lib/galaxy/tool_shed/galaxy_install/install_manager.py b/lib/galaxy/tool_shed/galaxy_install/install_manager.py index c91e4e34cb7..0b34531a41d 100644 --- a/lib/galaxy/tool_shed/galaxy_install/install_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/install_manager.py @@ -1,11 +1,7 @@ import json import logging import os -import sys -import tempfile -import traceback -from fabric.api import lcd from sqlalchemy import or_ from galaxy import ( @@ -17,498 +13,26 @@ from galaxy.tool_shed.galaxy_install.metadata.installed_repository_metadata_mana InstalledRepositoryMetadataManager, ) from galaxy.tool_shed.galaxy_install.repository_dependencies import repository_dependency_manager -from galaxy.tool_shed.galaxy_install.tool_dependencies.recipe.env_file_builder import EnvFileBuilder -from galaxy.tool_shed.galaxy_install.tool_dependencies.recipe.install_environment import InstallEnvironment -from galaxy.tool_shed.galaxy_install.tool_dependencies.recipe.recipe_manager import ( - StepManager, - TagManager, -) from galaxy.tool_shed.galaxy_install.tools import ( data_manager, tool_panel_manager, ) from galaxy.tool_shed.tools.data_table_manager import ShedToolDataTableManager from galaxy.tool_shed.util import ( - basic_util, hg_util, repository_util, ) from galaxy.tool_shed.util import shed_util_common as suc -from galaxy.tool_shed.util import ( - tool_dependency_util, - tool_util, -) +from galaxy.tool_shed.util import tool_util from galaxy.tool_util.deps import views from galaxy.util.tool_shed import ( common_util, encoding_util, - xml_util, ) log = logging.getLogger(__name__) -class InstallToolDependencyManager: - def __init__(self, app): - self.app = app - self.install_model = self.app.install_model - self.INSTALL_ACTIONS = [ - "download_binary", - "download_by_url", - "download_file", - "setup_perl_environment", - "setup_python_environment", - "setup_r_environment", - "setup_ruby_environment", - "shell_command", - ] - - def format_traceback(self): - ex_type, ex, tb = sys.exc_info() - return "".join(traceback.format_tb(tb)) - - def get_tool_shed_repository_install_dir(self, tool_shed_repository): - return os.path.abspath(tool_shed_repository.repo_files_directory(self.app)) - - def install_and_build_package(self, install_environment, tool_dependency, actions_dict): - """Install a Galaxy tool dependency package either via a url or a mercurial or git clone command.""" - install_dir = actions_dict["install_dir"] - package_name = actions_dict["package_name"] - actions = actions_dict.get("actions", None) - filtered_actions = [] - env_file_builder = EnvFileBuilder(install_dir) - step_manager = StepManager(self.app) - if actions: - with install_environment.use_tmp_dir() as work_dir: - with lcd(work_dir): - # The first action in the list of actions will be the one that defines the initial download process. - # There are currently three supported actions; download_binary, download_by_url and clone via a - # shell_command action type. The recipe steps will be filtered at this stage in the process, with - # the filtered actions being used in the next stage below. The installation directory (i.e., dir) - # is also defined in this stage and is used in the next stage below when defining current_dir. - action_type, action_dict = actions[0] - if action_type in self.INSTALL_ACTIONS: - # Some of the parameters passed here are needed only by a subset of the step handler classes, - # but to allow for a standard method signature we'll pass them along. We don't check the - # tool_dependency status in this stage because it should not have been changed based on a - # download. - tool_dependency, filtered_actions, dir = step_manager.execute_step( - tool_dependency=tool_dependency, - package_name=package_name, - actions=actions, - action_type=action_type, - action_dict=action_dict, - filtered_actions=filtered_actions, - env_file_builder=env_file_builder, - install_environment=install_environment, - work_dir=work_dir, - current_dir=None, - initial_download=True, - ) - else: - # We're handling a complex repository dependency where we only have a set_environment tag set. - # - # $INSTALL_DIR/bin - # - filtered_actions = [a for a in actions] - dir = install_dir - # We're in stage 2 of the installation process. The package has been down-loaded, so we can - # now perform all of the actions defined for building it. - for action_tup in filtered_actions: - if dir is None: - dir = "" - current_dir = os.path.abspath(os.path.join(work_dir, dir)) - with lcd(current_dir): - action_type, action_dict = action_tup - tool_dependency, tmp_filtered_actions, tmp_dir = step_manager.execute_step( - tool_dependency=tool_dependency, - package_name=package_name, - actions=actions, - action_type=action_type, - action_dict=action_dict, - filtered_actions=filtered_actions, - env_file_builder=env_file_builder, - install_environment=install_environment, - work_dir=work_dir, - current_dir=current_dir, - initial_download=False, - ) - if tool_dependency.status in [self.install_model.ToolDependency.installation_status.ERROR]: - # If the tool_dependency status is in an error state, return it with no additional - # processing. - return tool_dependency - # Make sure to handle the special case where the value of dir is reset (this happens when - # the action_type is change_directiory). In all other action types, dir will be returned as - # None. - if tmp_dir is not None: - dir = tmp_dir - return tool_dependency - - def install_and_build_package_via_fabric( - self, install_environment, tool_shed_repository, tool_dependency, actions_dict - ): - try: - # There is currently only one fabric method. - tool_dependency = self.install_and_build_package(install_environment, tool_dependency, actions_dict) - except Exception as e: - log.exception( - "Error installing tool dependency %s version %s.", tool_dependency.name, tool_dependency.version - ) - # Since there was an installation error, update the tool dependency status to Error. The remove_installation_path option must - # be left False here. - error_message = f"{self.format_traceback()}\n{util.unicodify(e)}" - tool_dependency = tool_dependency_util.set_tool_dependency_attributes( - self.app, - tool_dependency=tool_dependency, - status=self.app.install_model.ToolDependency.installation_status.ERROR, - error_message=error_message, - ) - tool_dependency = self.mark_tool_dependency_installed(tool_dependency) - return tool_dependency - - def install_specified_tool_dependencies(self, tool_shed_repository, tool_dependencies_config, tool_dependencies): - """ - Follow the recipe in the received tool_dependencies_config to install specified packages for - repository tools. The received list of tool_dependencies are the database records for those - dependencies defined in the tool_dependencies_config that are to be installed. This list may - be a subset of the set of dependencies defined in the tool_dependencies_config. This allows - for filtering out dependencies that have not been checked for installation on the 'Manage tool - dependencies' page for an installed Tool Shed repository. - """ - attr_tups_of_dependencies_for_install = [(td.name, td.version, td.type) for td in tool_dependencies] - installed_packages = [] - tag_manager = TagManager(self.app) - # Parse the tool_dependencies.xml config. - tree, error_message = xml_util.parse_xml(tool_dependencies_config) - if tree is None: - log.debug(f"The received tool_dependencies.xml file is likely invalid: {error_message}") - return installed_packages - root = tree.getroot() - elems = [] - for elem in root: - if elem.tag == "set_environment": - version = elem.get("version", "1.0") - if version != "1.0": - raise Exception("The tag must have a version attribute with value 1.0") - for sub_elem in elem: - elems.append(sub_elem) - else: - elems.append(elem) - for elem in elems: - name = elem.get("name", None) - version = elem.get("version", None) - type = elem.get("type", None) - if type is None: - if elem.tag in ["environment_variable", "set_environment"]: - type = "set_environment" - else: - type = "package" - if (name and type == "set_environment") or (name and version): - # elem is a package set_environment tag set. - attr_tup = (name, version, type) - try: - index = attr_tups_of_dependencies_for_install.index(attr_tup) - except ValueError: - index = None - if index is not None: - tool_dependency = tool_dependencies[index] - # If the tool_dependency.type is 'set_environment', then the call to process_tag_set() will - # handle everything - no additional installation is necessary. - tool_dependency, proceed_with_install, action_elem_tuples = tag_manager.process_tag_set( - tool_shed_repository, - tool_dependency, - elem, - name, - version, - tool_dependency_db_records=tool_dependencies, - ) - if tool_dependency.type == "package" and proceed_with_install: - try: - tool_dependency = self.install_package( - elem, tool_shed_repository, tool_dependencies=tool_dependencies - ) - except Exception as e: - error_message = f"Error installing tool dependency {name} version {version}: {e}" - log.exception(error_message) - if tool_dependency: - # Since there was an installation error, update the tool dependency status to Error. The - # remove_installation_path option must be left False here. - tool_dependency = tool_dependency_util.set_tool_dependency_attributes( - self.app, - tool_dependency=tool_dependency, - status=self.app.install_model.ToolDependency.installation_status.ERROR, - error_message=error_message, - ) - if tool_dependency and tool_dependency.status in [ - self.install_model.ToolDependency.installation_status.INSTALLED, - self.install_model.ToolDependency.installation_status.ERROR, - ]: - installed_packages.append(tool_dependency) - return installed_packages - - def install_via_fabric( - self, - tool_shed_repository, - tool_dependency, - install_dir, - package_name=None, - custom_fabfile_path=None, - actions_elem=None, - action_elem=None, - **kwd, - ): - """ - Parse a tool_dependency.xml file's tag set to gather information for installation using - self.install_and_build_package(). The use of fabric is being eliminated, so some of these functions - may need to be renamed at some point. - """ - if not os.path.exists(install_dir): - os.makedirs(install_dir) - actions_dict = dict(install_dir=install_dir) - if package_name: - actions_dict["package_name"] = package_name - actions = [] - is_binary_download = False - if actions_elem is not None: - elems = actions_elem - if elems.get("os") is not None and elems.get("architecture") is not None: - is_binary_download = True - elif action_elem is not None: - # We were provided with a single element to perform certain actions after a platform-specific tarball was downloaded. - elems = [action_elem] - else: - elems = [] - step_manager = StepManager(self.app) - tool_shed_repository_install_dir = self.get_tool_shed_repository_install_dir(tool_shed_repository) - install_environment = InstallEnvironment(self.app, tool_shed_repository_install_dir, install_dir) - for action_elem in elems: - # Make sure to skip all comments, since they are now included in the XML tree. - if action_elem.tag != "action": - continue - action_dict = {} - action_type = action_elem.get("type", None) - if action_type is not None: - action_dict = step_manager.prepare_step( - tool_dependency=tool_dependency, - action_type=action_type, - action_elem=action_elem, - action_dict=action_dict, - install_environment=install_environment, - is_binary_download=is_binary_download, - ) - action_tuple = (action_type, action_dict) - if action_type == "set_environment": - if action_tuple not in actions: - actions.append(action_tuple) - else: - actions.append(action_tuple) - if actions: - actions_dict["actions"] = actions - if custom_fabfile_path is not None: - # TODO: this is not yet supported or functional, but when it is handle it using the fabric api. - raise Exception("Tool dependency installation using proprietary fabric scripts is not yet supported.") - else: - tool_dependency = self.install_and_build_package_via_fabric( - install_environment, tool_shed_repository, tool_dependency, actions_dict - ) - return tool_dependency - - def install_package(self, elem, tool_shed_repository, tool_dependencies=None): - """ - Install a tool dependency package defined by the XML element elem. The value of tool_dependencies is - a partial or full list of ToolDependency records associated with the tool_shed_repository. - """ - tag_manager = TagManager(self.app) - # The value of package_name should match the value of the "package" type in the tool config's - # tag set, but it's not required. - package_name = elem.get("name", None) - package_version = elem.get("version", None) - if tool_dependencies and package_name and package_version: - tool_dependency = None - for tool_dependency in tool_dependencies: - if package_name == str(tool_dependency.name) and package_version == str(tool_dependency.version): - break - if tool_dependency is not None: - for package_elem in elem: - tool_dependency, proceed_with_install, actions_elem_tuples = tag_manager.process_tag_set( - tool_shed_repository, - tool_dependency, - package_elem, - package_name, - package_version, - tool_dependency_db_records=None, - ) - if proceed_with_install and actions_elem_tuples: - # Get the installation directory for tool dependencies that will be installed for the received - # tool_shed_repository. - install_dir = tool_dependency_util.get_tool_dependency_install_dir( - app=self.app, - repository_name=tool_shed_repository.name, - repository_owner=tool_shed_repository.owner, - repository_changeset_revision=tool_shed_repository.installed_changeset_revision, - tool_dependency_type="package", - tool_dependency_name=package_name, - tool_dependency_version=package_version, - ) - # At this point we have a list of elems that are either defined within an - # tag set with sub-elements that contains os and architecture attributes filtered by the - # platform into which the appropriate compiled binary will be installed, or not defined within an - # tag set and not filtered. Here is an example actions_elem_tuple. - # [(True, [)] - binary_installed = False - for actions_elem_tuple in actions_elem_tuples: - in_actions_group, actions_elems = actions_elem_tuple - if in_actions_group: - # Platform matching is only performed inside tag sets, os and architecture - # attributes are otherwise ignored. - can_install_from_source = False - for actions_elem in actions_elems: - system = actions_elem.get("os") - architecture = actions_elem.get("architecture") - # If this element has the os and architecture attributes defined, then we only - # want to process until a successful installation is achieved. - if system and architecture: - # If an tag has been defined that matches our current platform, and the - # recipe specified within that tag has been successfully processed, skip - # any remaining platform-specific tags. We cannot break out of the loop - # here because there may be tags at the end of the tag set - # that must be processed. - if binary_installed: - continue - # No platform-specific recipe has yet resulted in a successful installation. - tool_dependency = self.install_via_fabric( - tool_shed_repository, - tool_dependency, - install_dir, - package_name=package_name, - actions_elem=actions_elem, - action_elem=None, - ) - if ( - tool_dependency.status - == self.install_model.ToolDependency.installation_status.INSTALLED - ): - # If an tag was found that matches the current platform, and - # self.install_via_fabric() did not result in an error state, set binary_installed - # to True in order to skip any remaining platform-specific tags. - binary_installed = True - else: - # Process the next matching tag, or any defined tags that do not - # contain platform dependent recipes. - log.debug( - "Error downloading binary for tool dependency %s version %s: %s" - % ( - str(package_name), - str(package_version), - str(tool_dependency.error_message), - ) - ) - else: - if actions_elem.tag == "actions": - # We've reached an tag that defines the recipe for installing and compiling from - # source. If binary installation failed, we proceed with the recipe. - if not binary_installed: - installation_directory = tool_dependency.installation_directory( - self.app - ) - if os.path.exists(installation_directory): - # Delete contents of installation directory if attempt at binary installation failed. - installation_directory_contents = os.listdir(installation_directory) - if installation_directory_contents: - ( - removed, - error_message, - ) = tool_dependency_util.remove_tool_dependency( - self.app, tool_dependency - ) - if removed: - can_install_from_source = True - else: - log.debug( - "Error removing old files from installation directory %s: %s" - % (str(installation_directory, str(error_message))) - ) - else: - can_install_from_source = True - else: - can_install_from_source = True - if can_install_from_source: - # We now know that binary installation was not successful, so proceed with the - # tag set that defines the recipe to install and compile from source. - log.debug( - "Proceeding with install and compile recipe for tool dependency %s." - % str(tool_dependency.name) - ) - # Reset above error to installing - tool_dependency.status = ( - self.install_model.ToolDependency.installation_status.INSTALLING - ) - tool_dependency = self.install_via_fabric( - tool_shed_repository, - tool_dependency, - install_dir, - package_name=package_name, - actions_elem=actions_elem, - action_elem=None, - ) - if ( - actions_elem.tag == "action" - and tool_dependency.status - != self.install_model.ToolDependency.installation_status.ERROR - ): - # If the tool dependency is not in an error state, perform any final actions that have been - # defined within the actions_group tag set, but outside of an tag, which defines - # the recipe for installing and compiling from source. - tool_dependency = self.install_via_fabric( - tool_shed_repository, - tool_dependency, - install_dir, - package_name=package_name, - actions_elem=None, - action_elem=actions_elem, - ) - else: - # Checks for "os" and "architecture" attributes are not made for any tag sets outside of - # an tag set. If the attributes are defined, they will be ignored. All tags - # outside of an tag set will always be processed. - tool_dependency = self.install_via_fabric( - tool_shed_repository, - tool_dependency, - install_dir, - package_name=package_name, - actions_elem=actions_elems, - action_elem=None, - ) - if ( - tool_dependency.status - != self.install_model.ToolDependency.installation_status.ERROR - ): - log.debug( - "Tool dependency %s version %s has been installed in %s." - % (str(package_name), str(package_version), str(install_dir)) - ) - return tool_dependency - - def mark_tool_dependency_installed(self, tool_dependency): - if tool_dependency.status not in [ - self.install_model.ToolDependency.installation_status.ERROR, - self.install_model.ToolDependency.installation_status.INSTALLED, - ]: - log.debug( - "Changing status for tool dependency %s from %s to %s." - % ( - str(tool_dependency.name), - str(tool_dependency.status), - str(self.install_model.ToolDependency.installation_status.INSTALLED), - ) - ) - status = self.install_model.ToolDependency.installation_status.INSTALLED - tool_dependency = tool_dependency_util.set_tool_dependency_attributes( - self.app, tool_dependency=tool_dependency, status=status - ) - return tool_dependency - - class InstallRepositoryManager: def __init__(self, app, tpm=None): self.app = app @@ -623,10 +147,6 @@ class InstallRepositoryManager: tool_shed_repository.tool_shed_status = tool_shed_status_dict self.install_model.context.add(tool_shed_repository) self.install_model.context.flush() - if "tool_dependencies" in irmm_metadata_dict and not reinstalling: - tool_dependency_util.create_tool_dependency_objects( - self.app, tool_shed_repository, relative_install_dir, set_status=True - ) if "sample_files" in irmm_metadata_dict: sample_files = irmm_metadata_dict.get("sample_files", []) tool_index_sample_files = stdtm.get_tool_index_sample_files(sample_files) @@ -1130,22 +650,6 @@ class InstallRepositoryManager: if dependency_manager.cached: [dependency_manager.build_cache(r) for r in new_requirements] - if install_tool_dependencies and tool_shed_repository.tool_dependencies and "tool_dependencies" in metadata: - work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-itsr") - # Install tool dependencies. - self.update_tool_shed_repository_status( - tool_shed_repository, - self.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES, - ) - # Get the tool_dependencies.xml file from the repository. - tool_dependencies_config = hg_util.get_config_from_disk("tool_dependencies.xml", install_dir) - itdm = InstallToolDependencyManager(self.app) - itdm.install_specified_tool_dependencies( - tool_shed_repository=tool_shed_repository, - tool_dependencies_config=tool_dependencies_config, - tool_dependencies=tool_shed_repository.tool_dependencies, - ) - basic_util.remove_dir(work_dir) self.update_tool_shed_repository_status( tool_shed_repository, self.install_model.ToolShedRepository.installation_status.INSTALLED ) diff --git a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/__init__.py b/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/env_manager.py b/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/env_manager.py deleted file mode 100644 index a8bee1b8329..00000000000 --- a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/env_manager.py +++ /dev/null @@ -1,147 +0,0 @@ -import logging -import os - -from galaxy.tool_shed.util.repository_util import get_repository_for_dependency_relationship -from galaxy.util.tool_shed.common_util import remove_protocol_from_tool_shed_url - -log = logging.getLogger(__name__) - - -class EnvManager: - def __init__(self, app): - self.app = app - - def create_env_var_dict(self, elem, install_environment): - env_var_name = elem.get("name", "PATH") - env_var_action = elem.get("action", "prepend_to") - env_var_text = None - tool_dependency_install_dir = install_environment.install_dir - tool_shed_repository_install_dir = install_environment.tool_shed_repository_install_dir - if elem.text and elem.text.find("REPOSITORY_INSTALL_DIR") >= 0: - if tool_shed_repository_install_dir and elem.text.find("$REPOSITORY_INSTALL_DIR") != -1: - env_var_text = elem.text.replace("$REPOSITORY_INSTALL_DIR", tool_shed_repository_install_dir) - return dict(name=env_var_name, action=env_var_action, value=env_var_text) - else: - env_var_text = elem.text.replace("$REPOSITORY_INSTALL_DIR", tool_dependency_install_dir) - return dict(name=env_var_name, action=env_var_action, value=env_var_text) - if elem.text and elem.text.find("INSTALL_DIR") >= 0: - if tool_dependency_install_dir: - env_var_text = elem.text.replace("$INSTALL_DIR", tool_dependency_install_dir) - return dict(name=env_var_name, action=env_var_action, value=env_var_text) - else: - env_var_text = elem.text.replace("$INSTALL_DIR", tool_shed_repository_install_dir) - return dict(name=env_var_name, action=env_var_action, value=env_var_text) - # Allow for environment variables that contain neither REPOSITORY_INSTALL_DIR nor INSTALL_DIR - # since there may be command line parameters that are tuned for a Galaxy instance. Allowing them - # to be set in one location rather than being hard coded into each tool config is the best approach. - # For example: - # - # "--num_threads 4 --num_cpu_threads_per_data_thread 3 --phone_home STANDARD" - # - if elem.text is None: - # Also allow for an empty value for the environment variable, which - # may makes sense especially for the 'set_to' action - elem.text = "" - return dict(name=env_var_name, action=env_var_action, value=elem.text) - - def get_env_shell_file_path(self, installation_directory): - env_shell_file_name = "env.sh" - default_location = os.path.abspath(os.path.join(installation_directory, env_shell_file_name)) - if os.path.exists(default_location): - return default_location - for root, _dirs, files in os.walk(installation_directory): - for name in files: - if name == env_shell_file_name: - return os.path.abspath(os.path.join(root, name)) - return None - - def get_env_shell_file_paths(self, elem): - # Currently only the following tag set is supported. - # - # - # - env_shell_file_paths = [] - toolshed = elem.get("toolshed", None) - repository_name = elem.get("name", None) - repository_owner = elem.get("owner", None) - changeset_revision = elem.get("changeset_revision", None) - if toolshed and repository_name and repository_owner and changeset_revision: - # The protocol is not stored, but the port is if it exists. - toolshed = remove_protocol_from_tool_shed_url(toolshed) - repository = get_repository_for_dependency_relationship( - self.app, toolshed, repository_name, repository_owner, changeset_revision - ) - if repository: - for sub_elem in elem: - tool_dependency_type = sub_elem.tag - tool_dependency_name = sub_elem.get("name") - tool_dependency_version = sub_elem.get("version") - if tool_dependency_type and tool_dependency_name and tool_dependency_version: - # Get the tool_dependency so we can get its installation directory. - tool_dependency = None - for tool_dependency in repository.tool_dependencies: - if ( - tool_dependency.type == tool_dependency_type - and tool_dependency.name == tool_dependency_name - and tool_dependency.version == tool_dependency_version - ): - break - if tool_dependency: - installation_directory = tool_dependency.installation_directory(self.app) - env_shell_file_path = self.get_env_shell_file_path(installation_directory) - if env_shell_file_path: - env_shell_file_paths.append(env_shell_file_path) - else: - error_message = "Skipping tool dependency definition because unable to locate env.sh file for tool dependency " - error_message += f"type {tool_dependency_type}, name {tool_dependency_name}, version {tool_dependency_version} for repository {repository.name}" - log.debug(error_message) - continue - else: - error_message = ( - "Skipping tool dependency definition because unable to locate tool dependency " - ) - error_message += f"type {tool_dependency_type}, name {tool_dependency_name}, version {tool_dependency_version} for repository {repository.name}" - log.debug(error_message) - continue - else: - error_message = f"Skipping invalid tool dependency definition: type {tool_dependency_type}, name {tool_dependency_name}, version {tool_dependency_version}." - log.debug(error_message) - continue - else: - error_message = "Skipping set_environment_for_install definition because unable to locate required installed tool shed repository: " - error_message += f"toolshed {toolshed}, name {repository_name}, owner {repository_owner}, changeset_revision {changeset_revision}." - log.debug(error_message) - else: - error_message = ( - "Skipping invalid set_environment_for_install definition: toolshed %s, name %s, owner %s, changeset_revision %s." - % (str(toolshed), str(repository_name), str(repository_owner), str(changeset_revision)) - ) - log.debug(error_message) - return env_shell_file_paths - - def get_env_shell_file_paths_from_setup_environment_elem(self, all_env_shell_file_paths, elem, action_dict): - """ - Parse an XML tag set to discover all child repository dependency tags and define the path to an env.sh file associated - with the repository (this requires the repository dependency to be in an installed state). The received action_dict - will be updated with these discovered paths and returned to the caller. This method handles tool dependency definition - tag sets , , and . - """ - # An example elem is: - # - # - # - # - # - # - # - # http://search.cpan.org/CPAN/authors/id/T/TO/TODDR/XML-Parser-2.41.tar.gz - # http://search.cpan.org/CPAN/authors/id/L/LD/LDS/CGI.pm-3.43.tar.gz - # - for action_elem in elem: - if action_elem.tag == "repository": - env_shell_file_paths = self.get_env_shell_file_paths(action_elem) - all_env_shell_file_paths.extend(env_shell_file_paths) - if all_env_shell_file_paths: - action_dict["env_shell_file_paths"] = all_env_shell_file_paths - action_dict["action_shell_file_paths"] = all_env_shell_file_paths - return action_dict diff --git a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/__init__.py b/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/asynchronous_reader.py b/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/asynchronous_reader.py deleted file mode 100644 index 5e1d8c73127..00000000000 --- a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/asynchronous_reader.py +++ /dev/null @@ -1,33 +0,0 @@ -import logging -import threading - -from galaxy.util import unicodify - -log = logging.getLogger(__name__) - - -class AsynchronousReader(threading.Thread): - """ - A helper class to implement asynchronous reading of a stream in a separate thread. Read lines are pushed - onto a queue to be consumed in another thread. - """ - - def __init__(self, fd, queue): - threading.Thread.__init__(self) - self._fd = fd - self._queue = queue - self.lines = [] - - def run(self): - """Read lines and put them on the queue.""" - thread_lock = threading.Lock() - thread_lock.acquire() - for line in iter(self._fd.readline, b""): - stripped_line = unicodify(line).rstrip() - self.lines.append(stripped_line) - self._queue.put(stripped_line) - thread_lock.release() - - def installation_complete(self): - """Make sure there is more installation and compilation logging content expected.""" - return not self.is_alive() and self._queue.empty() diff --git a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/env_file_builder.py b/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/env_file_builder.py deleted file mode 100644 index f85cd23ecf5..00000000000 --- a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/env_file_builder.py +++ /dev/null @@ -1,94 +0,0 @@ -import logging -import os -import stat - -log = logging.getLogger(__name__) - - -class EnvFileBuilder: - def __init__(self, install_dir): - self.install_dir = install_dir - self.return_code = 0 - - def append_line(self, make_executable=True, **kwd): - env_var_dict = dict(**kwd) - env_entry, env_file = self.create_or_update_env_shell_file(self.install_dir, env_var_dict) - return_code = self.file_append(env_entry, env_file, make_executable=make_executable) - self.return_code = self.return_code or return_code - return self.return_code - - @staticmethod - def create_or_update_env_shell_file(install_dir, env_var_dict): - env_var_action = env_var_dict["action"] - env_var_value = env_var_dict["value"] - if env_var_action in ["prepend_to", "set_to", "append_to"]: - env_var_name = env_var_dict["name"] - if env_var_action == "prepend_to": - changed_value = f"{env_var_value}:${env_var_name}" - elif env_var_action == "set_to": - changed_value = f"{env_var_value}" - elif env_var_action == "append_to": - changed_value = f"${env_var_name}:{env_var_value}" - line = f"{env_var_name}={changed_value}; export {env_var_name}" - elif env_var_action == "source": - line = f"if [ -f {env_var_value} ] ; then . {env_var_value} ; fi" - else: - raise Exception(f"Unknown shell file action {env_var_action}") - env_shell_file_path = os.path.join(install_dir, "env.sh") - return line, env_shell_file_path - - def file_append(self, text, file_path, make_executable=True): - """ - Append a line to a file unless the line already exists in the file. This method creates the file if - it doesn't exist. If make_executable is True, the permissions on the file are set to executable by - the owner. - """ - file_dir = os.path.dirname(file_path) - if not os.path.exists(file_dir): - try: - os.makedirs(file_dir) - except Exception as e: - log.exception(str(e)) - return 1 - if os.path.exists(file_path): - try: - new_env_file_contents = [] - env_file_contents = open(file_path).readlines() - # Clean out blank lines from the env.sh file. - for line in env_file_contents: - line = line.rstrip() - if line: - new_env_file_contents.append(line) - env_file_contents = new_env_file_contents - except Exception as e: - log.exception(str(e)) - return 1 - else: - env_file_handle = open(file_path, "w") - env_file_handle.close() - env_file_contents = [] - if make_executable: - # Explicitly set the file's executable bits. - try: - os.chmod(file_path, os.stat(file_path).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) - except Exception as e: - log.exception(str(e)) - return 1 - # Convert the received text to a list, in order to support adding one or more lines to the file. - if isinstance(text, str): - text = [text] - for line in text: - line = line.rstrip() - if line and line not in env_file_contents: - env_file_contents.append(line) - try: - open(file_path, "w").write("\n".join(env_file_contents)) - except Exception as e: - log.exception(str(e)) - return 1 - return 0 - - def handle_action_shell_file_paths(self, action_dict): - shell_file_paths = action_dict.get("action_shell_file_paths", []) - for shell_file_path in shell_file_paths: - self.append_line(action="source", value=shell_file_path) diff --git a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py b/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py deleted file mode 100644 index 923bf1b3ed3..00000000000 --- a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py +++ /dev/null @@ -1,288 +0,0 @@ -import logging -import os -import queue -import shutil -import subprocess -import tempfile -import threading -import time -from contextlib import contextmanager - -# TODO: eliminate the use of fabric here. -from fabric import state -from fabric.operations import _AttributeString - -from galaxy.tool_shed.galaxy_install.tool_dependencies.recipe import asynchronous_reader -from galaxy.tool_shed.util.basic_util import ( - INSTALLATION_LOG, - NO_OUTPUT_TIMEOUT, -) -from galaxy.tool_shed.util.tool_dependency_util import set_tool_dependency_attributes -from galaxy.util import ( - DATABASE_MAX_STRING_SIZE, - DATABASE_MAX_STRING_SIZE_PRETTY, - shrink_string_by_size, - unicodify, -) - -log = logging.getLogger(__name__) - - -class InstallEnvironment: - """Object describing the environment built up as part of the process of building and installing a package.""" - - def __init__(self, app, tool_shed_repository_install_dir, install_dir): - """ - The value of the received tool_shed_repository_install_dir is the root installation directory - of the repository containing the tool dependency, and the value of the received install_dir is - the root installation directory of the tool dependency. - """ - self.app = app - self.env_shell_file_paths = [] - self.install_dir = install_dir - self.tool_shed_repository_install_dir = tool_shed_repository_install_dir - self.tmp_work_dir = os.path.abspath(tempfile.mkdtemp(prefix="tmp-toolshed-mtd")) - - def add_env_shell_file_paths(self, paths): - for path in paths: - self.env_shell_file_paths.append(str(path)) - - def build_command(self, command, action_type="shell_command"): - """ - Build command line for execution from simple command, but - configuring environment described by this object. - """ - env_cmds = self.environment_commands(action_type) - return "\n".join(env_cmds + [command]) - - def close_file_descriptor(self, fd): - """Attempt to close a file descriptor.""" - start_timer = time.time() - error = "" - while True: - try: - fd.close() - break - except OSError as e: - # Undoubtedly close() was called during a concurrent operation on the same file object. - log.debug(f"Error closing file descriptor: {e}") - time.sleep(0.5) - current_wait_time = time.time() - start_timer - if current_wait_time >= 600: - error = f"Error closing file descriptor: {e}" - break - return error - - def enqueue_output(self, stdout, stdout_queue, stderr, stderr_queue): - """ - This method places streamed stdout and stderr into a threaded IPC queue target. Received data - is printed and saved to that thread's queue. The calling thread can then retrieve the data using - thread.stdout and thread.stderr. - """ - stdout_logger = logging.getLogger("install_environment.STDOUT") - stderr_logger = logging.getLogger("install_environment.STDERR") - for line in iter(stdout.readline, b""): - output = unicodify(line).rstrip() - stdout_logger.debug(output) - stdout_queue.put(output) - stdout_queue.put(None) - for line in iter(stderr.readline, b""): - output = unicodify(line).rstrip() - stderr_logger.debug(output) - stderr_queue.put(output) - stderr_queue.put(None) - - def environment_commands(self, action_type): - """Build a list of commands used to construct the environment described by this object.""" - cmds = [] - for env_shell_file_path in self.env_shell_file_paths: - if os.path.exists(env_shell_file_path): - for env_setting in open(env_shell_file_path): - cmds.append(env_setting.strip("\n")) - else: - log.debug(f"Invalid file {env_shell_file_path} specified, ignoring {action_type} action.") - return cmds - - def environment_dict(self, action_type="template_command"): - env_vars = dict() - for env_shell_file_path in self.env_shell_file_paths: - if os.path.exists(env_shell_file_path): - for env_setting in open(env_shell_file_path): - env_string = env_setting.split(";")[0] - env_name, env_path = env_string.split("=") - env_vars[env_name] = env_path - else: - log.debug(f"Invalid file {env_shell_file_path} specified, ignoring template_command action.") - return env_vars - - def handle_command(self, tool_dependency, cmd, return_output=False, job_name=""): - """Handle a command and log the results.""" - command = str(cmd) - output = self.handle_complex_command(command, job_name=job_name) - self.log_results(cmd, output, os.path.join(self.install_dir, INSTALLATION_LOG)) - stdout = output.stdout - stderr = output.stderr - if len(stdout) > DATABASE_MAX_STRING_SIZE: - log.warning( - f"Length of stdout > {DATABASE_MAX_STRING_SIZE_PRETTY}, so only a portion will be saved in the database." - ) - stdout = shrink_string_by_size( - stdout, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True - ) - if len(stderr) > DATABASE_MAX_STRING_SIZE: - log.warning( - f"Length of stderr > {DATABASE_MAX_STRING_SIZE_PRETTY}, so only a portion will be saved in the database." - ) - stderr = shrink_string_by_size( - stderr, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True - ) - if output.return_code not in [0]: - status = self.app.install_model.ToolDependency.installation_status.ERROR - if stderr: - error_message = unicodify(stderr) - elif stdout: - error_message = unicodify(stdout) - else: - # We have a problem if there was no stdout and no stderr. - error_message = ( - f"Unknown error occurred executing shell command {cmd}, return_code: {output.return_code}" - ) - set_tool_dependency_attributes( - self.app, tool_dependency=tool_dependency, status=status, error_message=error_message - ) - if return_output: - return output - return output.return_code - - def handle_complex_command(self, command, job_name=""): - """ - Wrap subprocess.Popen in such a way that the stderr and stdout from running a shell command will - be captured and logged in nearly real time. This is similar to fabric.local, but allows us to - retain control over the process. This method is named "complex" because it uses queues and - threads to execute a command while capturing and displaying the output. - """ - # We define a "local logger" here such that we can give it a slightly - # different name. We use the package name as part of the logger to - # allow admins to easily distinguish between which package is currently - # being installed. - llog_name = __name__ - if len(job_name) > 0: - llog_name += f":{job_name}" - llog = logging.getLogger(llog_name) - # Print the command we're about to execute, ``set -x`` style. - llog.debug(f"+ {command}") - # Launch the command as subprocess. A bufsize of 1 means line buffered. - process_handle = subprocess.Popen( - str(command), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - bufsize=1, - close_fds=False, - shell=True, - cwd=state.env["lcwd"], - ) - pid = process_handle.pid - # Launch the asynchronous readers of the process' stdout and stderr. - stdout_queue = queue.Queue() - stdout_reader = asynchronous_reader.AsynchronousReader(process_handle.stdout, stdout_queue) - stdout_reader.start() - stderr_queue = queue.Queue() - stderr_reader = asynchronous_reader.AsynchronousReader(process_handle.stderr, stderr_queue) - stderr_reader.start() - # Place streamed stdout and stderr into a threaded IPC queue target so it can - # be printed and stored for later retrieval when generating the INSTALLATION.log. - stdio_thread = threading.Thread( - target=self.enqueue_output, args=(process_handle.stdout, stdout_queue, process_handle.stderr, stderr_queue) - ) - thread_lock = threading.Lock() - thread_lock.acquire() - stdio_thread.start() - # Check the queues for output until there is nothing more to get. - start_timer = time.time() - while not stdout_reader.installation_complete() or not stderr_reader.installation_complete(): - # Show what we received from standard output. - while not stdout_queue.empty(): - try: - line = stdout_queue.get() - except queue.Empty: - line = None - break - if line: - llog.debug(line) - start_timer = time.time() - else: - break - # Show what we received from standard error. - while not stderr_queue.empty(): - try: - line = stderr_queue.get() - except queue.Empty: - line = None - break - if line: - llog.debug(line) - start_timer = time.time() - else: - stderr_queue.task_done() - break - # Sleep a bit before asking the readers again. - time.sleep(0.1) - current_wait_time = time.time() - start_timer - if stdout_queue.empty() and stderr_queue.empty() and current_wait_time > NO_OUTPUT_TIMEOUT: - err_msg = ( - "\nShutting down process id %s because it generated no output for the defined timeout period of %.1f seconds.\n" - % (pid, NO_OUTPUT_TIMEOUT) - ) - stderr_reader.lines.append(err_msg) - process_handle.kill() - break - thread_lock.release() - # Wait until each of the threads we've started terminate. The following calls will block each thread - # until it terminates either normally, through an unhandled exception, or until the timeout occurs. - stdio_thread.join(NO_OUTPUT_TIMEOUT) - stdout_reader.join(NO_OUTPUT_TIMEOUT) - stderr_reader.join(NO_OUTPUT_TIMEOUT) - # Close subprocess' file descriptors. - self.close_file_descriptor(process_handle.stdout) - self.close_file_descriptor(process_handle.stderr) - stdout = "\n".join(stdout_reader.lines) - stderr = "\n".join(stderr_reader.lines) - # Handle error condition (deal with stdout being None, too) - output = _AttributeString(stdout.strip() if stdout else "") - errors = _AttributeString(stderr.strip() if stderr else "") - # Make sure the process has finished. - process_handle.poll() - output.return_code = process_handle.returncode - output.stderr = errors - return output - - def log_results(self, command, fabric_AttributeString, file_path): - """Write attributes of fabric.operations._AttributeString to a specified log file.""" - mode = "a" if os.path.exists(file_path) else "w" - with open(file_path, mode) as logfile: - logfile.write("\n#############################################\n") - logfile.write(command) - logfile.write("\nSTDOUT\n") - logfile.write(fabric_AttributeString.stdout) - logfile.write("\n#############################################\n") - logfile.write("\n#############################################\n") - logfile.write(command) - logfile.write("\nSTDERR\n") - logfile.write(fabric_AttributeString.stderr) - logfile.write("\n#############################################\n") - - @contextmanager - def use_tmp_dir(self): - work_dir = self.tmp_work_dir - yield work_dir - if os.path.exists(work_dir): - try: - shutil.rmtree(work_dir) - except Exception as e: - log.exception(str(e)) - - def __setup_environment(self): - return "&&".join(f". {_}" for _ in self.__valid_env_shell_file_paths()) - - def __valid_env_shell_file_paths(self): - return [_ for _ in self.env_shell_file_paths if os.path.exists(_)] diff --git a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/recipe_manager.py b/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/recipe_manager.py deleted file mode 100644 index 40d2a708418..00000000000 --- a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/recipe_manager.py +++ /dev/null @@ -1,136 +0,0 @@ -import logging - -from galaxy.tool_shed.galaxy_install.tool_dependencies.recipe import ( - step_handler, - tag_handler, -) - -log = logging.getLogger(__name__) - - -class StepManager: - def __init__(self, app): - self.app = app - self.step_handlers_by_type = self.load_step_handlers() - - def get_step_handler_by_type(self, type): - return self.step_handlers_by_type.get(type, None) - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_type, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - if actions: - step_handler = self.get_step_handler_by_type(action_type) - tool_dependency, filtered_actions, dir = step_handler.execute_step( - tool_dependency=tool_dependency, - package_name=package_name, - actions=actions, - action_dict=action_dict, - filtered_actions=filtered_actions, - env_file_builder=env_file_builder, - install_environment=install_environment, - work_dir=work_dir, - current_dir=current_dir, - initial_download=initial_download, - ) - else: - dir = None - return tool_dependency, filtered_actions, dir - - def load_step_handlers(self): - step_handlers_by_type = dict( - assert_directory_executable=step_handler.AssertDirectoryExecutable(self.app), - assert_directory_exists=step_handler.AssertDirectoryExists(self.app), - assert_file_executable=step_handler.AssertFileExecutable(self.app), - assert_file_exists=step_handler.AssertFileExists(self.app), - autoconf=step_handler.Autoconf(self.app), - change_directory=step_handler.ChangeDirectory(self.app), - chmod=step_handler.Chmod(self.app), - download_binary=step_handler.DownloadBinary(self.app), - download_by_url=step_handler.DownloadByUrl(self.app), - download_file=step_handler.DownloadFile(self.app), - make_directory=step_handler.MakeDirectory(self.app), - make_install=step_handler.MakeInstall(self.app), - move_directory_files=step_handler.MoveDirectoryFiles(self.app), - move_file=step_handler.MoveFile(self.app), - regex_replace=step_handler.RegexReplace(self.app), - set_environment=step_handler.SetEnvironment(self.app), - set_environment_for_install=step_handler.SetEnvironmentForInstall(self.app), - setup_perl_environment=step_handler.SetupPerlEnvironment(self.app), - setup_r_environment=step_handler.SetupREnvironment(self.app), - setup_ruby_environment=step_handler.SetupRubyEnvironment(self.app), - setup_python_environment=step_handler.SetupPythonEnvironment(self.app), - setup_virtualenv=step_handler.SetupVirtualEnv(self.app), - shell_command=step_handler.ShellCommand(self.app), - template_command=step_handler.TemplateCommand(self.app), - ) - return step_handlers_by_type - - def prepare_step( - self, tool_dependency, action_type, action_elem, action_dict, install_environment, is_binary_download - ): - """ - Prepare the recipe step for later execution. This generally alters the received action_dict - with new information needed during this step's execution. - """ - if action_elem is not None: - step_handler = self.get_step_handler_by_type(action_type) - action_dict = step_handler.prepare_step( - tool_dependency=tool_dependency, - action_elem=action_elem, - action_dict=action_dict, - install_environment=install_environment, - is_binary_download=is_binary_download, - ) - return action_dict - - -class TagManager: - def __init__(self, app): - self.app = app - self.tag_handlers = self.load_tag_handlers() - - def get_tag_handler_by_tag(self, tag): - return self.tag_handlers.get(tag, None) - - def process_tag_set( - self, - tool_shed_repository, - tool_dependency, - package_elem, - package_name, - package_version, - tool_dependency_db_records=None, - ): - tag_handler = self.get_tag_handler_by_tag(package_elem.tag) - tool_dependency, proceed_with_install, action_elem_tuples = tag_handler.process_tag_set( - tool_shed_repository, - tool_dependency, - package_elem, - package_name, - package_version, - tool_dependency_db_records=tool_dependency_db_records, - ) - return tool_dependency, proceed_with_install, action_elem_tuples - - def load_tag_handlers(self): - tag_handlers = dict( - environment_variable=tag_handler.SetEnvironment(self.app), - install=tag_handler.Install(self.app), - package=tag_handler.Package(self.app), - readme=tag_handler.ReadMe(self.app), - repository=tag_handler.Repository(self.app), - set_environment=tag_handler.SetEnvironment(self.app), - ) - return tag_handlers diff --git a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py b/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py deleted file mode 100644 index 73c4f04dc3d..00000000000 --- a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ /dev/null @@ -1,2084 +0,0 @@ -import hashlib -import logging -import os -import re -import shutil -import stat -import tarfile -import tempfile -import zipfile -from string import Template - -# TODO: eliminate the use of fabric here. -from fabric.api import ( - lcd, - settings, -) - -from galaxy.tool_shed.galaxy_install.tool_dependencies.env_manager import EnvManager -from galaxy.tool_shed.util import ( - basic_util, - tool_dependency_util, -) -from galaxy.util import ( - asbool, - download_to_file, -) -from galaxy.util.compression_utils import CompressedFile -from galaxy.util.template import fill_template - -log = logging.getLogger(__name__) - -VIRTUALENV_URL = "https://pypi.python.org/packages/d4/0c/9840c08189e030873387a73b90ada981885010dd9aea134d6de30cd24cb8/virtualenv-15.1.0.tar.gz" - - -class Download: - def url_download(self, install_dir, downloaded_file_name, download_url, extract=True, checksums=None): - """ - The given download_url can have an extension like #md5#, #sha256#, (or #md5= to support pypi defaults). - - https://pypi.python.org/packages/source/k/khmer/khmer-1.0.tar.gz#md5#b60639a8b2939836f66495b9a88df757 - - Alternatively, to not break HTTP spec, you can specify md5 and - sha256 as keys in the element. - - This indicates a checksum which will be checked after download. - If the checksum does not match an exception is thrown. - """ - checksums = checksums or {} - file_path = os.path.join(install_dir, downloaded_file_name) - if download_url.startswith("file://"): - local_file_source = download_url[len("file://") :].split("#")[0] - shutil.copyfile(local_file_source, file_path) - else: - try: - download_to_file(download_url, file_path, chunk_size=basic_util.CHUNK_SIZE) - except Exception as e: - err_msg = f"Error downloading from URL {download_url} : {e}" - raise Exception(err_msg) - - if "sha256sum" in checksums or "#sha256#" in download_url: - downloaded_checksum = hashlib.sha256(open(file_path, "rb").read()).hexdigest().lower() - - # Determine expected value - if "sha256sum" in checksums: - expected = checksums["sha256sum"].lower() - else: - expected = download_url.split("#sha256#")[1].lower() - - if downloaded_checksum != expected: - raise Exception( - f"Given sha256 checksum does not match with the one from the downloaded file ({downloaded_checksum} != {expected})." - ) - - if "md5sum" in checksums or "#md5#" in download_url or "#md5=" in download_url: - downloaded_checksum = hashlib.md5(open(file_path, "rb").read()).hexdigest().lower() - - # Determine expected value - if "md5sum" in checksums: - expected = checksums["md5sum"].lower() - else: - expected = re.split("#md5[#=]", download_url)[1].lower() - - if downloaded_checksum != expected: - raise Exception( - f"Given md5 checksum does not match with the one from the downloaded file ({downloaded_checksum} != {expected})." - ) - - if extract: - if tarfile.is_tarfile(file_path) or (zipfile.is_zipfile(file_path) and not file_path.endswith(".jar")): - archive = CompressedFile(file_path) - extraction_path = archive.extract(install_dir) - else: - extraction_path = os.path.abspath(install_dir) - else: - extraction_path = os.path.abspath(install_dir) - - return extraction_path - - def get_elem_checksums(self, elem): - rval = {} - for hash_type in ("md5sum", "sha256sum"): - if hash_type in elem.keys(): - rval[hash_type] = elem.get(hash_type).lower() - return rval - - def get_dict_checksums(self, dct): - return dict(i for i in dct.items() if i[0] in ["md5sum", "sha256sum"]) - - -class RecipeStep: - """Abstract class that defines a standard format for handling recipe steps when installing packages.""" - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - raise Exception("Unimplemented Method") - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - raise Exception("Unimplemented Method") - - -class AssertDirectoryExecutable(RecipeStep): - def __init__(self, app): - self.app = app - self.type = "assert_directory_executable" - - def assert_directory_executable(self, full_path): - """ - Return True if a symbolic link or directory exists and is executable, but if - full_path is a file, return False. - """ - if full_path is None: - return False - if os.path.isfile(full_path): - return False - if os.path.isdir(full_path): - # Make sure the owner has execute permission on the directory. - # See https://docs.python.org/2/library/stat.html - if stat.S_IXUSR & os.stat(full_path)[stat.ST_MODE] == 64: - return True - return False - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - """ - Make sure a symbolic link or directory on disk exists and is executable, but is not a file. - Since this class is not used in the initial download stage, no recipe step filtering is - performed here, and None values are always returned for filtered_actions and dir. - """ - if os.path.isabs(action_dict["full_path"]): - full_path = action_dict["full_path"] - else: - full_path = os.path.join(current_dir, action_dict["full_path"]) - if not self.assert_directory_executable(full_path=full_path): - status = self.app.install_model.ToolDependency.installation_status.ERROR - error_message = f"The path {full_path} is not a directory or is not executable by the owner." - tool_dependency = tool_dependency_util.set_tool_dependency_attributes( - self.app, tool_dependency, status=status, error_message=error_message - ) - return tool_dependency, None, None - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - # $INSTALL_DIR/mira/my_file - if action_elem.text: - action_dict["full_path"] = basic_util.evaluate_template(action_elem.text, install_environment) - return action_dict - - -class AssertDirectoryExists(RecipeStep): - def __init__(self, app): - self.app = app - self.type = "assert_directory_exists" - - def assert_directory_exists(self, full_path): - """ - Return True if a symbolic link or directory exists, but if full_path is a file, - return False. - """ - if full_path is None: - return False - if os.path.isfile(full_path): - return False - if os.path.isdir(full_path): - return True - return False - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - """ - Make sure a a symbolic link or directory on disk exists, but is not a file. Since this - class is not used in the initial download stage, no recipe step filtering is performed - here, and None values are always returned for filtered_actions and dir. - """ - if os.path.isabs(action_dict["full_path"]): - full_path = action_dict["full_path"] - else: - full_path = os.path.join(current_dir, action_dict["full_path"]) - if not self.assert_directory_exists(full_path=full_path): - status = self.app.install_model.ToolDependency.installation_status.ERROR - error_message = f"The path {full_path} is not a directory or does not exist." - tool_dependency = tool_dependency_util.set_tool_dependency_attributes( - self.app, tool_dependency, status=status, error_message=error_message - ) - return tool_dependency, None, None - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - # $INSTALL_DIR/mira - if action_elem.text: - action_dict["full_path"] = basic_util.evaluate_template(action_elem.text, install_environment) - return action_dict - - -class AssertFileExecutable(RecipeStep): - def __init__(self, app): - self.app = app - self.type = "assert_file_executable" - - def assert_file_executable(self, full_path): - """ - Return True if a symbolic link or file exists and is executable, but if full_path - is a directory, return False. - """ - if full_path is None: - return False - if os.path.isdir(full_path): - return False - if os.path.exists(full_path): - # Make sure the owner has execute permission on the file. - # See https://docs.python.org/2/library/stat.html - if stat.S_IXUSR & os.stat(full_path)[stat.ST_MODE] == 64: - return True - return False - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - """ - Make sure a symbolic link or file on disk exists and is executable, but is not a directory. - Since this class is not used in the initial download stage, no recipe step filtering is - performed here, and None values are always returned for filtered_actions and dir. - """ - if os.path.isabs(action_dict["full_path"]): - full_path = action_dict["full_path"] - else: - full_path = os.path.join(current_dir, action_dict["full_path"]) - if not self.assert_file_executable(full_path=full_path): - status = self.app.install_model.ToolDependency.installation_status.ERROR - error_message = f"The path {full_path} is not a file or is not executable by the owner." - tool_dependency = tool_dependency_util.set_tool_dependency_attributes( - self.app, tool_dependency, status=status, error_message=error_message - ) - return tool_dependency, None, None - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - # $INSTALL_DIR/mira/my_file - if action_elem.text: - action_dict["full_path"] = basic_util.evaluate_template(action_elem.text, install_environment) - return action_dict - - -class AssertFileExists(RecipeStep): - def __init__(self, app): - self.app = app - self.type = "assert_file_exists" - - def assert_file_exists(self, full_path): - """ - Return True if a symbolic link or file exists, but if full_path is a directory, - return False. - """ - if full_path is None: - return False - if os.path.isdir(full_path): - return False - if os.path.exists(full_path): - return True - return False - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - """ - Make sure a symbolic link or file on disk exists, but is not a directory. Since this - class is not used in the initial download stage, no recipe step filtering is performed - here, and None values are always returned for filtered_actions and dir. - """ - if os.path.isabs(action_dict["full_path"]): - full_path = action_dict["full_path"] - else: - full_path = os.path.join(current_dir, action_dict["full_path"]) - if not self.assert_file_exists(full_path=full_path): - status = self.app.install_model.ToolDependency.installation_status.ERROR - error_message = f"The path {full_path} is not a file or does not exist." - tool_dependency = tool_dependency_util.set_tool_dependency_attributes( - self.app, tool_dependency, status=status, error_message=error_message - ) - return tool_dependency, None, None - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - # $INSTALL_DIR/mira/my_file - if action_elem.text: - action_dict["full_path"] = basic_util.evaluate_template(action_elem.text, install_environment) - return action_dict - - -class Autoconf(RecipeStep): - def __init__(self, app): - self.app = app - self.type = "autoconf" - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - """ - Handle configure, make and make install in a shell, allowing for configuration options. Since this - class is not used in the initial download stage, no recipe step filtering is performed here, and None - values are always returned for filtered_actions and dir. - """ - with settings(warn_only=True): - configure_opts = action_dict.get("configure_opts", "") - if "prefix=" in configure_opts: - pre_cmd = f"./configure {configure_opts} && make && make install" - else: - pre_cmd = f"./configure --prefix=$INSTALL_DIR {configure_opts} && make && make install" - cmd = install_environment.build_command(basic_util.evaluate_template(pre_cmd, install_environment)) - install_environment.handle_command( - tool_dependency=tool_dependency, cmd=cmd, return_output=False, job_name=package_name - ) - # The caller should check the status of the returned tool_dependency since this function - # does nothing with the return_code. - return tool_dependency, None, None - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - # Handle configure, make and make install allow providing configuration options - if action_elem.text: - configure_opts = basic_util.evaluate_template(action_elem.text, install_environment) - action_dict["configure_opts"] = configure_opts - return action_dict - - -class ChangeDirectory(RecipeStep): - def __init__(self, app): - self.app = app - self.type = "change_directory" - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - """ - Change the working directory in a shell. Since this class is not used in the initial download stage, - no recipe step filtering is performed here and a None value is return for filtered_actions. However, - the new dir value is returned since it is needed for later steps. - """ - - def dir_valid(test_dir): - """ - Make sure the defined directory is within current_dir or work_dir. - """ - for valid_dir in [os.path.realpath(current_dir), os.path.realpath(work_dir)]: - if test_dir.startswith(valid_dir) and os.path.exists(test_dir): - return True - return False - - target_dir = os.path.realpath(os.path.normpath(action_dict["directory"])) - if dir_valid(target_dir): - # We have a directory that includes a path. - dir = target_dir - else: - target_dir = os.path.realpath(os.path.normpath(os.path.join(current_dir, action_dict["directory"]))) - if dir_valid(target_dir): - dir = target_dir - else: - log.debug( - "Invalid or nonexistent directory %s specified, ignoring change_directory action.", - str(action_dict["directory"]), - ) - dir = current_dir - return tool_dependency, None, dir - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - # PHYLIP-3.6b - if action_elem.text: - action_dict["directory"] = basic_util.evaluate_template(action_elem.text, install_environment) - return action_dict - - -class Chmod(RecipeStep): - def __init__(self, app): - self.app = app - self.type = "chmod" - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - """ - Change the mode setting for certain files in the installation environment. Since this class is not - used in the initial download stage, no recipe step filtering is performed here, and None values are - always returned for filtered_actions and dir. - """ - for target_file, mode in action_dict["change_modes"]: - if os.path.exists(target_file): - os.chmod(target_file, mode) - else: - log.debug("Invalid file %s specified, ignoring %s action.", target_file, self.type) - return tool_dependency, None, None - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - # Change the read, write, and execute bits on a file. - # - # $INSTALL_DIR/bin/faToTwoBit - # - file_elems = action_elem.findall("file") - chmod_actions = [] - # A unix octal mode is the sum of the following values: - # Owner: - # 400 Read 200 Write 100 Execute - # Group: - # 040 Read 020 Write 010 Execute - # World: - # 004 Read 002 Write 001 Execute - for file_elem in file_elems: - # So by the above table, owner read/write/execute and group read permission would be 740. - # Python's os.chmod uses base 10 modes, convert received unix-style octal modes to base 10. - received_mode = int(file_elem.get("mode", 600), base=8) - # For added security, ensure that the setuid and setgid bits are not set. - mode = received_mode & ~(stat.S_ISUID | stat.S_ISGID) - file = basic_util.evaluate_template(file_elem.text, install_environment) - chmod_tuple = (file, mode) - chmod_actions.append(chmod_tuple) - if chmod_actions: - action_dict["change_modes"] = chmod_actions - return action_dict - - -class DownloadBinary(Download, RecipeStep): - def __init__(self, app): - self.app = app - self.type = "download_binary" - - def download_binary(self, url, work_dir, checksums): - """Download a pre-compiled binary from the specified URL.""" - downloaded_filename = os.path.split(url)[-1] - self.url_download(work_dir, downloaded_filename, url, extract=False, checksums=checksums) - return downloaded_filename - - def filter_actions_after_binary_installation(self, actions): - """Filter out actions that should not be processed if a binary download succeeded.""" - filtered_actions = [] - for action in actions: - action_type, action_dict = action - if action_type in ["set_environment", "chmod", "download_binary"]: - filtered_actions.append(action) - return filtered_actions - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - """ - Download a binary file. If the value of initial_download is True, the recipe steps will be - filtered and returned and the installation directory (i.e., dir) will be defined and returned. - If we're not in the initial download stage, these actions will not occur, and None values will - be returned for them. - """ - url = action_dict["url"] - # Get the target directory for this download if the user has specified one. Default to the root of $INSTALL_DIR. - target_directory = action_dict.get("target_directory", None) - # Attempt to download a binary from the specified URL. - downloaded_filename = None - try: - checksums = self.get_dict_checksums(action_dict) - log.debug("Attempting to download from %s to %s", url, str(target_directory)) - downloaded_filename = self.download_binary(url, work_dir, checksums) - if initial_download: - # Filter out any actions that are not download_binary, chmod, or set_environment. - filtered_actions = self.filter_actions_after_binary_installation(actions[1:]) - # Set actions to the same, so that the current download_binary doesn't get re-run in the - # next stage. TODO: this may no longer be necessary... - actions = [item for item in filtered_actions] - except Exception as e: - log.exception(str(e)) - if initial_download: - # No binary exists, or there was an error downloading the binary from the generated URL. - # Filter the actions so that stage 2 can proceed with the remaining actions. - filtered_actions = actions[1:] - action_type, action_dict = filtered_actions[0] - # If the downloaded file exists, move it to $INSTALL_DIR. Put this outside the try/catch above so that - # any errors in the move step are correctly sent to the tool dependency error handler. - if downloaded_filename and os.path.exists(os.path.join(work_dir, downloaded_filename)): - if target_directory: - target_directory = os.path.realpath( - os.path.normpath(os.path.join(install_environment.install_dir, target_directory)) - ) - # Make sure the target directory is not outside of $INSTALL_DIR. - if target_directory.startswith(os.path.realpath(install_environment.install_dir)): - full_path_to_dir = os.path.abspath(os.path.join(install_environment.install_dir, target_directory)) - else: - full_path_to_dir = os.path.abspath(install_environment.install_dir) - else: - full_path_to_dir = os.path.abspath(install_environment.install_dir) - basic_util.move_file(current_dir=work_dir, source=downloaded_filename, destination=full_path_to_dir) - # Not sure why dir is ignored in this method, need to investigate... - dir = None - if initial_download: - return tool_dependency, filtered_actions, dir - return tool_dependency, None, None - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - platform_info_dict = tool_dependency_util.get_platform_info_dict() - platform_info_dict["name"] = str(tool_dependency.name) - platform_info_dict["version"] = str(tool_dependency.version) - url_template_elems = action_elem.findall("url_template") - # Check if there are multiple url_template elements, each with attrib entries for a specific platform. - if len(url_template_elems) > 1: - # - # http://hgdownload.cse.ucsc.edu/admin/exe/macOSX.${architecture}/faToTwoBit - # - # This method returns the url_elem that best matches the current platform as received from os.uname(). - # Currently checked attributes are os and architecture. These correspond to the values sysname and - # processor from the Python documentation for os.uname(). - url_template_elem = tool_dependency_util.get_download_url_for_platform( - url_template_elems, platform_info_dict - ) - else: - url_template_elem = url_template_elems[0] - action_dict["url"] = Template(url_template_elem.text.strip()).safe_substitute(platform_info_dict) - action_dict["target_directory"] = action_elem.get("target_directory", None) - action_dict.update(self.get_elem_checksums(action_elem)) - return action_dict - - -class DownloadByUrl(Download, RecipeStep): - def __init__(self, app): - self.app = app - self.type = "download_by_url" - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - """ - Download a file via HTTP. If the value of initial_download is True, the recipe steps will be - filtered and returned and the installation directory (i.e., dir) will be defined and returned. - If we're not in the initial download stage, these actions will not occur, and None values will - be returned for them. - """ - if initial_download: - # Eliminate the download_by_url action so remaining actions can be processed correctly. - filtered_actions = actions[1:] - url = action_dict["url"] - is_binary = action_dict.get("is_binary", False) - if "target_filename" in action_dict: - # Sometimes compressed archives extract their content to a folder other than the default - # defined file name. Using this attribute will ensure that the file name is set appropriately - # and can be located after download, decompression and extraction. - downloaded_filename = action_dict["target_filename"] - else: - downloaded_filename = os.path.split(url)[-1] - - checksums = self.get_dict_checksums(action_dict) - log.debug("Attempting to download via url: %s", url) - dir = self.url_download(work_dir, downloaded_filename, url, extract=True, checksums=checksums) - if is_binary: - log_file = os.path.join(install_environment.install_dir, basic_util.INSTALLATION_LOG) - if os.path.exists(log_file): - logfile = open(log_file, "a") - else: - logfile = open(log_file, "w") - logfile.write(f"Successfully downloaded from url: {action_dict['url']}\n") - logfile.close() - log.debug(f"Successfully downloaded from url: {action_dict['url']}") - if initial_download: - return tool_dependency, filtered_actions, dir - return tool_dependency, None, None - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - # - # http://sourceforge.net/projects/samtools/files/samtools/0.1.18/samtools-0.1.18.tar.bz2 - # - # - # - # http://sourceforge.net/projects/samtools/files/samtools/0.1.18/samtools-0.1.18.tar.bz2 - # - if is_binary_download: - action_dict["is_binary"] = True - if action_elem.text: - action_dict["url"] = action_elem.text.strip() - target_filename = action_elem.get("target_filename", None) - if target_filename: - action_dict["target_filename"] = target_filename - action_dict.update(self.get_elem_checksums(action_elem)) - return action_dict - - -class DownloadFile(Download, RecipeStep): - def __init__(self, app): - self.app = app - self.type = "download_file" - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - """ - Download a file. If the value of initial_download is True, the recipe steps will be - filtered and returned and the installation directory (i.e., dir) will be defined and returned. - If we're not in the initial download stage, these actions will not occur, and None values will - be returned for them. - """ - # http://effectors.org/download/version/TTSS_GUI-1.0.1.jar - # Download a single file to the working directory. - if initial_download: - filtered_actions = actions[1:] - url = action_dict["url"] - if "target_filename" in action_dict: - # Sometimes compressed archives extracts their content to a folder other than the default - # defined file name. Using this attribute will ensure that the file name is set appropriately - # and can be located after download, decompression and extraction. - filename = action_dict["target_filename"] - else: - filename = url.split("/")[-1] - if current_dir is not None: - work_dir = current_dir - checksums = self.get_dict_checksums(action_dict) - log.debug("Attempting to download via url: %s", url) - self.url_download(work_dir, filename, url, extract=action_dict["extract"], checksums=checksums) - if initial_download: - dir = os.path.curdir - return tool_dependency, filtered_actions, dir - return tool_dependency, None, None - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - # http://effectors.org/download/version/TTSS_GUI-1.0.1.jar - if action_elem.text: - action_dict["url"] = action_elem.text.strip() - target_filename = action_elem.get("target_filename", None) - if target_filename: - action_dict["target_filename"] = target_filename - action_dict["extract"] = asbool(action_elem.get("extract", False)) - action_dict.update(self.get_elem_checksums(action_elem)) - return action_dict - - -class MakeDirectory(RecipeStep): - def __init__(self, app): - self.app = app - self.type = "make_directory" - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - """ - Make a directory on disk. Since this class is not used in the initial download stage, no recipe step - filtering is performed here, and None values are always returned for filtered_actions and dir. - """ - if os.path.isabs(action_dict["full_path"]): - full_path = action_dict["full_path"] - else: - full_path = os.path.join(current_dir, action_dict["full_path"]) - self.make_directory(full_path=full_path) - return tool_dependency, None, None - - def make_directory(self, full_path): - if not os.path.exists(full_path): - os.makedirs(full_path) - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - # $INSTALL_DIR/lib/python - if action_elem.text: - action_dict["full_path"] = basic_util.evaluate_template(action_elem.text, install_environment) - return action_dict - - -class MakeInstall(RecipeStep): - def __init__(self, app): - self.app = app - self.type = "make_install" - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - """ - Execute a make_install command in a shell. Since this class is not used in the initial download stage, - no recipe step filtering is performed here, and None values are always returned for filtered_actions and dir. - """ - # make; make install; allow providing make options - with settings(warn_only=True): - make_opts = action_dict.get("make_opts", "") - cmd = install_environment.build_command(f"make {make_opts} && make install") - install_environment.handle_command( - tool_dependency=tool_dependency, cmd=cmd, return_output=False, job_name=package_name - ) - # The caller should check the status of the returned tool_dependency since this function - # does nothing with the return_code. - return tool_dependency, None, None - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - # make; make install; allow providing make options - if action_elem.text: - make_opts = basic_util.evaluate_template(action_elem.text, install_environment) - action_dict["make_opts"] = make_opts - return action_dict - - -class MoveDirectoryFiles(RecipeStep): - def __init__(self, app): - self.app = app - self.type = "move_directory_files" - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - """ - Move a directory of files. Since this class is not used in the initial download stage, no recipe step - filtering is performed here, and None values are always returned for filtered_actions and dir. - """ - self.move_directory_files( - current_dir=current_dir, - source_dir=os.path.join(action_dict["source_directory"]), - destination_dir=os.path.join(action_dict["destination_directory"]), - ) - return tool_dependency, None, None - - def move_directory_files(self, current_dir, source_dir, destination_dir): - source_directory = os.path.abspath(os.path.join(current_dir, source_dir)) - destination_directory = os.path.abspath(os.path.join(destination_dir)) - if not os.path.isdir(destination_directory): - os.makedirs(destination_directory) - for dir_entry in os.listdir(source_directory): - source_entry = os.path.join(source_directory, dir_entry) - if os.path.islink(source_entry): - destination_entry = os.path.join(destination_directory, dir_entry) - os.symlink(os.readlink(source_entry), destination_entry) - os.remove(source_entry) - else: - shutil.move(source_entry, destination_directory) - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - # - # bin - # $INSTALL_DIR/bin - # - for move_elem in action_elem: - move_elem_text = basic_util.evaluate_template(move_elem.text, install_environment) - if move_elem_text: - action_dict[move_elem.tag] = move_elem_text - return action_dict - - -class MoveFile(RecipeStep): - def __init__(self, app): - self.app = app - self.type = "move_file" - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - """ - Move a file on disk. Since this class is not used in the initial download stage, no recipe step - filtering is performed here, and None values are always returned for filtered_actions and dir. - """ - basic_util.move_file( - current_dir=current_dir, - source=os.path.join(action_dict["source"]), - destination=os.path.join(action_dict["destination"]), - rename_to=action_dict["rename_to"], - ) - return tool_dependency, None, None - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - # - # misc/some_file - # $INSTALL_DIR/bin - # - action_dict["source"] = basic_util.evaluate_template(action_elem.find("source").text, install_environment) - action_dict["destination"] = basic_util.evaluate_template( - action_elem.find("destination").text, install_environment - ) - action_dict["rename_to"] = action_elem.get("rename_to") - return action_dict - - -class RegexReplace(RecipeStep): - def __init__(self, app): - self.app = app - self.type = "regex_replace" - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - r""" - Search and replace text in a file using regular expressions. Since this class is not used in the initial - download stage, no recipe step filtering is performed here, and None values are always returned for - filtered_actions and dir. - - This step supports the full range of python's regular expression engine, including backreferences in - the replacement text. Example:: - - - ^CFLAGS(\s*)=\s*-g\s*-Wall\s*-O2\s*$$ - CFLAGS\1= -g -Wall -O2 -I$$(NCURSES_INCLUDE_PATH)/ncurses/ -I$$(NCURSES_INCLUDE_PATH) -L$$(NCURSES_LIB_PATH) - - - Before:: - - CFLAGS = -g -Wall -O2 - - After:: - - CFLAGS = -g -Wall -O2 -I$(NCURSES_INCLUDE_PATH)/ncurses/ -I$(NCURSES_INCLUDE_PATH) -L$(NCURSES_LIB_PATH) - """ - log_file = os.path.join(install_environment.install_dir, basic_util.INSTALLATION_LOG) - if os.path.exists(log_file): - logfile = open(log_file, "ab") - else: - logfile = open(log_file, "wb") - if os.path.isabs(action_dict["filename"]): - filename = action_dict["filename"] - if not (filename.startswith(current_dir) or filename.startswith(install_environment.install_dir)): - return tool_dependency, None, None - else: - filename = os.path.abspath(os.path.join(current_dir, action_dict["filename"])) - regex = re.compile(action_dict["regex"]) - replacement = action_dict["replacement"] - temp_fh = tempfile.NamedTemporaryFile(dir=current_dir) - ofh = temp_fh.file - total_replacements = 0 - with open(filename) as haystack: - for line in haystack: - altered_text, replacement_count = re.subn(regex, replacement, line) - if replacement_count > 0: - ofh.write(altered_text) - total_replacements += replacement_count - else: - ofh.write(line) - ofh.flush() - shutil.copyfile(temp_fh.name, filename) - log_text = "Successfully replaced pattern %s with text %s in file %s: %s replacements made\n" - log_text = log_text % (action_dict["regex"], action_dict["replacement"], filename, total_replacements) - log.debug(log_text) - logfile.write(log_text) - logfile.close() - return tool_dependency, None, None - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - """ - Populate action_dict with the provided filename, regex, and replacement text. - """ - action_dict["filename"] = basic_util.evaluate_template(action_elem.get("filename"), install_environment) - action_dict["regex"] = basic_util.evaluate_template(action_elem.find("regex").text, install_environment) - action_dict["replacement"] = basic_util.evaluate_template( - action_elem.find("replacement").text, install_environment - ) - return action_dict - - -class SetEnvironment(RecipeStep): - def __init__(self, app): - self.app = app - self.type = "set_environment" - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - """ - Configure an install environment. Since this class is not used in the initial download stage, - no recipe step filtering is performed here, and None values are always returned for filtered_actions - and dir. - """ - # Currently the only action supported in this category is "environment_variable". - cmds = install_environment.environment_commands("set_environment") - env_var_dicts = action_dict.get("environment_variable", []) - root_dir_dict = dict( - action="set_to", - name=f"{re.sub('[^A-Z0-9_]', '_', tool_dependency.name.upper())}_ROOT_DIR", - value=install_environment.install_dir, - ) - env_var_dicts.append(root_dir_dict) - for env_var_dict in env_var_dicts: - # Check for the presence of the $ENV[] key string and populate it if possible. - env_var_dict = self.handle_environment_variables( - install_environment=install_environment, - tool_dependency=tool_dependency, - env_var_dict=env_var_dict, - set_prior_environment_commands=cmds, - ) - env_file_builder.append_line(**env_var_dict) - # The caller should check the status of the returned tool_dependency since return_code is not - # returned by this function. - return tool_dependency, None, None - - def handle_environment_variables( - self, install_environment, tool_dependency, env_var_dict, set_prior_environment_commands - ): - """ - This method works with a combination of three tool dependency definition tag sets, which are defined in - the tool_dependencies.xml file in the order discussed here. The example for this discussion is the - tool_dependencies.xml file contained in the osra repository, which is available at: - - https://testtoolshed.g2.bx.psu.edu/view/bgruening/osra - - The first tag set defines a complex repository dependency like this. This tag set ensures that changeset - revision XXX of the repository named package_graphicsmagick_1_3 owned by YYY in the tool shed ZZZ has been - previously installed:: - - - - - - ... - - By the way, there is an env.sh file associated with version 1.3.18 of the graphicsmagick package which looks - something like this (we'll reference this file later in this discussion):: - - GRAPHICSMAGICK_ROOT_DIR=//graphicsmagick/1.3.18/YYY/package_graphicsmagick_1_3/XXX/gmagick; - export GRAPHICSMAGICK_ROOT_DIR - - The second tag set defines a specific package dependency that has been previously installed (guaranteed by the - tag set discussed above) and compiled, where the compiled dependency is needed by the tool dependency currently - being installed (osra version 2.0.0 in this case) and complied in order for its installation and compilation to - succeed. This tag set is contained within the tag set, which implies that - version 2.0.0 of the osra package requires version 1.3.18 of the graphicsmagick package in order to successfully - compile. When this tag set is handled, one of the effects is that the env.sh file associated with graphicsmagick - version 1.3.18 is "sourced", which undoubtedly sets or alters certain environment variables (e.g. PATH, PYTHONPATH, - etc):: - - - - - - - - - The third tag set enables discovery of the same required package dependency discussed above for correctly compiling - the osra version 2.0.0 package, but in this case the package can be discovered at tool execution time. Using the - $ENV[] option as shown in this example, the value of the environment variable named GRAPHICSMAGICK_ROOT_DIR (which - was set in the environment using the second tag set described above) will be used to automatically alter the env.sh - file associated with the osra version 2.0.0 tool dependency when it is installed into Galaxy. * Refer to where we - discussed the env.sh file for version 1.3.18 of the graphicsmagick package above:: - - - $ENV[GRAPHICSMAGICK_ROOT_DIR]/lib/ - $INSTALL_DIR/potrace/build/lib/ - $INSTALL_DIR/bin - - $INSTALL_DIR/share - - - The above tag will produce an env.sh file for version 2.0.0 of the osra package when it it installed into Galaxy - that looks something like this. Notice that the path to the gmagick binary is included here since it expands the - defined $ENV[GRAPHICSMAGICK_ROOT_DIR] value in the above tag set:: - - LD_LIBRARY_PATH=//graphicsmagick/1.3.18/YYY/package_graphicsmagick_1_3/XXX/gmagick/lib/:$LD_LIBRARY_PATH; - export LD_LIBRARY_PATH - LD_LIBRARY_PATH=//osra/1.4.0/YYY/depends_on/XXX/potrace/build/lib/:$LD_LIBRARY_PATH; - export LD_LIBRARY_PATH - PATH=//osra/1.4.0/YYY/depends_on/XXX/bin:$PATH; - export PATH - OSRA_DATA_FILES=//osra/1.4.0/YYY/depends_on/XXX/share; - export OSRA_DATA_FILES - """ - env_var_value = env_var_dict["value"] - # env_var_value is the text of an environment variable tag like this: - # - # Here is an example of what env_var_value could look like: $ENV[GRAPHICSMAGICK_ROOT_DIR]/lib/ - if "$ENV[" in env_var_value and "]" in env_var_value: - # Pull out the name of the environment variable to populate. - inherited_env_var_name = env_var_value.split("[")[1].split("]")[0] - to_replace = f"$ENV[{inherited_env_var_name}]" - found = False - for env_cmd in set_prior_environment_commands: - # LD_LIBRARY_PATH=//; export LD_LIBRARY_PATH - if env_cmd.startswith(inherited_env_var_name): - env_val = env_cmd.split("=")[1] - # //; export LD_LIBRARY_PATH - env_val = env_val.split(";")[0] - # // - log.info("Replacing %s with %s in env.sh for this repository.", to_replace, env_val) - env_var_value = env_var_value.replace(to_replace, env_val) - found = True - break - if not found: - # Replace the original $ENV[] with nothing, to avoid any shell misparsings later on. - log.debug("Environment variable %s not found, removing from set_environment.", inherited_env_var_name) - env_var_value = env_var_value.replace(to_replace, f"${inherited_env_var_name}") - env_var_dict["value"] = env_var_value - return env_var_dict - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - # This function is only called for set environment actions as defined above, not within a tool - # dependency type. Here is an example of the tag set this function does handle: - # - # $INSTALL_DIR/lib/python - # $INSTALL_DIR/bin - # - # Here is an example of the tag set this function does not handle: - # - # $INSTALL_DIR - # - env_manager = EnvManager(self.app) - env_var_dicts = [] - for env_elem in action_elem: - if env_elem.tag == "environment_variable": - env_var_dict = env_manager.create_env_var_dict(elem=env_elem, install_environment=install_environment) - if env_var_dict: - env_var_dicts.append(env_var_dict) - if env_var_dicts: - # The last child of an might be a comment, so manually set it to be 'environment_variable'. - action_dict["environment_variable"] = env_var_dicts - return action_dict - - -class SetEnvironmentForInstall(RecipeStep): - def __init__(self, app): - self.app = app - self.type = "set_environment_for_install" - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - """ - Configure an environment for compiling a package. Since this class is not used in the initial - download stage, no recipe step filtering is performed here, and None values are always returned - for filtered_actions and dir. - """ - # Currently the only action supported in this category is a list of paths to one or more tool - # dependency env.sh files, the environment setting in each of which will be injected into the - # environment for all tags that follow this - # tag set in the tool_dependencies.xml file. - env_shell_file_paths = action_dict.get("env_shell_file_paths", []) - install_environment.add_env_shell_file_paths(env_shell_file_paths) - return tool_dependency, None, None - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - # - # - # - # - # - # This action type allows for defining an environment that will properly compile a tool dependency. - # Currently, tag set definitions like that above are supported, but in the future other approaches - # to setting environment variables or other environment attributes can be supported. The above tag - # set will result in the installed and compiled numpy version 1.7.1 binary to be used when compiling - # the current tool dependency package. See the package_matplotlib_1_2 repository in the test tool - # shed for a real-world example. - all_env_shell_file_paths = [] - env_manager = EnvManager(self.app) - for env_elem in action_elem: - if env_elem.tag == "repository": - env_shell_file_paths = env_manager.get_env_shell_file_paths(env_elem) - if env_shell_file_paths: - all_env_shell_file_paths.extend(env_shell_file_paths) - action_dict["env_shell_file_paths"] = all_env_shell_file_paths - return action_dict - - -class SetupPerlEnvironment(Download, RecipeStep): - def __init__(self, app): - self.app = app - self.type = "setup_perl_environment" - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - """ - Initialize the environment for installing Perl packages. The class is called during the initial - download stage when installing packages, so the value of initial_download will generally be True. - However, the parameter value allows this class to also be used in the second stage of the installation, - although it may never be necessary. If initial_download is True, the recipe steps will be filtered - and returned and the installation directory (i.e., dir) will be defined and returned. If we're not - in the initial download stage, these actions will not occur, and None values will be returned for them. - """ - # - # - # - # - # - # XML::Parser - # http://search.cpan.org/CPAN/authors/id/C/CJ/CJFIELDS/BioPerl-1.6.922.tar.gz - # - dir = None - if initial_download: - filtered_actions = actions[1:] - env_shell_file_paths = action_dict.get("env_shell_file_paths", None) - if env_shell_file_paths is None: - log.debug("Missing Perl environment, make sure your specified Perl installation exists.") - if initial_download: - return tool_dependency, filtered_actions, dir - return tool_dependency, None, None - else: - install_environment.add_env_shell_file_paths(env_shell_file_paths) - log.debug( - "Handling setup_perl_environment for tool dependency %s with install_environment.env_shell_file_paths:\n%s" - % (str(tool_dependency.name), str(install_environment.env_shell_file_paths)) - ) - dir = os.path.curdir - current_dir = os.path.abspath(os.path.join(work_dir, dir)) - with lcd(current_dir): - with settings(warn_only=True): - perl_packages = action_dict.get("perl_packages", []) - for perl_package_dict in perl_packages: - perl_package = perl_package_dict["package"] - # If set to a true value then MakeMaker's prompt function will always - # return the default without waiting for user input. - cmd = """PERL_MM_USE_DEFAULT=1; export PERL_MM_USE_DEFAULT; """ - cmd += f"HOME={work_dir}; export HOME; " - cmd += "export PERL5LIB=$INSTALL_DIR/lib/perl5:$PERL5LIB;" - cmd += "export PATH=$INSTALL_DIR/bin:$PATH;" - if perl_package.find("://") != -1: - # We assume a URL to a gem file. - url = perl_package - perl_package_name = url.split("/")[-1] - checksums = perl_package_dict.get("checksums", {}) - dir = self.url_download(work_dir, perl_package_name, url, extract=True, checksums=checksums) - # Search for Build.PL or Makefile.PL (ExtUtils::MakeMaker vs. Module::Build). - tmp_work_dir = os.path.join(work_dir, dir) - if os.path.exists(os.path.join(tmp_work_dir, "Makefile.PL")): - cmd += """perl Makefile.PL INSTALL_BASE=$INSTALL_DIR && make && make install""" - elif os.path.exists(os.path.join(tmp_work_dir, "Build.PL")): - cmd += """perl Build.PL --install_base $INSTALL_DIR && perl Build && perl Build install""" - else: - log.debug( - "No Makefile.PL or Build.PL file found in %s. Skipping installation of %s." - % (url, perl_package_name) - ) - if initial_download: - return tool_dependency, filtered_actions, dir - return tool_dependency, None, None - with lcd(tmp_work_dir): - cmd = install_environment.build_command( - basic_util.evaluate_template(cmd, install_environment) - ) - return_code = install_environment.handle_command( - tool_dependency=tool_dependency, cmd=cmd, return_output=False, job_name=package_name - ) - if return_code: - if initial_download: - return tool_dependency, filtered_actions, dir - return tool_dependency, None, None - else: - # perl package from CPAN without version number. - # cpanm should be installed with the parent perl distribution, otherwise this will not work. - cmd += f"""cpanm --local-lib=$INSTALL_DIR {perl_package}""" - cmd = install_environment.build_command(basic_util.evaluate_template(cmd, install_environment)) - return_code = install_environment.handle_command( - tool_dependency=tool_dependency, cmd=cmd, return_output=False, job_name=package_name - ) - if return_code: - if initial_download: - return tool_dependency, filtered_actions, dir - return tool_dependency, None, None - # Pull in perl dependencies (runtime). - env_file_builder.handle_action_shell_file_paths(action_dict) - # Recursively add dependent PERL5LIB and PATH to env.sh & anything else needed. - env_file_builder.append_line( - name="PERL5LIB", - action="prepend_to", - value=os.path.join(install_environment.install_dir, "lib", "perl5"), - ) - env_file_builder.append_line( - name="PATH", action="prepend_to", value=os.path.join(install_environment.install_dir, "bin") - ) - return_code = env_file_builder.return_code - if return_code: - if initial_download: - return tool_dependency, filtered_actions, dir - return tool_dependency, None, None - if initial_download: - return tool_dependency, filtered_actions, dir - return tool_dependency, None, None - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - # setup a Perl environment. - # - # - # - # - # - # XML::Parser - # http://search.cpan.org/CPAN/authors/id/C/CJ/CJFIELDS/BioPerl-1.6.922.tar.gz - # - # Discover all child repository dependency tags and define the path to an env.sh file associated - # with each repository. This will potentially update the value of the 'env_shell_file_paths' entry - # in action_dict. - all_env_shell_file_paths = [] - env_manager = EnvManager(self.app) - action_dict = env_manager.get_env_shell_file_paths_from_setup_environment_elem( - all_env_shell_file_paths, action_elem, action_dict - ) - perl_packages = [] - for env_elem in action_elem: - if env_elem.tag == "package": - # A valid package definition can be: - # XML::Parser - # http://search.cpan.org/CPAN/authors/id/C/CJ/CJFIELDS/BioPerl-1.6.922.tar.gz - # Unfortunately CPAN does not support versioning, so if you want real reproducibility you need to specify - # the tarball path and the right order of different tarballs manually. - perl_packages.append(dict(package=env_elem.text.strip(), checksums=self.get_elem_checksums(env_elem))) - if perl_packages: - action_dict["perl_packages"] = perl_packages - return action_dict - - -class SetupREnvironment(Download, RecipeStep): - def __init__(self, app): - self.app = app - self.type = "setup_r_environment" - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - """ - Initialize the environment for installing R packages. The class is called during the initial - download stage when installing packages, so the value of initial_download will generally be True. - However, the parameter value allows this class to also be used in the second stage of the installation, - although it may never be necessary. If initial_download is True, the recipe steps will be filtered - and returned and the installation directory (i.e., dir) will be defined and returned. If we're not - in the initial download stage, these actions will not occur, and None values will be returned for them. - """ - # - # - # - # - # - # https://github.com/bgruening/download_store/raw/master/DESeq2-1_0_18/BiocGenerics_0.6.0.tar.gz - # - dir = None - if initial_download: - filtered_actions = actions[1:] - env_shell_file_paths = action_dict.get("env_shell_file_paths", None) - if env_shell_file_paths is None: - error_message = "Missing R environment. Please check your specified R installation exists." - log.error(error_message) - status = self.app.install_model.ToolDependency.installation_status.ERROR - tool_dependency = tool_dependency_util.set_tool_dependency_attributes( - self.app, tool_dependency, status=status, error_message=error_message - ) - return tool_dependency, [], None - else: - install_environment.add_env_shell_file_paths(env_shell_file_paths) - log.debug( - "Handling setup_r_environment for tool dependency %s with install_environment.env_shell_file_paths:\n%s" - % (str(tool_dependency.name), str(install_environment.env_shell_file_paths)) - ) - tarball_names = [] - for r_package_dict in action_dict["r_packages"]: - url = r_package_dict["package"] - filename = url.split("/")[-1] - tarball_names.append(filename) - checksums = r_package_dict.get("checksums", {}) - self.url_download(work_dir, filename, url, extract=False, checksums=checksums) - dir = os.path.curdir - current_dir = os.path.abspath(os.path.join(work_dir, dir)) - with lcd(current_dir): - with settings(warn_only=True): - for tarball_name in tarball_names: - # Use raw strings so that python won't automatically unescape the quotes before passing the command - # to subprocess.Popen. - cmd = r'''PATH=$PATH:$R_HOME/bin; export PATH; R_LIBS=$INSTALL_DIR:$R_LIBS; export R_LIBS; - Rscript -e "tryCatch( { install.packages(c('%s'), lib = '$INSTALL_DIR', repos = NULL, dependencies = FALSE) }, error = function(e) { print(e); quit(status = 1) }, warning = function(w) { if ( grepl('had non-zero exit status|is not writable|installation of one of more packages failed', as.character(w)) ) { print(w); quit(status = 1) } } )"''' % ( - str(tarball_name) - ) - cmd = install_environment.build_command(basic_util.evaluate_template(cmd, install_environment)) - return_code = install_environment.handle_command( - tool_dependency=tool_dependency, cmd=cmd, return_output=False, job_name=package_name - ) - if return_code: - if initial_download: - return tool_dependency, filtered_actions, dir - return tool_dependency, None, None - # R libraries are installed to $INSTALL_DIR (install_dir), we now set the R_LIBS path to that directory - # Pull in R environment (runtime). - env_file_builder.handle_action_shell_file_paths(action_dict) - env_file_builder.append_line(name="R_LIBS", action="prepend_to", value=install_environment.install_dir) - return_code = env_file_builder.return_code - if return_code: - if initial_download: - return tool_dependency, filtered_actions, dir - return tool_dependency, None, None - if initial_download: - return tool_dependency, filtered_actions, dir - return tool_dependency, None, None - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - # setup an R environment. - # - # - # - # - # - # https://github.com/bgruening/download_store/raw/master/DESeq2-1_0_18/BiocGenerics_0.6.0.tar.gz - # - # Discover all child repository dependency tags and define the path to an env.sh file - # associated with each repository. This will potentially update the value of the - # 'env_shell_file_paths' entry in action_dict. - all_env_shell_file_paths = [] - env_manager = EnvManager(self.app) - action_dict = env_manager.get_env_shell_file_paths_from_setup_environment_elem( - all_env_shell_file_paths, action_elem, action_dict - ) - r_packages = list() - for env_elem in action_elem: - if env_elem.tag == "package": - r_packages.append(dict(package=env_elem.text.strip(), checksums=self.get_elem_checksums(env_elem))) - if r_packages: - action_dict["r_packages"] = r_packages - return action_dict - - -class SetupRubyEnvironment(Download, RecipeStep): - def __init__(self, app): - self.app = app - self.type = "setup_ruby_environment" - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - """ - Initialize the environment for installing Ruby packages. The class is called during the initial - download stage when installing packages, so the value of initial_download will generally be True. - However, the parameter value allows this class to also be used in the second stage of the installation, - although it may never be necessary. If initial_download is True, the recipe steps will be filtered - and returned and the installation directory (i.e., dir) will be defined and returned. If we're not - in the initial download stage, these actions will not occur, and None values will be returned for them. - """ - # - # - # - # - # - # protk - # protk=1.2.4 - # http://url-to-some-gem-file.de/protk.gem - # - dir = None - if initial_download: - filtered_actions = actions[1:] - env_shell_file_paths = action_dict.get("env_shell_file_paths", None) - if env_shell_file_paths is None: - log.debug("Missing Ruby environment, make sure your specified Ruby installation exists.") - if initial_download: - return tool_dependency, filtered_actions, dir - return tool_dependency, None, None - else: - install_environment.add_env_shell_file_paths(env_shell_file_paths) - log.debug( - "Handling setup_ruby_environment for tool dependency %s with install_environment.env_shell_file_paths:\n%s" - % (str(tool_dependency.name), str(install_environment.env_shell_file_paths)) - ) - dir = os.path.curdir - current_dir = os.path.abspath(os.path.join(work_dir, dir)) - with lcd(current_dir): - with settings(warn_only=True): - ruby_package_tups = action_dict.get("ruby_package_tups", []) - for ruby_package_tup_dict in ruby_package_tups: - ruby_package_tup = ruby_package_tup_dict["package"] - gem, gem_version, gem_parameters = ruby_package_tup - if gem_parameters: - gem_parameters = f"-- {gem_parameters}" - else: - gem_parameters = "" - if os.path.isfile(gem): - # we assume a local shipped gem file - cmd = f"""PATH=$PATH:$RUBY_HOME/bin; export PATH; GEM_HOME=$INSTALL_DIR; export GEM_HOME; - gem install --local {gem} {gem_parameters}""" - elif gem.find("://") != -1: - # We assume a URL to a gem file. - url = gem - gem_name = url.split("/")[-1] - checksums = ruby_package_tup_dict.get("checksums", {}) - self.url_download(work_dir, gem_name, url, extract=False, checksums=checksums) - cmd = f"""PATH=$PATH:$RUBY_HOME/bin; export PATH; GEM_HOME=$INSTALL_DIR; export GEM_HOME; - gem install --local {gem_name} {gem_parameters}""" - else: - # gem file from rubygems.org with or without version number - if gem_version: - # Specific ruby gem version was requested. - # Use raw strings so that python won't automatically unescape the quotes before passing the command - # to subprocess.Popen. - cmd = rf"""PATH=$PATH:$RUBY_HOME/bin; export PATH; GEM_HOME=$INSTALL_DIR; export GEM_HOME; - gem install {gem} --version "={gem_version}" {gem_parameters}""" - else: - # no version number given - cmd = f"""PATH=$PATH:$RUBY_HOME/bin; export PATH; GEM_HOME=$INSTALL_DIR; export GEM_HOME; - gem install {gem} {gem_parameters}""" - cmd = install_environment.build_command(basic_util.evaluate_template(cmd, install_environment)) - return_code = install_environment.handle_command( - tool_dependency=tool_dependency, cmd=cmd, return_output=False, job_name=package_name - ) - if return_code: - if initial_download: - return tool_dependency, filtered_actions, dir - return tool_dependency, None, None - # Pull in ruby dependencies (runtime). - env_file_builder.handle_action_shell_file_paths(action_dict) - env_file_builder.append_line( - name="GEM_PATH", action="prepend_to", value=install_environment.install_dir - ) - env_file_builder.append_line( - name="PATH", action="prepend_to", value=os.path.join(install_environment.install_dir, "bin") - ) - return_code = env_file_builder.return_code - if return_code: - if initial_download: - return tool_dependency, filtered_actions, dir - return tool_dependency, None, None - if initial_download: - return tool_dependency, filtered_actions, dir - return tool_dependency, None, None - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - # setup a Ruby environment. - # - # - # - # - # - # protk - # protk=1.2.4 - # http://url-to-some-gem-file.de/protk.gem - # - # Discover all child repository dependency tags and define the path to an env.sh file - # associated with each repository. This will potentially update the value of the - # 'env_shell_file_paths' entry in action_dict. - all_env_shell_file_paths = [] - env_manager = EnvManager(self.app) - action_dict = env_manager.get_env_shell_file_paths_from_setup_environment_elem( - all_env_shell_file_paths, action_elem, action_dict - ) - ruby_package_tups = [] - for env_elem in action_elem: - if env_elem.tag == "package": - # A valid gem definition can be: - # protk=1.2.4 - # protk - # ftp://ftp.gruening.de/protk.gem - gem_token = env_elem.text.strip().split("=") - gem_parameters = env_elem.get("parameters", None) - if len(gem_token) == 2: - # version string - gem_name = gem_token[0] - gem_version = gem_token[1] - tup = (gem_name, gem_version, gem_parameters) - else: - # gem name for rubygems.org without version number - gem = env_elem.text.strip() - tup = (gem, None, gem_parameters) - ruby_package_tups.append(dict(package=tup, checksums=self.get_elem_checksums(env_elem))) - if ruby_package_tups: - action_dict["ruby_package_tups"] = ruby_package_tups - return action_dict - - -class SetupPythonEnvironment(Download, RecipeStep): - def __init__(self, app): - self.app = app - self.type = "setup_python_environment" - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - """ - Initialize the environment for installing Python packages. The class is called during the initial - download stage when installing packages, so the value of initial_download will generally be True. - However, the parameter value allows this class to also be used in the second stage of the installation, - although it may never be necessary. If initial_download is True, the recipe steps will be filtered - and returned and the installation directory (i.e., dir) will be defined and returned. If we're not - in the initial download stage, these actions will not occur, and None values will be returned for them. - """ - # - # - # - # - # - # pysam.tar.gz - # http://url-to-some-python-package.de/pysam.tar.gz - # - dir = None - if initial_download: - filtered_actions = actions[1:] - env_shell_file_paths = action_dict.get("env_shell_file_paths", None) - if env_shell_file_paths is None: - log.debug("Missing Python environment, make sure your specified Python installation exists.") - if initial_download: - return tool_dependency, filtered_actions, dir - return tool_dependency, None, None - else: - install_environment.add_env_shell_file_paths(env_shell_file_paths) - log.debug( - "Handling setup_python_environment for tool dependency %s with install_environment.env_shell_file_paths:\n%s" - % (str(tool_dependency.name), str(install_environment.env_shell_file_paths)) - ) - dir = os.path.curdir - current_dir = os.path.abspath(os.path.join(work_dir, dir)) - with lcd(current_dir): - with settings(warn_only=True): - python_package_tups = action_dict.get("python_package_tups", []) - for python_package_tup_dict in python_package_tups: - python_package_tup = python_package_tup_dict["package"] - package, package_version = python_package_tup - package_path = os.path.join(install_environment.tool_shed_repository_install_dir, package) - if os.path.isfile(package_path): - # We assume a local shipped python package. - package_to_install = package_path - elif package.find("://") != -1: - # We assume a URL to a python package. - url = package - package_name = url.split("/")[-1] - checksums = python_package_tup_dict.get("checksums", {}) - self.url_download(work_dir, package_name, url, extract=False, checksums=checksums) - package_to_install = os.path.join(work_dir, package_name) - else: - # pypi support is currently not working - pip can not install wheels into user specified directories - pass - archive = CompressedFile(package_to_install) - uncompressed_path = archive.extract(work_dir) - cmd = r"""PATH=$PYTHONHOME/bin:$PATH; export PATH; - mkdir -p $INSTALL_DIR/lib/python; - export PYTHONPATH=$INSTALL_DIR/lib/python:$PYTHONPATH; - cd %s; - python setup.py install --install-lib $INSTALL_DIR/lib/python --install-scripts $INSTALL_DIR/bin - """ % ( - uncompressed_path - ) - - cmd = install_environment.build_command(basic_util.evaluate_template(cmd, install_environment)) - return_code = install_environment.handle_command( - tool_dependency=tool_dependency, cmd=cmd, return_output=False, job_name=package_name - ) - if return_code: - if initial_download: - return tool_dependency, filtered_actions, dir - return tool_dependency, None, None - # Pull in python dependencies (runtime). - env_file_builder.handle_action_shell_file_paths(action_dict) - env_file_builder.append_line( - name="PYTHONPATH", - action="prepend_to", - value=os.path.join(install_environment.install_dir, "lib", "python"), - ) - env_file_builder.append_line( - name="PYTHONPATH", action="prepend_to", value=os.path.join(install_environment.install_dir) - ) - env_file_builder.append_line( - name="PATH", action="prepend_to", value=os.path.join(install_environment.install_dir, "bin") - ) - return_code = env_file_builder.return_code - if return_code: - if initial_download: - return tool_dependency, filtered_actions, dir - return tool_dependency, None, None - if initial_download: - return tool_dependency, filtered_actions, dir - return tool_dependency, None, None - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - # setup a Python environment. - # - # - # - # - # - # pysam.tar.gz - # http://url-to-some-python-package.de/pysam.tar.gz - # - # Discover all child repository dependency tags and define the path to an env.sh file - # associated with each repository. This will potentially update the value of the - # 'env_shell_file_paths' entry in action_dict. - all_env_shell_file_paths = [] - env_manager = EnvManager(self.app) - action_dict = env_manager.get_env_shell_file_paths_from_setup_environment_elem( - all_env_shell_file_paths, action_elem, action_dict - ) - python_package_tups = [] - for env_elem in action_elem: - if env_elem.tag == "package": - # A valid package definitions can be: - # pysam.tar.gz -> locally shipped tarball - # ftp://ftp.gruening.de/pysam.tar.gz -> online tarball - python_token = env_elem.text.strip().split("=") - if len(python_token) == 2: - # version string - package_name = python_token[0] - package_version = python_token[1] - tup = (package_name, package_version) - else: - # package name for pypi.org without version number - package = env_elem.text.strip() - tup = (package, None) - python_package_tups.append(dict(package=tup, checksums=self.get_elem_checksums(env_elem))) - if python_package_tups: - action_dict["python_package_tups"] = python_package_tups - action_dict.update(self.get_elem_checksums(action_elem)) - return action_dict - - -class SetupVirtualEnv(Download, RecipeStep): - def __init__(self, app): - self.app = app - self.type = "setup_virtualenv" - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - """ - Initialize a virtual environment for installing packages. If initial_download is True, the recipe - steps will be filtered and returned and the installation directory (i.e., dir) will be defined and - returned. If we're not in the initial download stage, these actions will not occur, and None values - will be returned for them. - """ - # This class is not currently used during stage 1 of the installation process, so filter_actions - # are not affected, and dir is not set. Enhancements can easily be made to this function if this - # class is needed in stage 1. - venv_src_directory = os.path.abspath(os.path.join(self.app.tool_dependency_dir, "__virtualenv_src")) - if not self.install_virtualenv(install_environment, venv_src_directory): - log.debug("Unable to install virtualenv") - return tool_dependency, None, None - requirements = action_dict["requirements"] - if os.path.exists(os.path.join(install_environment.install_dir, requirements)): - # requirements specified as path to a file - requirements_path = requirements - else: - # requirements specified directly in XML, create a file with these for pip. - requirements_path = os.path.join(install_environment.install_dir, "requirements.txt") - with open(requirements_path, "w") as f: - f.write(requirements) - venv_directory = os.path.join(install_environment.install_dir, "venv") - python_cmd = action_dict["python"] - # TODO: Consider making --no-site-packages optional. - setup_command = f"{python_cmd} {venv_src_directory}/virtualenv.py --no-site-packages '{venv_directory}'" - # POSIXLY_CORRECT forces shell commands . and source to have the same - # and well defined behavior in bash/zsh. - activate_command = f"POSIXLY_CORRECT=1; . {os.path.join(venv_directory, 'bin', 'activate')}" - if action_dict["use_requirements_file"]: - install_command = "python '{}' install -r '{}' --log '{}'".format( - os.path.join(venv_directory, "bin", "pip"), - requirements_path, - os.path.join(install_environment.install_dir, "pip_install.log"), - ) - else: - install_command = "" - with open(requirements_path, "rb") as f: - while True: - line = f.readline() - if not line: - break - line = line.strip() - if line: - line_install_command = "python '{}' install {} --log '{}'".format( - os.path.join(venv_directory, "bin", "pip"), - line, - os.path.join(install_environment.install_dir, "pip_install_%s.log" % (line)), - ) - if not install_command: - install_command = line_install_command - else: - install_command = f"{install_command} && {line_install_command}" - full_setup_command = f"{setup_command}; {activate_command}; {install_command}" - return_code = install_environment.handle_command( - tool_dependency=tool_dependency, cmd=full_setup_command, return_output=False, job_name=package_name - ) - if return_code: - log.error("Failed to do setup_virtualenv install, exit code='%s'", return_code) - # would it be better to try to set env variables anway, instead of returning here? - return tool_dependency, None, None - site_packages_directory, site_packages_directory_list = self.__get_site_packages_directory( - install_environment, self.app, tool_dependency, python_cmd, venv_directory - ) - env_file_builder.append_line(name="PATH", action="prepend_to", value=os.path.join(venv_directory, "bin")) - if site_packages_directory is None: - log.error("virtualenv's site-packages directory '%s' does not exist", site_packages_directory_list) - else: - env_file_builder.append_line(name="PYTHONPATH", action="prepend_to", value=site_packages_directory) - # The caller should check the status of the returned tool_dependency since this function does nothing - # with the return_code. - return_code = env_file_builder.return_code - return tool_dependency, None, None - - def install_virtualenv(self, install_environment, venv_dir): - if not os.path.exists(venv_dir): - with install_environment.use_tmp_dir() as work_dir: - downloaded_filename = VIRTUALENV_URL.rsplit("/", 1)[-1] - try: - dir = self.url_download(work_dir, downloaded_filename, VIRTUALENV_URL) - except Exception: - log.error( - "Failed to download virtualenv: url_download( '%s', '%s', '%s' ) threw an exception", - work_dir, - downloaded_filename, - VIRTUALENV_URL, - ) - return False - full_path_to_dir = os.path.abspath(os.path.join(work_dir, dir)) - shutil.move(full_path_to_dir, venv_dir) - return True - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - # - # Install requirements from file requirements.txt of downloaded bundle - or - - # tools/requirements.txt - # Install requirements from specified file from downloaded bundle -or - - # pyyaml==3.2.0 - # lxml==2.3.0 - # Manually specify contents of requirements.txt file to create dynamically. - action_dict["use_requirements_file"] = asbool(action_elem.get("use_requirements_file", True)) - action_dict["requirements"] = basic_util.evaluate_template( - action_elem.text or "requirements.txt", install_environment - ) - action_dict["python"] = action_elem.get("python", "python") - return action_dict - - def __get_site_packages_directory(self, install_environment, app, tool_dependency, python_cmd, venv_directory): - lib_dir = os.path.join(venv_directory, "lib") - rval = os.path.join(lib_dir, python_cmd, "site-packages") - site_packages_directory_list = [rval] - if os.path.exists(rval): - return (rval, site_packages_directory_list) - for _dirpath, dirnames, _filenames in os.walk(lib_dir): - for dirname in dirnames: - rval = os.path.join(lib_dir, dirname, "site-packages") - site_packages_directory_list.append(rval) - if os.path.exists(rval): - return (rval, site_packages_directory_list) - break - # fall back to python call to get site packages - # FIXME: This is probably more robust?, but there is currently an issue with handling the output.stdout - # preventing the entire path from being included (it gets truncated) - # Use raw strings so that python won't automatically unescape the quotes before passing the command - # to subprocess.Popen. - for site_packages_command in [ - r"""%s -c 'import site; site.getsitepackages()[0]'""" % os.path.join(venv_directory, "bin", "python"), - r"""%s -c 'from __future__ import print_function; import os, sys; print(os.path.join(sys.prefix, "lib", "python" + sys.version[:3], "site-packages"))'""" - % os.path.join(venv_directory, "bin", "python"), - ]: - output = install_environment.handle_command( - tool_dependency=tool_dependency, - cmd=site_packages_command, - return_output=True, - job_name="_get_site_packages", - ) - site_packages_directory_list.append(output.stdout) - if not output.return_code and os.path.exists(output.stdout): - return (output.stdout, site_packages_directory_list) - return (None, site_packages_directory_list) - - -class ShellCommand(RecipeStep): - def __init__(self, app): - self.app = app - self.type = "shell_command" - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - """ - Execute a command in a shell. If the value of initial_download is True, the recipe steps will - be filtered and returned and the installation directory (i.e., dir) will be defined and returned. - If we're not in the initial download stage, these actions will not occur, and None values will - be returned for them. - """ - # git clone --recursive git://github.com/ekg/freebayes.git - # Eliminate the shell_command clone action so remaining actions can be processed correctly. - if initial_download: - # I'm not sure why we build the cmd differently in stage 1 vs stage 2. Should this process - # be the same no matter the stage? - dir = package_name - filtered_actions = actions[1:] - cmd = action_dict["command"] - else: - cmd = install_environment.build_command(action_dict["command"]) - with settings(warn_only=True): - # The caller should check the status of the returned tool_dependency since this function - # does nothing with return_code. - install_environment.handle_command( - tool_dependency=tool_dependency, cmd=cmd, return_output=False, job_name=package_name - ) - if initial_download: - return tool_dependency, filtered_actions, dir - return tool_dependency, None, None - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - # make - action_elem_text = basic_util.evaluate_template(action_elem.text, install_environment) - if action_elem_text: - action_dict["command"] = action_elem_text - return action_dict - - -class TemplateCommand(RecipeStep): - def __init__(self, app): - self.app = app - self.type = "template_command" - - def execute_step( - self, - tool_dependency, - package_name, - actions, - action_dict, - filtered_actions, - env_file_builder, - install_environment, - work_dir, - current_dir=None, - initial_download=False, - ): - """ - Execute a template command in a shell. If the value of initial_download is True, the recipe steps - will be filtered and returned and the installation directory (i.e., dir) will be defined and returned. - If we're not in the initial download stage, these actions will not occur, and None values will be - returned for them. - """ - env_vars = dict() - env_vars = install_environment.environment_dict() - env_vars.update(basic_util.get_env_var_values(install_environment)) - language = action_dict["language"] - with settings(warn_only=True, **env_vars): - if language == "cheetah": - # We need to import fabric.api.env so that we can access all collected environment variables. - cmd = fill_template(f"#from fabric.api import env\n{action_dict['command']}", context=env_vars) - # The caller should check the status of the returned tool_dependency since this function - # does nothing with return_code. - install_environment.handle_command( - tool_dependency=tool_dependency, cmd=cmd, return_output=False, job_name=package_name - ) - return tool_dependency, None, None - - def prepare_step(self, tool_dependency, action_elem, action_dict, install_environment, is_binary_download): - # Default to Cheetah as it's the first template language supported. - language = action_elem.get("language", "cheetah").lower() - if language == "cheetah": - # Cheetah template syntax. - # - # #if env.PATH: - # make - # #end if - # - action_elem_text = action_elem.text.strip() - if action_elem_text: - action_dict["language"] = language - action_dict["command"] = action_elem_text - else: - log.debug(f"Unsupported template language '{language}'. Not proceeding.") - raise Exception(f"Unsupported template language '{language}' in tool dependency definition.") - return action_dict diff --git a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py b/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py deleted file mode 100644 index fb29205880c..00000000000 --- a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/tag_handler.py +++ /dev/null @@ -1,757 +0,0 @@ -import logging -import os -import tempfile - -from galaxy.tool_shed.galaxy_install.tool_dependencies.env_manager import EnvManager -from galaxy.tool_shed.galaxy_install.tool_dependencies.recipe.env_file_builder import EnvFileBuilder -from galaxy.tool_shed.galaxy_install.tool_dependencies.recipe.install_environment import InstallEnvironment -from galaxy.tool_shed.util import tool_dependency_util -from galaxy.tool_shed.util.basic_util import ( - INSTALLATION_LOG, - remove_dir, -) -from galaxy.tool_shed.util.metadata_util import get_updated_changeset_revisions_from_tool_shed -from galaxy.tool_shed.util.repository_util import ( - get_absolute_path_to_file_in_repository, - get_repository_for_dependency_relationship, -) -from galaxy.tool_util.deps.resolvers import NullDependency -from galaxy.util import ( - listify, - url_get, -) -from galaxy.util.tool_shed.common_util import ( - get_tool_shed_url_from_tool_shed_registry, - remove_protocol_from_tool_shed_url, -) -from galaxy.util.tool_shed.xml_util import parse_xml - -log = logging.getLogger(__name__) - - -class RecipeTag: - """Abstract class that defines a standard format for handling recipe tags when installing packages.""" - - def process_tag_set( - self, - tool_shed_repository, - tool_dependency, - package_elem, - package_name, - package_version, - tool_dependency_db_records=None, - ): - raise Exception("Unimplemented Method") - - -class SyncDatabase: - def sync_database_with_file_system( - self, - app, - tool_shed_repository, - tool_dependency_name, - tool_dependency_version, - tool_dependency_install_dir, - tool_dependency_type="package", - ): - """ - The installation directory defined by the received tool_dependency_install_dir exists, so check for - the presence of INSTALLATION_LOG. If the files exists, we'll assume the tool dependency is installed, - but not necessarily successfully (it could be in an error state on disk. However, we can justifiably - assume here that no matter the state, an associated database record will exist. - """ - # This method should be reached very rarely. It implies that either the Galaxy environment - # became corrupted (i.e., the database records for installed tool dependencies is not synchronized - # with tool dependencies on disk) or the Tool Shed's install and test framework is running. The Tool - # Shed's install and test framework installs repositories in 2 stages, those of type tool_dependency_definition - # followed by those containing valid tools and tool functional test components. - log.debug("Synchronizing the database with the file system...") - try: - log.debug( - "The value of app.config.running_functional_tests is: %s" % str(app.config.running_functional_tests) - ) - except Exception: - pass - sa_session = app.install_model.context - can_install_tool_dependency = False - tool_dependency = tool_dependency_util.get_tool_dependency_by_name_version_type_repository( - app, tool_shed_repository, tool_dependency_name, tool_dependency_version, tool_dependency_type - ) - if tool_dependency.status == app.install_model.ToolDependency.installation_status.INSTALLING: - # The tool dependency is in an Installing state, so we don't want to do anything to it. If the tool - # dependency is being installed by someone else, we don't want to interfere with that. This assumes - # the installation by "someone else" is not hung in an Installing state, which is a weakness if that - # "someone else" never repaired it. - log.debug( - "Skipping installation of tool dependency %s version %s because it has a status of %s" - % (str(tool_dependency.name), str(tool_dependency.version), str(tool_dependency.status)) - ) - else: - # We have a pre-existing installation directory on the file system, but our associated database record is - # in a state that allowed us to arrive here. At this point, we'll inspect the installation directory to - # see if we have a "proper installation" and if so, synchronize the database record rather than reinstalling - # the dependency if we're "running_functional_tests". If we're not "running_functional_tests, we'll set - # the tool dependency's installation status to ERROR. - tool_dependency_installation_directory_contents = os.listdir(tool_dependency_install_dir) - if INSTALLATION_LOG in tool_dependency_installation_directory_contents: - # Since this tool dependency's installation directory contains an installation log, we consider it to be - # installed. In some cases the record may be missing from the database due to some activity outside of - # the control of the Tool Shed. Since a new record was created for it and we don't know the state of the - # files on disk, we will set it to an error state (unless we are running Tool Shed functional tests - see - # below). - log.debug( - "Skipping installation of tool dependency %s version %s because it is installed in %s" - % (str(tool_dependency.name), str(tool_dependency.version), str(tool_dependency_install_dir)) - ) - if app.config.running_functional_tests: - # If we are running functional tests, the state will be set to Installed because previously compiled - # tool dependencies are not deleted by default, from the "install and test" framework.. - tool_dependency.status = app.install_model.ToolDependency.installation_status.INSTALLED - else: - error_message = "The installation directory for this tool dependency had contents but the database had no record. " - error_message += ( - "The installation log may show this tool dependency to be correctly installed, but due to the " - ) - error_message += "missing database record it is now being set to Error." - tool_dependency.status = app.install_model.ToolDependency.installation_status.ERROR - tool_dependency.error_message = error_message - else: - error_message = ( - "\nInstallation path %s for tool dependency %s version %s exists, but the expected file %s" - % ( - str(tool_dependency_install_dir), - str(tool_dependency_name), - str(tool_dependency_version), - str(INSTALLATION_LOG), - ) - ) - error_message += " is missing. This indicates an installation error so the tool dependency is being" - error_message += " prepared for re-installation." - log.error(error_message) - tool_dependency.status = app.install_model.ToolDependency.installation_status.NEVER_INSTALLED - remove_dir(tool_dependency_install_dir) - can_install_tool_dependency = True - sa_session.add(tool_dependency) - sa_session.flush() - try: - log.debug( - "Returning from sync_database_with_file_system with tool_dependency %s, can_install_tool_dependency %s." - % (str(tool_dependency.name), str(can_install_tool_dependency)) - ) - except Exception as e: - log.debug(str(e)) - return tool_dependency, can_install_tool_dependency - - -class Install(RecipeTag, SyncDatabase): - def __init__(self, app): - self.app = app - self.tag = "install" - - def process_tag_set( - self, - tool_shed_repository, - tool_dependency, - package_elem, - package_name, - package_version, - tool_dependency_db_records=None, - ): - # - # Get the installation directory for tool dependencies that will be installed for the received tool_shed_repository. - actions_elem_tuples = [] - proceed_with_install = False - install_dir = tool_dependency_util.get_tool_dependency_install_dir( - app=self.app, - repository_name=tool_shed_repository.name, - repository_owner=tool_shed_repository.owner, - repository_changeset_revision=tool_shed_repository.installed_changeset_revision, - tool_dependency_type="package", - tool_dependency_name=package_name, - tool_dependency_version=package_version, - ) - if os.path.exists(install_dir): - # Notice that we'll throw away the following tool_dependency if it can be installed. - tool_dependency, proceed_with_install = self.sync_database_with_file_system( - self.app, - tool_shed_repository, - package_name, - package_version, - install_dir, - tool_dependency_type="package", - ) - if not proceed_with_install: - log.debug( - "Tool dependency %s version %s cannot be installed (it was probably previously installed), so returning it." - % (str(tool_dependency.name), str(tool_dependency.version)) - ) - return tool_dependency, proceed_with_install, actions_elem_tuples - else: - proceed_with_install = True - if proceed_with_install: - package_install_version = package_elem.get("version", "1.0") - status = self.app.install_model.ToolDependency.installation_status.INSTALLING - tool_dependency = tool_dependency_util.create_or_update_tool_dependency( - app=self.app, - tool_shed_repository=tool_shed_repository, - name=package_name, - version=package_version, - type="package", - status=status, - set_status=True, - ) - # Get the information about the current platform in case the tool dependency definition includes tag sets - # for installing compiled binaries. - platform_info_dict = tool_dependency_util.get_platform_info_dict() - if package_install_version == "1.0": - # Handle tool dependency installation using a fabric method included in the Galaxy framework. - actions_elem_tuples = tool_dependency_util.parse_package_elem( - package_elem, platform_info_dict=platform_info_dict, include_after_install_actions=True - ) - if not actions_elem_tuples: - proceed_with_install = False - error_message = ( - f"Version {package_version} of the {package_name} package cannot be installed because " - ) - error_message += "the recipe for installing the package is missing either an <actions> tag set or an <actions_group> " - error_message += "tag set." - # Since there was an installation error, update the tool dependency status to Error. - # The remove_installation_path option must be left False here. - tool_dependency = tool_dependency_util.set_tool_dependency_attributes( - self.app, - tool_dependency=tool_dependency, - status=self.app.install_model.ToolDependency.installation_status.ERROR, - error_message=error_message, - ) - else: - raise NotImplementedError( - 'Only install version 1.0 is currently supported (i.e., change your tag to be ).' - ) - return tool_dependency, proceed_with_install, actions_elem_tuples - - -class Package(RecipeTag): - def __init__(self, app): - self.app = app - self.tag = "package" - - def process_tag_set( - self, - tool_shed_repository, - tool_dependency, - package_elem, - package_name, - package_version, - tool_dependency_db_records=None, - ): - action_elem_tuples = [] - proceed_with_install = False - # Only install the tool_dependency if it is not already installed and it is associated with a database - # record in the received tool_dependencies. - if package_name and package_version: - dependencies_ignored = not self.app.toolbox.dependency_manager.uses_tool_shed_dependencies() - if dependencies_ignored: - log.debug( - "Skipping installation of tool dependency package %s because tool shed dependency resolver not enabled." - % str(package_name) - ) - # Tool dependency resolves have been configured and they do not include the tool shed. Do not install package. - dep = self.app.toolbox.dependency_manager.find_dep(package_name, package_version, type="package") - if not isinstance(dep, NullDependency): - # TODO: Do something here such as marking it installed or configured externally. - pass - tool_dependency = tool_dependency_util.set_tool_dependency_attributes( - self.app, - tool_dependency=tool_dependency, - status=self.app.install_model.ToolDependency.installation_status.ERROR, - ) - else: - proceed_with_install = True - return tool_dependency, proceed_with_install, action_elem_tuples - - -class ReadMe(RecipeTag): - def __init__(self, app): - self.app = app - self.tag = "readme" - - def process_tag_set( - self, - tool_shed_repository, - tool_dependency, - package_elem, - package_name, - package_version, - tool_dependency_db_records=None, - ): - # Nothing to be done. - action_elem_tuples = [] - proceed_with_install = False - return tool_dependency, proceed_with_install, action_elem_tuples - - -class Repository(RecipeTag, SyncDatabase): - def __init__(self, app): - self.app = app - self.tag = "repository" - - def create_temporary_tool_dependencies_config(self, tool_shed_url, name, owner, changeset_revision): - """Make a call to the tool shed to get the required repository's tool_dependencies.xml file.""" - tool_shed_url = get_tool_shed_url_from_tool_shed_registry(self.app, tool_shed_url) - if tool_shed_url is None or name is None or owner is None or changeset_revision is None: - message = ( - "Unable to retrieve required tool_dependencies.xml file from the Tool Shed because one or more of the " - ) - message += ( - "following required parameters is None: tool_shed_url: %s, name: %s, owner: %s, changeset_revision: %s " - % (str(tool_shed_url), str(name), str(owner), str(changeset_revision)) - ) - raise Exception(message) - params = dict(name=name, owner=owner, changeset_revision=changeset_revision) - pathspec = ["repository", "get_tool_dependencies_config_contents"] - text = url_get( - tool_shed_url, auth=self.app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params - ) - if text: - # Write the contents to a temporary file on disk so it can be reloaded and parsed. - fh = tempfile.NamedTemporaryFile("w", prefix="tmp-toolshed-cttdc") - tmp_filename = fh.name - fh.close() - fh = open(tmp_filename, "w") - fh.write(text) - fh.close() - return tmp_filename - else: - message = "Unable to retrieve required tool_dependencies.xml file from the Tool Shed for revision " - message += f"{changeset_revision} of installed repository {name} owned by {owner}." - raise Exception(message) - - def create_tool_dependency_with_initialized_env_sh_file( - self, - dependent_install_dir, - tool_shed_repository, - required_repository, - package_name, - package_version, - tool_dependencies_config, - ): - """ - Create or get a tool_dependency record that is defined by the received package_name and package_version. - An env.sh file will be created for the tool_dependency in the received dependent_install_dir. - """ - # The received required_repository refers to a tool_shed_repository record that is defined as a complex - # repository dependency for this tool_dependency. The required_repository may or may not be currently - # installed (it doesn't matter). If it is installed, it is associated with a tool_dependency that has - # an env.sh file that this new tool_dependency must be able to locate and "source". If it is not installed, - # we can still determine where that env.sh file will be, so we'll initialize this new tool_dependency's env.sh - # file in either case. If the required repository ends up with an installation error, this new tool - # dependency will still be fine because its containing repository will be defined as missing dependencies. - tool_dependencies = [] - if not os.path.exists(dependent_install_dir): - os.makedirs(dependent_install_dir) - required_tool_dependency_env_file_path = None - if tool_dependencies_config: - required_td_tree, error_message = parse_xml(tool_dependencies_config) - if required_td_tree: - required_td_root = required_td_tree.getroot() - for required_td_elem in required_td_root: - # Find the appropriate package name and version. - if required_td_elem.tag == "package": - # - required_td_package_name = required_td_elem.get("name", None) - required_td_package_version = required_td_elem.get("version", None) - # Check the database to see if we have a record for the required tool dependency (we may not which is ok). If we - # find a record, we need to see if it is in an error state and if so handle it appropriately. - required_tool_dependency = ( - tool_dependency_util.get_tool_dependency_by_name_version_type_repository( - self.app, - required_repository, - required_td_package_name, - required_td_package_version, - "package", - ) - ) - if required_td_package_name == package_name and required_td_package_version == package_version: - # Get or create a database tool_dependency record with which the installed package on disk will be associated. - tool_dependency = tool_dependency_util.create_or_update_tool_dependency( - app=self.app, - tool_shed_repository=tool_shed_repository, - name=package_name, - version=package_version, - type="package", - status=self.app.install_model.ToolDependency.installation_status.NEVER_INSTALLED, - set_status=True, - ) - # Create an env.sh file for the tool_dependency whose first line will source the env.sh file located in - # the path defined by required_tool_dependency_env_file_path. It doesn't matter if the required env.sh - # file currently exists.. - required_tool_dependency_env_file_path = self.get_required_repository_package_env_sh_path( - package_name, package_version, required_repository - ) - env_file_builder = EnvFileBuilder(tool_dependency.installation_directory(self.app)) - env_file_builder.append_line(action="source", value=required_tool_dependency_env_file_path) - return_code = env_file_builder.return_code - if return_code: - error_message = ( - f"Error defining env.sh file for package {package_name}, return_code: {return_code}" - ) - tool_dependency = tool_dependency_util.set_tool_dependency_attributes( - self.app, - tool_dependency=tool_dependency, - status=self.app.install_model.ToolDependency.installation_status.ERROR, - error_message=error_message, - ) - elif required_tool_dependency is not None and required_tool_dependency.in_error_state: - error_message = ( - "This tool dependency's required tool dependency %s version %s has status %s." - % ( - str(required_tool_dependency.name), - str(required_tool_dependency.version), - str(required_tool_dependency.status), - ) - ) - tool_dependency = tool_dependency_util.set_tool_dependency_attributes( - self.app, - tool_dependency=tool_dependency, - status=self.app.install_model.ToolDependency.installation_status.ERROR, - error_message=error_message, - ) - else: - tool_dependency = tool_dependency_util.set_tool_dependency_attributes( - self.app, - tool_dependency=tool_dependency, - status=self.app.install_model.ToolDependency.installation_status.INSTALLED, - ) - tool_dependencies.append(tool_dependency) - return tool_dependencies - - def get_required_repository_package_env_sh_path(self, package_name, package_version, required_repository): - """Return path to env.sh file in required repository if the required repository has been installed.""" - env_sh_file_dir = tool_dependency_util.get_tool_dependency_install_dir( - app=self.app, - repository_name=required_repository.name, - repository_owner=required_repository.owner, - repository_changeset_revision=required_repository.installed_changeset_revision, - tool_dependency_type="package", - tool_dependency_name=package_name, - tool_dependency_version=package_version, - ) - env_sh_file_path = os.path.join(env_sh_file_dir, "env.sh") - return env_sh_file_path - - def handle_complex_repository_dependency_for_package( - self, elem, package_name, package_version, tool_shed_repository - ): - """ - Inspect the repository defined by a complex repository dependency definition and take certain steps to - enable installation of the received package name and version to proceed. The received elem is the - tag set which defines the complex repository dependency. The received tool_shed_repository - is the installed tool shed repository for which the tool dependency defined by the received package_name - and package_version is being installed. - """ - handled_tool_dependencies = [] - tool_shed_url = elem.attrib["toolshed"] - required_repository_name = elem.attrib["name"] - required_repository_owner = elem.attrib["owner"] - default_required_repository_changeset_revision = elem.attrib["changeset_revision"] - required_repository = get_repository_for_dependency_relationship( - self.app, - tool_shed_url, - required_repository_name, - required_repository_owner, - default_required_repository_changeset_revision, - ) - tool_shed = remove_protocol_from_tool_shed_url(tool_shed_url) - tmp_filename = None - if required_repository: - required_repository_changeset_revision = required_repository.installed_changeset_revision - # Define the installation directory for the required tool dependency package in the required repository. - required_repository_package_install_dir = tool_dependency_util.get_tool_dependency_install_dir( - app=self.app, - repository_name=required_repository_name, - repository_owner=required_repository_owner, - repository_changeset_revision=required_repository_changeset_revision, - tool_dependency_type="package", - tool_dependency_name=package_name, - tool_dependency_version=package_version, - ) - # Define this dependent repository's tool dependency installation directory that will contain - # the env.sh file with a path to the required repository's installed tool dependency package. - dependent_install_dir = tool_dependency_util.get_tool_dependency_install_dir( - app=self.app, - repository_name=tool_shed_repository.name, - repository_owner=tool_shed_repository.owner, - repository_changeset_revision=tool_shed_repository.installed_changeset_revision, - tool_dependency_type="package", - tool_dependency_name=package_name, - tool_dependency_version=package_version, - ) - if os.path.exists(dependent_install_dir): - # Notice that we'll throw away the following tool_dependency if it can be installed. - tool_dependency, can_install_tool_dependency = self.sync_database_with_file_system( - self.app, - tool_shed_repository, - package_name, - package_version, - dependent_install_dir, - tool_dependency_type="package", - ) - if not can_install_tool_dependency: - log.debug( - "Tool dependency %s version %s cannot be installed (it was probably previously installed), " - "so appending it to the list of handled tool dependencies.", - str(tool_dependency.name), - str(tool_dependency.version), - ) - handled_tool_dependencies.append(tool_dependency) - else: - can_install_tool_dependency = True - if can_install_tool_dependency: - # Set this dependent repository's tool dependency env.sh file with a path to the required repository's - # installed tool dependency package. We can get everything we need from the discovered installed - # required_repository. - if required_repository.is_deactivated_or_installed: - if not os.path.exists(required_repository_package_install_dir): - log.error( - f"Missing required tool dependency directory {required_repository_package_install_dir}" - ) - repo_files_dir = required_repository.repo_files_directory(self.app) - if not repo_files_dir: - message = ( - "Unable to locate the repository directory for revision %s of installed repository %s owned by %s." - % ( - str(required_repository.changeset_revision), - str(required_repository.name), - str(required_repository.owner), - ) - ) - raise Exception(message) - tool_dependencies_config = get_absolute_path_to_file_in_repository( - repo_files_dir, "tool_dependencies.xml" - ) - if tool_dependencies_config: - config_to_use = tool_dependencies_config - else: - message = ( - "Unable to locate required tool_dependencies.xml file for revision %s of installed repository %s owned by %s." - % ( - str(required_repository.changeset_revision), - str(required_repository.name), - str(required_repository.owner), - ) - ) - raise Exception(message) - else: - # Make a call to the tool shed to get the changeset revision to which the current value of required_repository_changeset_revision - # should be updated if it's not current. - text = get_updated_changeset_revisions_from_tool_shed( - app=self.app, - tool_shed_url=tool_shed, - name=required_repository_name, - owner=required_repository_owner, - changeset_revision=required_repository_changeset_revision, - ) - if text: - updated_changeset_revisions = listify(text) - # The list of changeset revisions is in reverse order, so the newest will be first. - required_repository_changeset_revision = updated_changeset_revisions[0] - # Make a call to the tool shed to get the required repository's tool_dependencies.xml file. - tmp_filename = self.create_temporary_tool_dependencies_config( - tool_shed, - required_repository_name, - required_repository_owner, - required_repository_changeset_revision, - ) - config_to_use = tmp_filename - handled_tool_dependencies = self.create_tool_dependency_with_initialized_env_sh_file( - dependent_install_dir=dependent_install_dir, - tool_shed_repository=tool_shed_repository, - required_repository=required_repository, - package_name=package_name, - package_version=package_version, - tool_dependencies_config=config_to_use, - ) - self.remove_file(tmp_filename) - else: - message = f"Unable to locate required tool shed repository named {required_repository_name} owned by {required_repository_owner} with revision {default_required_repository_changeset_revision}." - raise Exception(message) - return handled_tool_dependencies - - def process_tag_set( - self, - tool_shed_repository, - tool_dependency, - package_elem, - package_name, - package_version, - tool_dependency_db_records=None, - ): - # We have a complex repository dependency definition. - action_elem_tuples = [] - proceed_with_install = False - rd_tool_dependencies = self.handle_complex_repository_dependency_for_package( - package_elem, package_name, package_version, tool_shed_repository - ) - for rd_tool_dependency in rd_tool_dependencies: - if rd_tool_dependency.status == self.app.install_model.ToolDependency.installation_status.ERROR: - # We'll log the error here, but continue installing packages since some may not require this dependency. - log.error( - f"Error installing tool dependency for required repository: {rd_tool_dependency.error_message}" - ) - return tool_dependency, proceed_with_install, action_elem_tuples - - def remove_file(self, file_name): - """Attempt to remove a file from disk.""" - if file_name: - if os.path.exists(file_name): - try: - os.remove(file_name) - except Exception: - pass - - -class SetEnvironment(RecipeTag): - def __init__(self, app): - self.app = app - self.tag = "set_environment" - - def process_tag_set( - self, - tool_shed_repository, - tool_dependency, - package_elem, - package_name, - package_version, - tool_dependency_db_records=None, - ): - # We need to handle two tag sets for package_elem here, this: - # - # $REPOSITORY_INSTALL_DIR - # - # or this: - # $REPOSITORY_INSTALL_DIR - action_elem_tuples = [] - proceed_with_install = False - if tool_dependency_db_records is None: - attr_tups_of_dependencies_for_install = [] - else: - attr_tups_of_dependencies_for_install = [ - (td.name, td.version, td.type) for td in tool_dependency_db_records - ] - try: - self.set_environment(package_elem, tool_shed_repository, attr_tups_of_dependencies_for_install) - except Exception as e: - error_message = f"Error setting environment for tool dependency: {e}" - log.debug(error_message) - return tool_dependency, proceed_with_install, action_elem_tuples - - def set_environment(self, elem, tool_shed_repository, attr_tups_of_dependencies_for_install): - """ - Create a ToolDependency to set an environment variable. This is different from the process used to - set an environment variable that is associated with a package. An example entry in a tool_dependencies.xml - file is:: - - - $REPOSITORY_INSTALL_DIR - - - This method must also handle the sub-element tag:: - $REPOSITORY_INSTALL_DIR - """ - # TODO: Add support for a repository dependency definition within this tool dependency type's tag set. This should look something like - # the following. See the implementation of support for this in the tool dependency package type's method above. - # This function is only called for set environment actions as defined below, not within an tool - # dependency type. Here is an example of the tag set this function does handle: - # - # $INSTALL_DIR - # - # Here is an example of the tag set this function does not handle: - # - # - # - env_manager = EnvManager(self.app) - tool_dependencies = [] - env_var_version = elem.get("version", "1.0") - tool_shed_repository_install_dir = os.path.abspath(tool_shed_repository.repo_files_directory(self.app)) - if elem.tag == "environment_variable": - # $REPOSITORY_INSTALL_DIR - elems = [elem] - else: - # - # $REPOSITORY_INSTALL_DIR - # - elems = [env_var_elem for env_var_elem in elem] - for env_var_elem in elems: - env_var_name = env_var_elem.get("name") - if not env_var_name: - raise Exception("The tag must have a name attribute") - # The value of env_var_name must match the text value of at least 1 tag in the - # tool config's tag set whose "type" attribute is "set_environment" (e.g., - # R_SCRIPT_PATH). - env_var_action = env_var_elem.get("action") - if not env_var_action: - raise Exception("The tag must have an action attribute") - # Tool dependencies of type "set_environment" always have the version attribute set to None. - attr_tup = (env_var_name, None, "set_environment") - if attr_tup in attr_tups_of_dependencies_for_install: - install_dir = tool_dependency_util.get_tool_dependency_install_dir( - app=self.app, - repository_name=tool_shed_repository.name, - repository_owner=tool_shed_repository.owner, - repository_changeset_revision=tool_shed_repository.installed_changeset_revision, - tool_dependency_type="set_environment", - tool_dependency_name=env_var_name, - tool_dependency_version=None, - ) - install_environment = InstallEnvironment( - app=self.app, - tool_shed_repository_install_dir=tool_shed_repository_install_dir, - install_dir=install_dir, - ) - env_var_dict = env_manager.create_env_var_dict( - elem=env_var_elem, install_environment=install_environment - ) - if not os.path.exists(install_dir): - os.makedirs(install_dir) - status = self.app.install_model.ToolDependency.installation_status.INSTALLING - tool_dependency = tool_dependency_util.create_or_update_tool_dependency( - app=self.app, - tool_shed_repository=tool_shed_repository, - name=env_var_name, - version=None, - type="set_environment", - status=status, - set_status=True, - ) - if env_var_version == "1.0": - # Create this tool dependency's env.sh file. - env_file_builder = EnvFileBuilder(install_dir) - return_code = env_file_builder.append_line(make_executable=True, **env_var_dict) - if return_code: - error_message = f"Error creating env.sh file for tool dependency {tool_dependency.name}, return_code: {return_code}" - log.debug(error_message) - status = self.app.install_model.ToolDependency.installation_status.ERROR - tool_dependency = tool_dependency_util.set_tool_dependency_attributes( - self.app, tool_dependency=tool_dependency, status=status, error_message=error_message - ) - else: - if tool_dependency.status not in [ - self.app.install_model.ToolDependency.installation_status.ERROR, - self.app.install_model.ToolDependency.installation_status.INSTALLED, - ]: - status = self.app.install_model.ToolDependency.installation_status.INSTALLED - tool_dependency = tool_dependency_util.set_tool_dependency_attributes( - self.app, tool_dependency=tool_dependency, status=status - ) - log.debug( - "Environment variable %s set in %s for tool dependency %s." - % (str(env_var_name), str(install_dir), str(tool_dependency.name)) - ) - else: - error_message = 'Only set_environment version 1.0 is currently supported (i.e., change your tag to be ).' - status = self.app.install_model.ToolDependency.installation_status.ERROR - tool_dependency = tool_dependency_util.set_tool_dependency_attributes( - self.app, tool_dependency=tool_dependency, status=status, error_message=error_message - ) - tool_dependencies.append(tool_dependency) - return tool_dependencies diff --git a/lib/galaxy/tool_shed/util/tool_dependency_util.py b/lib/galaxy/tool_shed/util/tool_dependency_util.py index a4c8b64d9b0..37215ec3ea8 100644 --- a/lib/galaxy/tool_shed/util/tool_dependency_util.py +++ b/lib/galaxy/tool_shed/util/tool_dependency_util.py @@ -5,8 +5,6 @@ import shutil from sqlalchemy import and_ from galaxy import util -from galaxy.tool_shed.util.hg_util import get_config_from_disk -from galaxy.util.tool_shed.xml_util import parse_xml from galaxy.web.form_builder import SelectField log = logging.getLogger(__name__) @@ -67,58 +65,6 @@ def create_or_update_tool_dependency(app, tool_shed_repository, name, version, t return tool_dependency -def create_tool_dependency_objects(app, tool_shed_repository, relative_install_dir, set_status=True): - """ - Create or update a ToolDependency for each entry in tool_dependencies_config. This method is called when - installing a new tool_shed_repository. - """ - tool_dependency_objects = [] - shed_config_dict = tool_shed_repository.get_shed_config_dict(app) - if shed_config_dict.get("tool_path"): - relative_install_dir = os.path.join(shed_config_dict.get("tool_path"), relative_install_dir) - # Get the tool_dependencies.xml file from the repository. - tool_dependencies_config = get_config_from_disk("tool_dependencies.xml", relative_install_dir) - tree, error_message = parse_xml(tool_dependencies_config) - if tree is None: - return tool_dependency_objects - root = tree.getroot() - for elem in root: - tool_dependency_type = elem.tag - if tool_dependency_type == "package": - name = elem.get("name", None) - version = elem.get("version", None) - if name and version: - status = app.install_model.ToolDependency.installation_status.NEVER_INSTALLED - tool_dependency = create_or_update_tool_dependency( - app, - tool_shed_repository, - name=name, - version=version, - type=tool_dependency_type, - status=status, - set_status=set_status, - ) - tool_dependency_objects.append(tool_dependency) - elif tool_dependency_type == "set_environment": - for env_elem in elem: - # $REPOSITORY_INSTALL_DIR - name = env_elem.get("name", None) - action = env_elem.get("action", None) - if name and action: - status = app.install_model.ToolDependency.installation_status.NEVER_INSTALLED - tool_dependency = create_or_update_tool_dependency( - app, - tool_shed_repository, - name=name, - version=None, - type=tool_dependency_type, - status=status, - set_status=set_status, - ) - tool_dependency_objects.append(tool_dependency) - return tool_dependency_objects - - def get_download_url_for_platform(url_templates, platform_info_dict): """ Compare the dict returned by get_platform_info() with the values specified in the url_template element. Return diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index fa02c41a9ed..c8b3d202fec 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -302,52 +302,6 @@ class AdminToolshed(AdminGalaxy): repo_information_dict = json.loads(raw_text) return repo_information_dict - @web.expose - @web.require_admin - @legacy_tool_shed_endpoint - def initiate_tool_dependency_installation(self, trans, tool_dependencies, **kwd): - """ - Install specified dependencies for repository tools. The received list of tool_dependencies - are the database records for those dependencies defined in the tool_dependencies.xml file - (contained in the repository) that should be installed. This allows for filtering out dependencies - that have not been checked for installation on the 'Manage tool dependencies' page for an installed - tool shed repository. - """ - # Get the tool_shed_repository from one of the tool_dependencies. - message = str(escape(kwd.get("message", ""))) - status = kwd.get("status", "done") - err_msg = "" - tool_shed_repository = tool_dependencies[0].tool_shed_repository - # Get the tool_dependencies.xml file from the repository. - tool_dependencies_config = hg_util.get_config_from_disk( - rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME, tool_shed_repository.repo_path(trans.app) - ) - itdm = install_manager.InstallToolDependencyManager(trans.app) - installed_tool_dependencies = itdm.install_specified_tool_dependencies( - tool_shed_repository=tool_shed_repository, - tool_dependencies_config=tool_dependencies_config, - tool_dependencies=tool_dependencies, - ) - for installed_tool_dependency in installed_tool_dependencies: - if installed_tool_dependency.status == trans.app.install_model.ToolDependency.installation_status.ERROR: - text = unicodify(installed_tool_dependency.error_message) - if text is not None: - err_msg += f" {text}" - if err_msg: - message += err_msg - status = "error" - message += f"Installed tool dependencies: {', '.join(td.name for td in installed_tool_dependencies)}" - td_ids = [trans.security.encode_id(td.id) for td in tool_shed_repository.tool_dependencies] - return trans.response.send_redirect( - web.url_for( - controller="admin_toolshed", - action="manage_tool_dependencies", - tool_dependency_ids=td_ids, - message=message, - status=status, - ) - ) - @web.expose @web.require_admin @legacy_tool_shed_endpoint @@ -436,10 +390,6 @@ class AdminToolshed(AdminGalaxy): else: tool_dependencies_dict = {} encoded_relative_install_dir = kwd.get("encoded_relative_install_dir", None) - if encoded_relative_install_dir is not None: - relative_install_dir = encoding_util.tool_shed_decode(encoded_relative_install_dir) - else: - relative_install_dir = "" updating_to_changeset_revision = kwd.get("updating_to_changeset_revision", None) updating_to_ctx_rev = kwd.get("updating_to_ctx_rev", None) encoded_updated_metadata = kwd.get("encoded_updated_metadata", None) @@ -458,27 +408,6 @@ class AdminToolshed(AdminGalaxy): updated_changeset_revision=updating_to_changeset_revision, updated_ctx_rev=updating_to_ctx_rev, ) - if tool_dependencies_dict: - tool_dependencies = tool_dependency_util.create_tool_dependency_objects( - trans.app, repository, relative_install_dir, set_status=False - ) - message = f"The installed repository named '{escape(str(repository.name))}' has been updated to change set revision '{updating_to_changeset_revision}'. " - self.initiate_tool_dependency_installation(trans, tool_dependencies, message=message, status=status) - # Handle tool dependencies check box. - if not trans.app.toolbox.dependency_manager.uses_tool_shed_dependencies: - if tool_dependencies_dict: - message = ( - "Tool dependencies defined in this repository can be automatically installed if you set " - "the value of your tool_dependency_dir setting in your Galaxy config file " - "(galaxy.yml) and restart your Galaxy server." - ) - status = "warning" - install_tool_dependencies_check_box_checked = False - else: - install_tool_dependencies_check_box_checked = True - install_tool_dependencies_check_box = CheckboxField( - "install_tool_dependencies", value=install_tool_dependencies_check_box_checked - ) view = views.DependencyResolversView(self.app) if view.installable_resolvers: install_resolver_dependencies_check_box = CheckboxField("install_resolver_dependencies", value=True) @@ -494,7 +423,6 @@ class AdminToolshed(AdminGalaxy): encoded_relative_install_dir=encoded_relative_install_dir, encoded_tool_dependencies_dict=encoded_tool_dependencies_dict, install_resolver_dependencies_check_box=install_resolver_dependencies_check_box, - install_tool_dependencies_check_box=install_tool_dependencies_check_box, tool_dependencies_dict=tool_dependencies_dict, message=message, status=status, diff --git a/lib/tool_shed/test/base/twilltestcase.py b/lib/tool_shed/test/base/twilltestcase.py index 6cd0d31df80..3aacbae0398 100644 --- a/lib/tool_shed/test/base/twilltestcase.py +++ b/lib/tool_shed/test/base/twilltestcase.py @@ -354,22 +354,6 @@ class ShedTwillTestCase(DrivenFunctionalTestCase): tool_panel_section == expected_tool_panel_section ), f"Expected to find tool panel section *{expected_tool_panel_section}*, but instead found *{tool_panel_section}*\nMetadata: {metadata}\n" - def check_installed_repository_tool_dependencies( - self, installed_repository, strings_displayed=None, strings_not_displayed=None, dependencies_installed=False - ): - # Tool dependencies are not being installed in these functional tests. If this is changed, the test method will also need to be updated. - if not dependencies_installed: - strings_displayed.append("Missing tool dependencies") - else: - strings_displayed.append("Tool dependencies") - if dependencies_installed: - strings_displayed.append("Installed") - else: - strings_displayed.append("Never installed") - params = {"id": self.security.encode_id(installed_repository.id)} - self.visit_galaxy_url("/admin_toolshed/manage_repository", params=params) - self.check_for_strings(strings_displayed, strings_not_displayed) - def check_repository_changelog(self, repository, strings_displayed=None, strings_not_displayed=None): params = {"id": self.security.encode_id(repository.id)} self.visit_url("/repository/view_changelog", params=params) @@ -898,13 +882,6 @@ class ShedTwillTestCase(DrivenFunctionalTestCase): datatypes = loads(html) return len(datatypes) - def get_env_sh_path(self, tool_dependency_name, tool_dependency_version, repository): - """Return the absolute path to an installed repository's env.sh file.""" - env_sh_path = os.path.join( - self.get_tool_dependency_path(tool_dependency_name, tool_dependency_version, repository), "env.sh" - ) - return env_sh_path - def get_filename(self, filename, filepath=None): if filepath is not None: return os.path.abspath(os.path.join(filepath, filename)) @@ -931,17 +908,6 @@ class ShedTwillTestCase(DrivenFunctionalTestCase): self.visit_url(url) self.check_for_strings(strings_displayed, strings_not_displayed) - def get_tool_dependency_path(self, tool_dependency_name, tool_dependency_version, repository): - """Return the absolute path for an installed tool dependency.""" - return os.path.join( - self.galaxy_tool_dependency_dir, - tool_dependency_name, - tool_dependency_version, - repository.owner, - repository.name, - repository.installed_changeset_revision, - ) - def get_or_create_repository(self, owner=None, strings_displayed=None, strings_not_displayed=None, **kwd): # If not checking for a specific string, it should be safe to assume that # we expect repository creation to be successful. diff --git a/lib/tool_shed/test/functional/test_1010_install_repository_with_tool_dependencies.py b/lib/tool_shed/test/functional/test_1010_install_repository_with_tool_dependencies.py index a3deab4e386..86b20ceb41c 100644 --- a/lib/tool_shed/test/functional/test_1010_install_repository_with_tool_dependencies.py +++ b/lib/tool_shed/test/functional/test_1010_install_repository_with_tool_dependencies.py @@ -161,10 +161,6 @@ class ToolWithToolDependencies(ShedTwillTestCase): self.display_galaxy_browse_repositories_page(strings_displayed=strings_displayed) strings_displayed.extend(["Installed tool shed repository", "Valid tools", "FreeBayes"]) self.display_installed_repository_manage_page(installed_repository, strings_displayed=strings_displayed) - strings_displayed = ["freebayes", "0.9.4_9696d0ce8a9", "samtools", "0.1.18"] - self.check_installed_repository_tool_dependencies( - installed_repository, strings_displayed=strings_displayed, dependencies_installed=False - ) self.verify_tool_metadata_for_installed_repository(installed_repository) def test_0020_verify_installed_repository_metadata(self): diff --git a/lib/tool_shed/test/functional/test_1020_install_repository_with_repository_dependencies.py b/lib/tool_shed/test/functional/test_1020_install_repository_with_repository_dependencies.py index 1d9bc7c1521..74f9222f84a 100644 --- a/lib/tool_shed/test/functional/test_1020_install_repository_with_repository_dependencies.py +++ b/lib/tool_shed/test/functional/test_1020_install_repository_with_repository_dependencies.py @@ -137,10 +137,6 @@ class ToolWithRepositoryDependencies(ShedTwillTestCase): self.display_galaxy_browse_repositories_page(strings_displayed=strings_displayed) strings_displayed.extend(["Installed tool shed repository", "Valid tools", "antigenic"]) self.display_installed_repository_manage_page(installed_repository, strings_displayed=strings_displayed) - strings_displayed = ["emboss", "5.0.0", "package"] - self.check_installed_repository_tool_dependencies( - installed_repository, strings_displayed=strings_displayed, dependencies_installed=False - ) self.verify_tool_metadata_for_installed_repository(installed_repository) current_datatypes = int(self.get_datatypes_count()) assert ( diff --git a/lib/tool_shed/test/functional/test_1030_install_repository_with_dependency_revisions.py b/lib/tool_shed/test/functional/test_1030_install_repository_with_dependency_revisions.py index 01f112ec772..e9b1f59497d 100644 --- a/lib/tool_shed/test/functional/test_1030_install_repository_with_dependency_revisions.py +++ b/lib/tool_shed/test/functional/test_1030_install_repository_with_dependency_revisions.py @@ -215,10 +215,6 @@ class RepositoryWithDependencyRevisions(ShedTwillTestCase): self.display_galaxy_browse_repositories_page(strings_displayed=strings_displayed) strings_displayed.extend(["Installed tool shed repository", "Valid tools", "antigenic"]) self.display_installed_repository_manage_page(installed_repository, strings_displayed=strings_displayed) - strings_displayed = ["emboss", "5.0.0", "package"] - self.check_installed_repository_tool_dependencies( - installed_repository, strings_displayed=strings_displayed, dependencies_installed=False - ) self.verify_tool_metadata_for_installed_repository(installed_repository) self.update_installed_repository(installed_repository, strings_displayed=["there are no updates available"]) current_datatypes = int(self.get_datatypes_count()) diff --git a/lib/tool_shed/test/functional/test_1120_install_repository_with_complex_dependencies.py b/lib/tool_shed/test/functional/test_1120_install_repository_with_complex_dependencies.py index 888acf969f4..3b230f3ecd0 100644 --- a/lib/tool_shed/test/functional/test_1120_install_repository_with_complex_dependencies.py +++ b/lib/tool_shed/test/functional/test_1120_install_repository_with_complex_dependencies.py @@ -321,49 +321,3 @@ class TestInstallingComplexRepositoryDependencies(ShedTwillTestCase): self.display_installed_repository_manage_page( base_repository, strings_displayed=strings_displayed, strings_not_displayed=strings_not_displayed ) - - def test_0055_verify_complex_tool_dependency(self): - """Verify that the generated env.sh contains the right data.""" - base_repository = self.test_db_util.get_installed_repository_by_name_owner( - bwa_base_repository_name, common.test_user_1_name - ) - tool_repository = self.test_db_util.get_installed_repository_by_name_owner( - bwa_package_repository_name, common.test_user_1_name - ) - env_sh_path = self.get_env_sh_path( - tool_dependency_name="bwa", tool_dependency_version="0.5.9", repository=base_repository - ) - assert os.path.exists(env_sh_path), f"env.sh was not generated in {env_sh_path} for this dependency." - contents = open(env_sh_path).read() - if tool_repository.installed_changeset_revision not in contents: - raise AssertionError( - "Installed changeset revision %s not found in env.sh.\nContents of env.sh: %s" - % (tool_repository.installed_changeset_revision, contents) - ) - if "package_bwa_0_5_9_0100" not in contents: - raise AssertionError( - f"Repository name package_bwa_0_5_9_0100 not found in env.sh.\nContents of env.sh: {contents}" - ) - - def test_0060_verify_tool_dependency_uninstallation(self): - """Uninstall the package_bwa_0_5_9_0100 repository. - - Uninstall the repository that defines a tool dependency relationship on BWA 0.5.9, and verify - that this results in the compiled binary package also being removed. - """ - tool_repository = self.test_db_util.get_installed_repository_by_name_owner( - bwa_package_repository_name, common.test_user_1_name - ) - self.deactivate_repository(tool_repository) - env_sh_path = os.path.join( - self.galaxy_tool_dependency_dir, - "bwa", - "0.5.9", - tool_repository.owner, - tool_repository.name, - tool_repository.installed_changeset_revision, - "env.sh", - ) - assert os.path.exists( - env_sh_path - ), f"Path {env_sh_path} does not exist after deactivating the repository that generated it." diff --git a/lib/tool_shed/test/functional/test_1420_tool_dependency_environment_inheritance.py b/lib/tool_shed/test/functional/test_1420_tool_dependency_environment_inheritance.py deleted file mode 100644 index 319a91678e4..00000000000 --- a/lib/tool_shed/test/functional/test_1420_tool_dependency_environment_inheritance.py +++ /dev/null @@ -1,411 +0,0 @@ -import logging - -from ..base.twilltestcase import ( - common, - ShedTwillTestCase, -) - -log = logging.getLogger(__name__) - -category_name = "Test 1420 Tool dependency environment variable inheritance" -category_description = "Test script 1420 for interpolation of inherited environment variables." -package_atlas_repository_name = "package_atlas_3_10_1420" -package_bzlib_repository_name = "package_bzlib_1_0_1420" -package_boost_repository_name = "package_boost_1_53_1420" -package_numpy_repository_name = "package_numpy_1_7_1420" -package_rdkit_repository_name = "package_rdkit_2012_12_1420" -package_lapack_repository_name = "package_lapack_3_4_1420" -package_atlas_repository_description = "Automatically Tuned Linear Algebra Software" -package_bzlib_repository_description = ( - "Contains a tool dependency definition that downloads and compiles version 1.0 of the bzlib library." -) -package_boost_repository_description = ( - "Contains a tool dependency definition that downloads and compiles version 1.53 of the boost C++ libraries" -) -package_numpy_repository_description = ( - "Contains a tool dependency definition that downloads and compiles version 1.7 of the the python numpy package" -) -package_rdkit_repository_description = "Contains a tool dependency definition that downloads and compiles version 2012-12 of the RDKit cheminformatics and machine-learning package." -package_lapack_repository_description = "Linear Algebra PACKage" -package_atlas_repository_long_description = f"{package_atlas_repository_name}: {package_atlas_repository_description}" -package_bzlib_repository_long_description = f"{package_bzlib_repository_name}: {package_bzlib_repository_description}" -package_boost_repository_long_description = f"{package_boost_repository_name}: {package_boost_repository_description}" -package_numpy_repository_long_description = f"{package_numpy_repository_name}: {package_numpy_repository_description}" -package_rdkit_repository_long_description = f"{package_rdkit_repository_name}: {package_rdkit_repository_description}" -package_lapack_repository_long_description = ( - f"{package_lapack_repository_name}: {package_lapack_repository_description}" -) - -""" -1. Create repository package_lapack_3_4_1420 - -2. Create repository package_atlas_3_10_1420 - -3. Create repository package_bzlib_1_0_1420 - -4. Create repository package_boost_1_53_1420 - -5. Create repository package_numpy_1_7_1420 - -6. Create repository package_rdkit_2012_12_1420 - -Repository dependency structure should be as follows: - Repository package_rdkit_2012_12_1420 - Repository package_boost_1_53_1420 (prior install required) - Repository package_bzlib_1_0_1420 (prior install required) - Repository package_numpy_1_7_1420 (prior install required) - Repository package_lapack_3_4_1420 (prior install required) - Repository package_atlas_3_10_1420 (prior install required) - -8. Install package_rdkit_2012_12 into Galaxy. - -9. Verify that the env.sh file for package_rdkit_2012_12_1420 also defines the variables inherited from package_numpy_1_7_1420 and package_boost_1_53_1420. -""" - - -class TestEnvironmentInheritance(ShedTwillTestCase): - """Test referencing environment variables that were defined in a separate tool dependency.""" - - def test_0000_initiate_users_and_category(self): - """Create necessary user accounts and login as an admin user.""" - self.login(email=common.admin_email, username=common.admin_username) - admin_user = self.test_db_util.get_user(common.admin_email) - assert admin_user is not None, f"Problem retrieving user with email {common.admin_email} from the database" - self.test_db_util.get_private_role(admin_user) - self.create_category(name=category_name, description=category_description) - self.login(email=common.test_user_2_email, username=common.test_user_2_name) - test_user_2 = self.test_db_util.get_user(common.test_user_2_email) - assert ( - test_user_2 is not None - ), f"Problem retrieving user with email {common.test_user_2_email} from the database" - self.test_db_util.get_private_role(test_user_2) - self.login(email=common.test_user_1_email, username=common.test_user_1_name) - test_user_1 = self.test_db_util.get_user(common.test_user_1_email) - assert ( - test_user_1 is not None - ), f"Problem retrieving user with email {common.test_user_1_email} from the database" - self.test_db_util.get_private_role(test_user_1) - - def test_0005_create_lapack_repository(self): - """Create and populate package_lapack_3_4_1420. - - This is step 1 - Create repository package_lapack_3_4_1420. - - All tool dependency definitions should download and extract a tarball containing precompiled binaries from the local - filesystem and install them into the path specified by $INSTALL_DIR. - """ - category = self.test_db_util.get_category_by_name(category_name) - repository = self.get_or_create_repository( - name=package_lapack_repository_name, - description=package_lapack_repository_description, - long_description=package_lapack_repository_long_description, - owner=common.test_user_1_name, - category_id=self.security.encode_id(category.id), - strings_displayed=[], - ) - # Load the original tool dependency definition into memory, then fill in the __PATH__ placeholder with the - # actual system path where the binary tarball is found. - tool_dependency_path = self.generate_temp_path( - "1420_tool_dependency", additional_paths=["package_lapack_3_4_1420"] - ) - precompiled_binary_tarball = self.get_filename("1420_files/binary_tarballs/lapack.tar") - edited_tool_dependency_filename = self.get_filename( - filepath=tool_dependency_path, filename="tool_dependencies.xml" - ) - original_tool_dependency = self.get_filename("1420_files/package_lapack_3_4_1420/tool_dependencies.xml") - tool_dependency_definition = ( - open(original_tool_dependency).read().replace("__PATH__", precompiled_binary_tarball) - ) - open(edited_tool_dependency_filename, "w").write(tool_dependency_definition) - # Upload the edited tool dependency definition to the package_lapack_3_4_1420 repository. - self.upload_file( - repository, - filename="tool_dependencies.xml", - filepath=tool_dependency_path, - valid_tools_only=True, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message="Populate package_lapack_3_4_1420 with tool dependency definitions.", - strings_displayed=[], - strings_not_displayed=[], - ) - - def test_0010_create_atlas_repository(self): - """Create and populate package_atlas_3_10_1420. - - This is step 1 - Create repository package_atlas_3_10_1420. - - All tool dependency definitions should download and extract a tarball containing precompiled binaries from the local - filesystem and install them into the path specified by $INSTALL_DIR. - """ - category = self.test_db_util.get_category_by_name(category_name) - repository = self.get_or_create_repository( - name=package_atlas_repository_name, - description=package_atlas_repository_description, - long_description=package_atlas_repository_long_description, - owner=common.test_user_1_name, - category_id=self.security.encode_id(category.id), - strings_displayed=[], - ) - # Load the original tool dependency definition into memory, then fill in the __PATH__ placeholder with the - # actual system path where the binary tarball is found. - tool_dependency_path = self.generate_temp_path( - "1420_tool_dependency", additional_paths=["package_atlas_3_10_1420"] - ) - precompiled_binary_tarball = self.get_filename("1420_files/binary_tarballs/atlas.tar") - edited_tool_dependency_filename = self.get_filename( - filepath=tool_dependency_path, filename="tool_dependencies.xml" - ) - original_tool_dependency = self.get_filename("1420_files/package_atlas_3_10_1420/tool_dependencies.xml") - tool_dependency_definition = ( - open(original_tool_dependency).read().replace("__PATH__", precompiled_binary_tarball) - ) - open(edited_tool_dependency_filename, "w").write(tool_dependency_definition) - # Upload the edited tool dependency definition to the package_atlas_3_10_1420 repository. - self.upload_file( - repository, - filename="tool_dependencies.xml", - filepath=tool_dependency_path, - valid_tools_only=True, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message="Populate package_atlas_3_10_1420 with tool dependency definitions.", - strings_displayed=[], - strings_not_displayed=[], - ) - - def test_0015_create_bzlib_repository(self): - """Create and populate package_bzlib_1_0_1420. - - This is step 1 - Create repository package_bzlib_1_0_1420. - - All tool dependency definitions should download and extract a tarball containing precompiled binaries from the local - filesystem and install them into the path specified by $INSTALL_DIR. - """ - category = self.test_db_util.get_category_by_name(category_name) - repository = self.get_or_create_repository( - name=package_bzlib_repository_name, - description=package_bzlib_repository_description, - long_description=package_bzlib_repository_long_description, - owner=common.test_user_1_name, - category_id=self.security.encode_id(category.id), - strings_displayed=[], - ) - # Load the original tool dependency definition into memory, then fill in the __PATH__ placeholder with the - # actual system path where the binary tarball is found. - tool_dependency_path = self.generate_temp_path( - "1420_tool_dependency", additional_paths=["package_bzlib_1_0_1420"] - ) - precompiled_binary_tarball = self.get_filename("1420_files/binary_tarballs/bzlib.tar") - edited_tool_dependency_filename = self.get_filename( - filepath=tool_dependency_path, filename="tool_dependencies.xml" - ) - original_tool_dependency = self.get_filename("1420_files/package_bzlib_1_0_1420/tool_dependencies.xml") - tool_dependency_definition = ( - open(original_tool_dependency).read().replace("__PATH__", precompiled_binary_tarball) - ) - open(edited_tool_dependency_filename, "w").write(tool_dependency_definition) - # Upload the edited tool dependency definition to the package_bzlib_1_0_1420 repository. - self.upload_file( - repository, - filename="tool_dependencies.xml", - filepath=tool_dependency_path, - valid_tools_only=True, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message="Populate package_bzlib_1_0_1420 with tool dependency definitions.", - strings_displayed=[], - strings_not_displayed=[], - ) - - def test_0020_create_boost_repository(self): - """Create and populate package_boost_1_53_1420. - - This is step 1 - Create repository package_boost_1_53_1420. - - All tool dependency definitions should download and extract a tarball containing precompiled binaries from the local - filesystem and install them into the path specified by $INSTALL_DIR. - """ - category = self.test_db_util.get_category_by_name(category_name) - repository = self.get_or_create_repository( - name=package_boost_repository_name, - description=package_boost_repository_description, - long_description=package_boost_repository_long_description, - owner=common.test_user_1_name, - category_id=self.security.encode_id(category.id), - strings_displayed=[], - ) - # Load the original tool dependency definition into memory, then fill in the __PATH__ placeholder with the - # actual system path where the binary tarball is found. - tool_dependency_path = self.generate_temp_path( - "1420_tool_dependency", additional_paths=["package_boost_1_53_1420"] - ) - precompiled_binary_tarball = self.get_filename("1420_files/binary_tarballs/boost.tar") - edited_tool_dependency_filename = self.get_filename( - filepath=tool_dependency_path, filename="tool_dependencies.xml" - ) - original_tool_dependency = self.get_filename("1420_files/package_boost_1_53_1420/tool_dependencies.xml") - tool_dependency_definition = ( - open(original_tool_dependency).read().replace("__PATH__", precompiled_binary_tarball) - ) - open(edited_tool_dependency_filename, "w").write(tool_dependency_definition) - # Upload the edited tool dependency definition to the package_boost_1_53_1420 repository. - self.upload_file( - repository, - filename="tool_dependencies.xml", - filepath=tool_dependency_path, - valid_tools_only=True, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message="Populate package_boost_1_53_1420 with tool dependency definitions.", - strings_displayed=[], - strings_not_displayed=[], - ) - - def test_0025_create_numpy_repository(self): - """Create and populate package_numpy_1_7_1420. - - This is step 1 - Create repository package_numpy_1_7_1420. - - All tool dependency definitions should download and extract a tarball containing precompiled binaries from the local - filesystem and install them into the path specified by $INSTALL_DIR. - """ - category = self.test_db_util.get_category_by_name(category_name) - repository = self.get_or_create_repository( - name=package_numpy_repository_name, - description=package_numpy_repository_description, - long_description=package_numpy_repository_long_description, - owner=common.test_user_1_name, - category_id=self.security.encode_id(category.id), - strings_displayed=[], - ) - # Load the original tool dependency definition into memory, then fill in the __PATH__ placeholder with the - # actual system path where the binary tarball is found. - tool_dependency_path = self.generate_temp_path( - "1420_tool_dependency", additional_paths=["package_numpy_1_7_1420"] - ) - precompiled_binary_tarball = self.get_filename("1420_files/binary_tarballs/numpy.tar") - edited_tool_dependency_filename = self.get_filename( - filepath=tool_dependency_path, filename="tool_dependencies.xml" - ) - original_tool_dependency = self.get_filename("1420_files/package_numpy_1_7_1420/tool_dependencies.xml") - tool_dependency_definition = ( - open(original_tool_dependency).read().replace("__PATH__", precompiled_binary_tarball) - ) - open(edited_tool_dependency_filename, "w").write(tool_dependency_definition) - # Upload the edited tool dependency definition to the package_numpy_1_7_1420 repository. - self.upload_file( - repository, - filename="tool_dependencies.xml", - filepath=tool_dependency_path, - valid_tools_only=True, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message="Populate package_numpy_1_7_1420 with tool dependency definitions.", - strings_displayed=[], - strings_not_displayed=[], - ) - - def test_0030_create_rdkit_repository(self): - """Create and populate package_rdkit_2012_12_1420. - - This is step 1 - Create repository package_rdkit_2012_12_1420. - - All tool dependency definitions should download and extract a tarball containing precompiled binaries from the local - filesystem and install them into the path specified by $INSTALL_DIR. - """ - category = self.test_db_util.get_category_by_name(category_name) - repository = self.get_or_create_repository( - name=package_rdkit_repository_name, - description=package_rdkit_repository_description, - long_description=package_rdkit_repository_long_description, - owner=common.test_user_1_name, - category_id=self.security.encode_id(category.id), - strings_displayed=[], - ) - # Load the original tool dependency definition into memory, then fill in the __PATH__ placeholder with the - # actual system path where the binary tarball is found. - tool_dependency_path = self.generate_temp_path( - "1420_tool_dependency", additional_paths=["package_rdkit_2012_12_1420"] - ) - precompiled_binary_tarball = self.get_filename("1420_files/binary_tarballs/rdkit.tar") - edited_tool_dependency_filename = self.get_filename( - filepath=tool_dependency_path, filename="tool_dependencies.xml" - ) - original_tool_dependency = self.get_filename("1420_files/package_rdkit_2012_12_1420/tool_dependencies.xml") - tool_dependency_definition = ( - open(original_tool_dependency).read().replace("__PATH__", precompiled_binary_tarball) - ) - open(edited_tool_dependency_filename, "w").write(tool_dependency_definition) - # Upload the edited tool dependency definition to the package_rdkit_2012_12_1420 repository. - self.upload_file( - repository, - filename="tool_dependencies.xml", - filepath=tool_dependency_path, - valid_tools_only=True, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message="Populate package_rdkit_2012_12_1420 with tool dependency definitions.", - strings_displayed=[], - strings_not_displayed=[], - ) - - def test_0035_install_rdkit_2012_12_repository(self): - """Install the package_rdkit_2012_12_1420 repository into Galaxy. - - This is step 4 - Install package_rdkit_2012_12_1420 into Galaxy. - - Install package_rdkit_2012_12_1420 with tool dependencies selected to be installed. The result of this should be - package_atlas_3_10_1420, package_bzlib_1_0_1420, package_boost_1_53_1420, package_numpy_1_7_1420, package_rdkit_2012_12_1420, - and package_lapack_3_4_1420 being installed, and an env.sh generated for package_rdkit_2012_12_1420 that - contains environment variables defined in package_boost_1_53_1420 and package_numpy_1_7_1420. - """ - self.galaxy_login(email=common.admin_email, username=common.admin_username) - post_submit_strings_displayed = [ - "package_rdkit_2012_12_1420", - "package_atlas_3_10_1420", - "package_bzlib_1_0_1420", - "package_numpy_1_7_1420", - "package_lapack_3_4_1420", - "package_boost_1_53_1420", - ] - self.install_repository( - "package_rdkit_2012_12_1420", - common.test_user_1_name, - category_name, - install_tool_dependencies=True, - post_submit_strings_displayed=post_submit_strings_displayed, - ) - - def test_0040_verify_env_sh_contents(self): - """Check the env.sh file for the appropriate contents. - - This is step 5 - Verify that the env.sh file for package_rdkit_2012_12_1420 also defines the variables inherited from package_numpy_1_7_1420 - and package_boost_1_53_1420. Test for the numpy and boost tool dependency paths. - """ - package_rdkit_repository = self.test_db_util.get_installed_repository_by_name_owner( - "package_rdkit_2012_12_1420", common.test_user_1_name - ) - package_numpy_repository = self.test_db_util.get_installed_repository_by_name_owner( - "package_numpy_1_7_1420", common.test_user_1_name - ) - package_boost_repository = self.test_db_util.get_installed_repository_by_name_owner( - "package_boost_1_53_1420", common.test_user_1_name - ) - rdkit_env_sh = self.get_env_sh_path( - tool_dependency_name="rdkit", tool_dependency_version="2012_12_1", repository=package_rdkit_repository - ) - numpy_tool_dependency_path = self.get_tool_dependency_path( - tool_dependency_name="numpy", tool_dependency_version="1.7.1", repository=package_numpy_repository - ) - boost_tool_dependency_path = self.get_tool_dependency_path( - tool_dependency_name="boost", tool_dependency_version="1.53.0", repository=package_boost_repository - ) - rdkit_env_file_contents = open(rdkit_env_sh).read() - if ( - numpy_tool_dependency_path not in rdkit_env_file_contents - or boost_tool_dependency_path not in rdkit_env_file_contents - ): - message = "Environment file for package_rdkit_2012_12_1420 does not contain expected path." - message += f"\nExpected:\n{numpy_tool_dependency_path}\n{boost_tool_dependency_path}\nContents:\n{rdkit_env_file_contents}" - raise AssertionError(message) diff --git a/lib/tool_shed/test/functional/test_1440_missing_env_sh_files.py b/lib/tool_shed/test/functional/test_1440_missing_env_sh_files.py deleted file mode 100644 index 4c9415c7f47..00000000000 --- a/lib/tool_shed/test/functional/test_1440_missing_env_sh_files.py +++ /dev/null @@ -1,146 +0,0 @@ -import logging - -from ..base.twilltestcase import ( - common, - ShedTwillTestCase, -) - -log = logging.getLogger(__name__) - -category_name = "Test 1440 Tool dependency missing env.sh" -category_description = "Test script 1440 for detection of missing environment settings." -package_repository_name = "package_env_sh_1_0_1440" -tool_repository_name = "filter_1440" -package_repository_description = "Repository that should result in an env.sh file, but does not." -tool_repository_description = "Galaxy filtering tool." -package_repository_long_description = f"{package_repository_name}: {package_repository_description}" -tool_repository_long_description = f"{tool_repository_name}: {tool_repository_description}" - -""" -1. Create a tool dependency type repository that reliably fails to install successfully. This repository should define - an action that would have created an env.sh file on success, resulting in an env.sh file that should exist, but is missing. - -2. Create a repository that defines a complex repository dependency in the repository created in step 1, with prior_install_required - and set_environment_for_install. - -3. Attempt to install the second repository into a galaxy instance, verify that it is installed but missing tool dependencies. - -""" - - -class TestMissingEnvSh(ShedTwillTestCase): - """Test installing a repository that should create an env.sh file, but does not.""" - - def test_0000_initiate_users_and_category(self): - """Create necessary user accounts and login as an admin user.""" - self.login(email=common.admin_email, username=common.admin_username) - admin_user = self.test_db_util.get_user(common.admin_email) - assert admin_user is not None, f"Problem retrieving user with email {common.admin_email} from the database" - self.test_db_util.get_private_role(admin_user) - self.create_category(name=category_name, description=category_description) - self.login(email=common.test_user_2_email, username=common.test_user_2_name) - test_user_2 = self.test_db_util.get_user(common.test_user_2_email) - assert ( - test_user_2 is not None - ), f"Problem retrieving user with email {common.test_user_2_email} from the database" - self.test_db_util.get_private_role(test_user_2) - self.login(email=common.test_user_1_email, username=common.test_user_1_name) - test_user_1 = self.test_db_util.get_user(common.test_user_1_email) - assert ( - test_user_1 is not None - ), f"Problem retrieving user with email {common.test_user_1_email} from the database" - self.test_db_util.get_private_role(test_user_1) - - def test_0005_create_package_repository(self): - """Create and populate package_env_sh_1_0_1440. - - This is step 1 - Create repository package_env_sh_1_0_1440. - - Create and populate a repository that is designed to fail a tool dependency installation. This tool dependency should - also define one or more environment variables. - """ - category = self.test_db_util.get_category_by_name(category_name) - repository = self.get_or_create_repository( - name=package_repository_name, - description=package_repository_description, - long_description=package_repository_long_description, - owner=common.test_user_1_name, - category_id=self.security.encode_id(category.id), - strings_displayed=[], - ) - # Upload the edited tool dependency definition to the package_lapack_3_4_1440 repository. - self.upload_file( - repository, - filename="1440_files/dependency_definition/tool_dependencies.xml", - filepath=None, - valid_tools_only=True, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message="Populate package_env_sh_1_0_1440 with a broken tool dependency definition.", - strings_displayed=[], - strings_not_displayed=[], - ) - - def test_0010_create_filter_repository(self): - """Create and populate filter_1440. - - This is step 2 - Create a repository that defines a complex repository dependency on the repository created in - step 1, with prior_install_required and set_environment_for_install. - """ - category = self.test_db_util.get_category_by_name(category_name) - repository = self.get_or_create_repository( - name=tool_repository_name, - description=tool_repository_description, - long_description=tool_repository_long_description, - owner=common.test_user_1_name, - category_id=self.security.encode_id(category.id), - strings_displayed=[], - ) - # Upload the edited tool dependency definition to the package_lapack_3_4_1440 repository. - self.upload_file( - repository, - filename="filtering/filtering_2.2.0.tar", - filepath=None, - valid_tools_only=True, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message="Populate filter_1440 with the filtering tool.", - strings_displayed=[], - strings_not_displayed=[], - ) - self.upload_file( - repository, - filename="1440_files/complex_dependency/tool_dependencies.xml", - filepath=None, - valid_tools_only=True, - uncompress_file=False, - remove_repo_files_not_in_tar=False, - commit_message="Populate filter_1440 with a dependency on package_env_sh_1_0_1440.", - strings_displayed=[], - strings_not_displayed=[], - ) - - def test_0015_install_filter_repository(self): - """Install the filter_1440 repository to galaxy. - - This is step 3 - Attempt to install the second repository into a galaxy instance, verify that it is installed but - missing tool dependencies. - """ - self.galaxy_login(email=common.admin_email, username=common.admin_username) - post_submit_strings_displayed = ["filter_1440", "package_env_sh_1_0_1440"] - self.install_repository( - "filter_1440", - common.test_user_1_name, - category_name, - install_tool_dependencies=True, - post_submit_strings_displayed=post_submit_strings_displayed, - ) - - def test_0020_verify_missing_tool_dependency(self): - """Verify that the filter_1440 repository is installed and missing tool dependencies.""" - repository = self.test_db_util.get_installed_repository_by_name_owner("filter_1440", common.test_user_1_name) - strings_displayed = ["Missing tool dependencies"] - self.display_installed_repository_manage_page(repository, strings_displayed=strings_displayed) - assert ( - len(repository.missing_tool_dependencies) == 1 - ), "filter_1440 should have a missing tool dependency, but does not." diff --git a/lib/tool_shed/webapp/api/repositories.py b/lib/tool_shed/webapp/api/repositories.py index 17e7ad9989d..4f372ef74bd 100644 --- a/lib/tool_shed/webapp/api/repositories.py +++ b/lib/tool_shed/webapp/api/repositories.py @@ -872,10 +872,6 @@ class RepositoriesController(BaseAPIController): ) else: metadata_dict["repository_dependencies"] = [] - if metadata.includes_tool_dependencies and recursive: - metadata_dict["tool_dependencies"] = repository.get_tool_dependencies(self.app, changehash) - else: - metadata_dict["tool_dependencies"] = {} if metadata.includes_tools: metadata_dict["tools"] = metadata.metadata["tools"] all_metadata[f"{int(changeset)}:{changehash}"] = metadata_dict diff --git a/packages/app/setup.py b/packages/app/setup.py index 6c6e9756017..bc9dc624e7b 100644 --- a/packages/app/setup.py +++ b/packages/app/setup.py @@ -68,8 +68,6 @@ PACKAGES = [ "galaxy.tool_shed.galaxy_install.datatypes", "galaxy.tool_shed.galaxy_install.metadata", "galaxy.tool_shed.galaxy_install.repository_dependencies", - "galaxy.tool_shed.galaxy_install.tool_dependencies", - "galaxy.tool_shed.galaxy_install.tool_dependencies.recipe", "galaxy.tool_shed.galaxy_install.tools", "galaxy.tool_shed.util", "galaxy.tool_shed.metadata", diff --git a/templates/admin/tool_shed_repository/install_tool_dependencies_with_update.mako b/templates/admin/tool_shed_repository/install_tool_dependencies_with_update.mako index 366eeb3bd3c..3ded80b2ee4 100644 --- a/templates/admin/tool_shed_repository/install_tool_dependencies_with_update.mako +++ b/templates/admin/tool_shed_repository/install_tool_dependencies_with_update.mako @@ -28,21 +28,6 @@ ${render_galaxy_repository_actions( repository )} %if tool_dependencies_dict: - %if install_tool_dependencies_check_box is not None: -
- - <% disabled = trans.app.tool_dependency_dir is None %> - ${render_checkbox(install_tool_dependencies_check_box, disabled=disabled)} -
- %if disabled: - Set the tool_dependency_dir configuration value in your Galaxy config to automatically handle tool dependencies. - %else: - Un-check to skip automatic handling of these tool dependencies. - %endif -
-
-
- %endif
diff --git a/test/unit/shed_unit/test_fabric_util.py b/test/unit/shed_unit/test_fabric_util.py deleted file mode 100644 index 2785ca388ad..00000000000 --- a/test/unit/shed_unit/test_fabric_util.py +++ /dev/null @@ -1,46 +0,0 @@ -from contextlib import contextmanager - -from galaxy.tool_shed.galaxy_install.tool_dependencies.recipe.env_file_builder import EnvFileBuilder - - -def test_env_file_builder(): - install_dir = "/opt/galaxy/dependencies/foo/" - env_file_builder = EnvFileBuilder(install_dir) - added_lines = [] - mock_return = dict(value=0) - - def mock_file_append(text, file_path, **kwds): - added_lines.append(text) - return mock_return["value"] - - with __mock_env_file_builder_method(env_file_builder, "file_append", mock_file_append): - env_file_builder.append_line(name="PATH", action="prepend_to", value="/usr/bin/local/R") - assert added_lines == ["PATH=/usr/bin/local/R:$PATH; export PATH"] - assert env_file_builder.return_code == 0 - - # Reset mock lines - del added_lines[:] - # Next time file_append will fail - mock_return["value"] = 1 - - env_file_builder.append_line(action="source", value="/usr/bin/local/R/env.sh") - assert added_lines == ["if [ -f /usr/bin/local/R/env.sh ] ; then . /usr/bin/local/R/env.sh ; fi"] - # Check failure - assert env_file_builder.return_code == 1 - - mock_return["value"] = 0 - env_file_builder.append_line(name="LD_LIBRARY_PATH", action="append_to", value="/usr/bin/local/R/lib") - # Verify even though last append succeeded, previous failure still recorded. - assert env_file_builder.return_code == 1 - - -# Poor man's mocking. Need to get a real mocking library as real Galaxy development -# dependnecy. -@contextmanager -def __mock_env_file_builder_method(env_file_builder, name, mock_method): - real_method = getattr(env_file_builder, name) - try: - setattr(env_file_builder, name, mock_method) - yield - finally: - setattr(env_file_builder, name, real_method) diff --git a/test/unit/shed_unit/test_td_common_util.py b/test/unit/shed_unit/test_td_common_util.py deleted file mode 100644 index ad212f9ae50..00000000000 --- a/test/unit/shed_unit/test_td_common_util.py +++ /dev/null @@ -1,99 +0,0 @@ -from contextlib import contextmanager -from os.path import join -from typing import ( - Any, - Dict, -) - -from galaxy.tool_shed.galaxy_install.tool_dependencies.env_manager import EnvManager -from galaxy.tool_shed.galaxy_install.tool_dependencies.recipe.env_file_builder import EnvFileBuilder -from galaxy.util import parse_xml_string - -TEST_DEPENDENCIES_DIR = "/opt/galaxy/dependencies" -TEST_INSTALL_DIR = "%s/test_install_dir" % TEST_DEPENDENCIES_DIR - - -class MockApp: - def __init__(self): - pass - - -def test_create_or_update_env_shell_file(): - test_path = "/usr/share/R/libs" - env_file_builder = EnvFileBuilder(test_path) - line, path = env_file_builder.create_or_update_env_shell_file( - TEST_INSTALL_DIR, dict(action="append_to", name="R_LIBS", value=test_path) - ) - assert path == join(TEST_INSTALL_DIR, "env.sh") - assert line == "R_LIBS=$R_LIBS:/usr/share/R/libs; export R_LIBS" - - line, path = env_file_builder.create_or_update_env_shell_file( - TEST_INSTALL_DIR, dict(action="prepend_to", name="R_LIBS", value=test_path) - ) - assert path == join(TEST_INSTALL_DIR, "env.sh") - assert line == "R_LIBS=/usr/share/R/libs:$R_LIBS; export R_LIBS" - - line, path = env_file_builder.create_or_update_env_shell_file( - TEST_INSTALL_DIR, dict(action="set_to", name="R_LIBS", value=test_path) - ) - assert path == join(TEST_INSTALL_DIR, "env.sh") - assert line == "R_LIBS=/usr/share/R/libs; export R_LIBS" - - line, path = env_file_builder.create_or_update_env_shell_file( - TEST_INSTALL_DIR, dict(action="source", value=test_path) - ) - assert path == join(TEST_INSTALL_DIR, "env.sh") - assert line == "if [ -f /usr/share/R/libs ] ; then . /usr/share/R/libs ; fi" - - -def test_get_env_shell_file_paths_from_setup_environment_elem(): - xml = """ - - - - - - - - """ - mock_app = MockApp() - action_elem = parse_xml_string(xml) - required_for_install_env_sh = "/path/to/existing.sh" - all_env_paths = [required_for_install_env_sh] - action_dict: Dict[str, Any] = {} - env_manager = EnvManager(mock_app) - - r_env_sh = "/path/to/go/env.sh" - - def mock_get_env_shell_file_paths(elem): - assert elem.get("name") in ["package_r_3_0_1", "package_zlib_1_2_8"] - return [r_env_sh] - - with __mock_common_util_method(env_manager, "get_env_shell_file_paths", mock_get_env_shell_file_paths): - env_manager.get_env_shell_file_paths_from_setup_environment_elem(all_env_paths, action_elem, action_dict) - # Verify old env files weren't deleted. - assert required_for_install_env_sh in all_env_paths - # Verify new ones added. - assert r_env_sh in all_env_paths - # env_shell_file_paths includes everything - assert all(env in action_dict["env_shell_file_paths"] for env in all_env_paths) - # for every given repository there should be one env - # file + the required_for_install_env_sh file - assert len(action_dict["env_shell_file_paths"]) == 3 - - # action_shell_file_paths includes only env files defined - # inside the setup_ action element. - assert required_for_install_env_sh in action_dict["action_shell_file_paths"] - assert r_env_sh in action_dict["action_shell_file_paths"] - - -# Poor man's mocking. Need to get a real mocking library as real Galaxy development -# dependnecy. -@contextmanager -def __mock_common_util_method(env_manager, name, mock_method): - real_method = getattr(env_manager, name) - try: - setattr(env_manager, name, mock_method) - yield - finally: - setattr(env_manager, name, real_method) From 362a1807ee1da61e8268e16ad08ab8e06daec746 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 26 May 2022 11:11:43 -0400 Subject: [PATCH 2/5] More cleanup tool_dependencies.xml handling. --- .../galaxy_install/install_manager.py | 12 ----- .../galaxy/controllers/admin_toolshed.py | 47 ------------------- ...stall_repository_with_tool_dependencies.py | 2 - ..._repository_basic_circular_dependencies.py | 2 - ...stall_repository_with_tool_dependencies.py | 9 +--- .../admin/tool_shed_repository/common.mako | 16 +------ .../select_tool_panel_section.mako | 2 +- .../admin/tool_shed_repository/common.mako | 16 +------ .../reselect_tool_panel_section.mako | 2 +- .../select_tool_panel_section.mako | 2 +- 10 files changed, 6 insertions(+), 104 deletions(-) diff --git a/lib/galaxy/tool_shed/galaxy_install/install_manager.py b/lib/galaxy/tool_shed/galaxy_install/install_manager.py index 0b34531a41d..806cf2e1db9 100644 --- a/lib/galaxy/tool_shed/galaxy_install/install_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/install_manager.py @@ -667,7 +667,6 @@ class InstallRepositoryManager: original_metadata_dict = repository.metadata_ original_repository_dependencies_dict = original_metadata_dict.get("repository_dependencies", {}) original_repository_dependencies = original_repository_dependencies_dict.get("repository_dependencies", []) - original_tool_dependencies_dict = original_metadata_dict.get("tool_dependencies", {}) shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( self.app, repository ) @@ -740,7 +739,6 @@ class InstallRepositoryManager: if "repository_dependencies" in irmm_metadata_dict or "tool_dependencies" in irmm_metadata_dict: new_repository_dependencies_dict = irmm_metadata_dict.get("repository_dependencies", {}) new_repository_dependencies = new_repository_dependencies_dict.get("repository_dependencies", []) - new_tool_dependencies_dict = irmm_metadata_dict.get("tool_dependencies", {}) if new_repository_dependencies: # [[http://localhost:9009', package_picard_1_56_0', devteam', 910b0b056666', False', False']] if new_repository_dependencies == original_repository_dependencies: @@ -792,16 +790,6 @@ class InstallRepositoryManager: self.install( tool_shed_url, new_name, new_owner, new_changeset_revision, install_options ) - # Updates received did not include any newly defined repository dependencies but did include - # newly defined tool dependencies. If the newly defined tool dependencies are not the same - # as the originally defined tool dependencies, we need to install them. - if not install_new_dependencies: - for new_key, new_val in new_tool_dependencies_dict.items(): - if new_key not in original_tool_dependencies_dict: - return ("tool", irmm_metadata_dict) - original_val = original_tool_dependencies_dict[new_key] - if new_val != original_val: - return ("tool", irmm_metadata_dict) # Updates received did not include any newly defined repository dependencies or newly defined # tool dependencies that need to be installed. repository = self.app.update_repository_manager.update_repository_record( diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index c8b3d202fec..ada7edd734b 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -856,19 +856,6 @@ class AdminToolshed(AdminGalaxy): containers_dicts.append(containers_dict) # Merge all containers into a single container. containers_dict = dd.merge_containers_dicts_for_new_install(containers_dicts) - # Handle tool dependencies check box. - if not trans.app.toolbox.dependency_manager.uses_tool_shed_dependencies: - if includes_tool_dependencies: - message = "Tool dependencies defined in this repository can be automatically installed if you set " - message += "the value of your tool_dependency_dir setting in your Galaxy config file " - message += "(galaxy.yml) and restart your Galaxy server before installing the repository." - status = "warning" - install_tool_dependencies_check_box_checked = False - else: - install_tool_dependencies_check_box_checked = True - install_tool_dependencies_check_box = CheckboxField( - "install_tool_dependencies", value=install_tool_dependencies_check_box_checked - ) # Handle repository dependencies check box. install_repository_dependencies_check_box = CheckboxField("install_repository_dependencies", value=True) view = views.DependencyResolversView(self.app) @@ -889,7 +876,6 @@ class AdminToolshed(AdminGalaxy): includes_tools=includes_tools, includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel, includes_tool_dependencies=includes_tool_dependencies, - install_tool_dependencies_check_box=install_tool_dependencies_check_box, install_resolver_dependencies_check_box=install_resolver_dependencies_check_box, has_repository_dependencies=has_repository_dependencies, install_repository_dependencies_check_box=install_repository_dependencies_check_box, @@ -1252,18 +1238,6 @@ class AdminToolshed(AdminGalaxy): containers_dict = dd.merge_missing_repository_dependencies_to_installed_container(containers_dict) # Handle repository dependencies check box. install_repository_dependencies_check_box = CheckboxField("install_repository_dependencies", value=True) - # Handle tool dependencies check box. - if not trans.app.toolbox.dependency_manager.uses_tool_shed_dependencies: - if includes_tool_dependencies: - message += "Tool dependencies defined in this repository can be automatically installed if you set the value of your tool_dependency_dir " - message += "setting in your Galaxy config file (galaxy.yml) and restart your Galaxy server before installing the repository. " - status = "warning" - install_tool_dependencies_check_box_checked = False - else: - install_tool_dependencies_check_box_checked = True - install_tool_dependencies_check_box = CheckboxField( - "install_tool_dependencies", value=install_tool_dependencies_check_box_checked - ) view = views.DependencyResolversView(self.app) if view.installable_resolvers: install_resolver_dependencies_check_box = CheckboxField("install_resolver_dependencies", value=True) @@ -1282,7 +1256,6 @@ class AdminToolshed(AdminGalaxy): includes_workflows=includes_workflows, has_repository_dependencies=has_repository_dependencies, install_repository_dependencies_check_box=install_repository_dependencies_check_box, - install_tool_dependencies_check_box=install_tool_dependencies_check_box, install_resolver_dependencies_check_box=install_resolver_dependencies_check_box, containers_dict=containers_dict, tool_panel_section_select_field=tool_panel_section_select_field, @@ -1388,26 +1361,6 @@ class AdminToolshed(AdminGalaxy): updating=True, ) return self.prepare_for_install(trans, **new_kwd) - elif install_dependencies == "tool": - # Updates received did not include any newly defined repository dependencies but did include - # newly defined tool dependencies. If the newly defined tool dependencies are not the same - # as the originally defined tool dependencies, we need to install them. - relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir(trans.app, repository)[2] - encoded_tool_dependencies_dict = encoding_util.tool_shed_encode( - irmm_metadata_dict.get("tool_dependencies", {}) - ) - encoded_relative_install_dir = encoding_util.tool_shed_encode(relative_install_dir) - new_kwd = dict( - updating_repository_id=trans.security.encode_id(repository.id), - updating_to_ctx_rev=latest_ctx_rev, - updating_to_changeset_revision=latest_changeset_revision, - encoded_updated_metadata=encoding_util.tool_shed_encode(irmm_metadata_dict), - encoded_relative_install_dir=encoded_relative_install_dir, - encoded_tool_dependencies_dict=encoded_tool_dependencies_dict, - message=message, - status=status, - ) - return self.install_tool_dependencies_with_update(trans, **new_kwd) message = f"The installed repository named '{name}' has been updated to change set revision '{latest_changeset_revision}'. " else: message = ( diff --git a/lib/tool_shed/test/functional/test_1010_install_repository_with_tool_dependencies.py b/lib/tool_shed/test/functional/test_1010_install_repository_with_tool_dependencies.py index 86b20ceb41c..1296376a286 100644 --- a/lib/tool_shed/test/functional/test_1010_install_repository_with_tool_dependencies.py +++ b/lib/tool_shed/test/functional/test_1010_install_repository_with_tool_dependencies.py @@ -136,8 +136,6 @@ class ToolWithToolDependencies(ShedTwillTestCase): strings_displayed = [ "Never installed", "install all needed dependencies", - "install Tool Shed managed", - "tool dependencies", ] strings_displayed.extend(["freebayes", "0.9.4_9696d0ce8a9", "samtools", "0.1.18"]) self.install_repository( diff --git a/lib/tool_shed/test/functional/test_1040_install_repository_basic_circular_dependencies.py b/lib/tool_shed/test/functional/test_1040_install_repository_basic_circular_dependencies.py index be9c6173f66..80da1e0f5c3 100644 --- a/lib/tool_shed/test/functional/test_1040_install_repository_basic_circular_dependencies.py +++ b/lib/tool_shed/test/functional/test_1040_install_repository_basic_circular_dependencies.py @@ -153,7 +153,6 @@ class TestInstallingCircularDependencies(ShedTwillTestCase): self.display_installed_repository_manage_page(installed_filtering_repository) self.deactivate_repository(installed_filtering_repository) self.test_db_util.ga_refresh(installed_filtering_repository) - self.check_galaxy_repository_tool_panel_section(installed_filtering_repository, "Get Data") strings_displayed = [ "Missing repository", "filtering", @@ -191,7 +190,6 @@ class TestInstallingCircularDependencies(ShedTwillTestCase): self.display_installed_repository_manage_page(installed_filtering_repository) self.deactivate_repository(installed_freebayes_repository) self.test_db_util.ga_refresh(installed_freebayes_repository) - self.check_galaxy_repository_tool_panel_section(installed_freebayes_repository, "Get Data") strings_displayed = [ "Missing repository", "freebayes", diff --git a/lib/tool_shed/test/functional/test_1210_uninstall_reinstall_repository_with_tool_dependencies.py b/lib/tool_shed/test/functional/test_1210_uninstall_reinstall_repository_with_tool_dependencies.py index e63f9f47663..c5119012405 100644 --- a/lib/tool_shed/test/functional/test_1210_uninstall_reinstall_repository_with_tool_dependencies.py +++ b/lib/tool_shed/test/functional/test_1210_uninstall_reinstall_repository_with_tool_dependencies.py @@ -117,14 +117,7 @@ class UninstallingAndReinstallingRepositories(ShedTwillTestCase): def test_0010_install_freebayes_repository(self): """Install the freebayes repository into the Galaxy instance.""" self.galaxy_login(email=common.admin_email, username=common.admin_username) - strings_displayed = [ - "install Tool Shed managed", - "tool dependencies", - "freebayes", - "0.9.4_9696d0ce8a9", - "samtools", - "0.1.18", - ] + strings_displayed = [] self.install_repository( "freebayes_0010", common.test_user_1_name, diff --git a/lib/tool_shed/webapp/templates/admin/tool_shed_repository/common.mako b/lib/tool_shed/webapp/templates/admin/tool_shed_repository/common.mako index fcd2ff6a9a8..2f7173a4f02 100644 --- a/lib/tool_shed/webapp/templates/admin/tool_shed_repository/common.mako +++ b/lib/tool_shed/webapp/templates/admin/tool_shed_repository/common.mako @@ -68,7 +68,7 @@ -<%def name="render_dependencies_section( install_resolver_dependencies_check_box, repository_dependencies_check_box, install_tool_dependencies_check_box, containers_dict, revision_label=None, export=False, requirements_status=None )"> +<%def name="render_dependencies_section( install_resolver_dependencies_check_box, repository_dependencies_check_box, containers_dict, revision_label=None, export=False, requirements_status=None )">
New tool dependencies included in update