From 18977d4132556a0c2bd0348db510cf765d434e6a Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Fri, 29 Sep 2017 10:51:06 -0400 Subject: [PATCH 1/4] Don't load hidden tools into the tool panel at all. --- lib/galaxy/tools/toolbox/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index 18e03e4ac69..460ee37a16c 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -264,6 +264,9 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin, object): # See if a version of this tool is already loaded into the tool panel. # The value of panel_component will be a ToolSection (if the value of # section=True) or self._tool_panel (if section=False). + if tool.hidden: + log.debug("Skipping tool panel addition of hidden tool: %s, version: %s", tool.id, tool.version) + return tool_id = str(tool.id) tool = self._tools_by_id[tool_id] log_msg = "" From 2eed478545857ab2c3b4c958bc3e2c360656af42 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 10 Aug 2017 16:04:26 +0100 Subject: [PATCH 2/4] Update conda channel order to sync with Bioconda --- config/galaxy.ini.sample | 3 +-- doc/source/admin/dependency_resolvers.rst | 4 ++-- lib/galaxy/tools/deps/resolvers/conda.py | 6 +++++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/config/galaxy.ini.sample b/config/galaxy.ini.sample index 5720c826698..3c0ea4283e1 100644 --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -209,8 +209,7 @@ paste.app_factory = galaxy.web.buildapp:app_factory # Pass debug flag to conda commands. #conda_debug = False # conda channels to enable by default (http://conda.pydata.org/docs/custom-channels.html) -# the recommended channel order is the one from BioConda (https://github.com/bioconda/bioconda-recipes/blob/master/config.yml#L8) -#conda_ensure_channels = iuc,bioconda,r,defaults,conda-forge +#conda_ensure_channels = iuc,bioconda,conda-forge,defaults,r # Set to True to instruct Galaxy to look for and install missing tool # dependencies before each job runs. #conda_auto_install = False diff --git a/doc/source/admin/dependency_resolvers.rst b/doc/source/admin/dependency_resolvers.rst index 0d6930d1ef7..12e93a77357 100644 --- a/doc/source/admin/dependency_resolvers.rst +++ b/doc/source/admin/dependency_resolvers.rst @@ -220,8 +220,8 @@ debug ensure_channels conda channels to enable by default. See http://conda.pydata.org/docs/custom-channels.html for more - information about channels. This defaults to ``iuc,bioconda,r,defaults,conda-forge``. - This order should be consistent with `Bioconda prescribed order `__ + information about channels. This defaults to ``iuc,bioconda,conda-forge,defaults,r``. + This order should be consistent with the `Bioconda prescribed order `__ if it includes ``bioconda``. auto_install diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py index 908fb1cef08..caa8dd5bbb0 100644 --- a/lib/galaxy/tools/deps/resolvers/conda.py +++ b/lib/galaxy/tools/deps/resolvers/conda.py @@ -39,7 +39,11 @@ from ..resolvers import ( DEFAULT_BASE_PATH_DIRECTORY = "_conda" DEFAULT_CONDARC_OVERRIDE = "_condarc" -DEFAULT_ENSURE_CHANNELS = "iuc,bioconda,r,defaults,conda-forge" +# Conda channel order from highest to lowest, following the one used in +# https://github.com/bioconda/bioconda-recipes/blob/master/config.yml , but +# adding `iuc` as first channel (for Galaxy-specific packages) and `r` as last +# (for old R packages) +DEFAULT_ENSURE_CHANNELS = "iuc,bioconda,conda-forge,defaults,r" CONDA_SOURCE_CMD = """[ "$CONDA_DEFAULT_ENV" = "%s" ] || MAX_TRIES=3 COUNT=0 From a5b7dce09fb690f09d25f1b806d3339bdf2171c9 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 14 Aug 2017 11:41:19 +0100 Subject: [PATCH 3/4] Remove _ensure_conda_context() function and make conda_context argument mandatory in functions/methods. This should help making sure the correct context (e.g. channels) is used for all Conda operations. Also remove unused is_target_available() function. Conflicts: lib/galaxy/tools/deps/conda_util.py --- lib/galaxy/tools/deps/conda_util.py | 54 ++++++------------------ lib/galaxy/tools/deps/resolvers/conda.py | 4 +- 2 files changed, 14 insertions(+), 44 deletions(-) diff --git a/lib/galaxy/tools/deps/conda_util.py b/lib/galaxy/tools/deps/conda_util.py index 00ca742f24e..bf32cb03b20 100644 --- a/lib/galaxy/tools/deps/conda_util.py +++ b/lib/galaxy/tools/deps/conda_util.py @@ -165,7 +165,7 @@ class CondaContext(installable.InstallableContext): if self.use_local and not self.conda_build_available: conda_targets = [CondaTarget("conda-build", version=CONDA_BUILD_VERSION)] # Cannot use --use-local during installation fo conda-build. - return install_conda_targets(conda_targets, env_name=None, conda_context=self, allow_local=False) + return install_conda_targets(conda_targets, conda_context=self, env_name=None, allow_local=False) else: return 0 @@ -428,8 +428,7 @@ def hash_conda_packages(conda_packages, conda_target=None): # shell makes sense for planemo, in Galaxy this should just execute # these commands as Python -def install_conda(conda_context=None, force_conda_build=False): - conda_context = _ensure_conda_context(conda_context) +def install_conda(conda_context, force_conda_build=False): f, script_path = tempfile.mkstemp(suffix=".sh", prefix="conda_install") os.close(f) download_cmd = " ".join(commands.download_command(conda_link(), to=script_path, quote_url=True)) @@ -449,8 +448,7 @@ def install_conda(conda_context=None, force_conda_build=False): os.remove(script_path) -def install_conda_targets(conda_targets, env_name=None, conda_context=None, allow_local=True): - conda_context = _ensure_conda_context(conda_context) +def install_conda_targets(conda_targets, conda_context, env_name=None, allow_local=True): conda_context.ensure_channels_configured() if env_name is not None: create_args = [ @@ -463,10 +461,9 @@ def install_conda_targets(conda_targets, env_name=None, conda_context=None, allo return conda_context.exec_install([t.package_specifier for t in conda_targets], allow_local=allow_local) -def install_conda_target(conda_target, conda_context=None, skip_environment=False): +def install_conda_target(conda_target, conda_context, skip_environment=False): """ Install specified target into a its own environment. """ - conda_context = _ensure_conda_context(conda_context) conda_context.ensure_channels_configured() if not skip_environment: create_args = [ @@ -478,8 +475,7 @@ def install_conda_target(conda_target, conda_context=None, skip_environment=Fals return conda_context.exec_install([conda_target.package_specifier]) -def cleanup_failed_install_of_environment(env, conda_context=None): - conda_context = _ensure_conda_context(conda_context) +def cleanup_failed_install_of_environment(env, conda_context): if conda_context.has_env(env): conda_context.exec_remove([env]) @@ -488,12 +484,11 @@ def cleanup_failed_install(conda_target, conda_context=None): cleanup_failed_install_of_environment(conda_target.install_environment, conda_context=conda_context) -def best_search_result(conda_target, conda_context=None, channels_override=None): +def best_search_result(conda_target, conda_context, channels_override=None): """Find best "conda search" result for specified target. Return ``None`` if no results match. """ - conda_context = _ensure_conda_context(conda_context) if not channels_override: conda_context.ensure_channels_configured() @@ -527,22 +522,7 @@ def is_search_hit_exact(conda_target, search_hit): return not target_version or search_hit['version'] == target_version -def is_target_available(conda_target, conda_context=None, channels_override=None): - """Check if a specified target is available for installation. - - If the package name exists return ``True`` (the ``bool``). If in addition - the version matches exactly return "exact" (a string). Otherwise return - ``False``. - """ - (best_hit, exact) = best_search_result(conda_target, conda_context, channels_override) - if best_hit: - return 'exact' if exact else True - else: - return False - - -def is_conda_target_installed(conda_target, conda_context=None): - conda_context = _ensure_conda_context(conda_context) +def is_conda_target_installed(conda_target, conda_context): # fail by default if conda_context.has_env(conda_target.install_environment): return True @@ -550,8 +530,7 @@ def is_conda_target_installed(conda_target, conda_context=None): return False -def filter_installed_targets(conda_targets, conda_context=None): - conda_context = _ensure_conda_context(conda_context) +def filter_installed_targets(conda_targets, conda_context): installed = functools.partial(is_conda_target_installed, conda_context=conda_context) return list(filter(installed, conda_targets)) @@ -559,9 +538,9 @@ def filter_installed_targets(conda_targets, conda_context=None): def build_isolated_environment( conda_packages, + conda_context, path=None, copy=False, - conda_context=None, quiet=False, ): """ Build a new environment (or reuse an existing one from hashes) @@ -571,7 +550,6 @@ def build_isolated_environment( conda_packages = [conda_packages] # Lots we could do in here, hashing, checking revisions, etc... - conda_context = _ensure_conda_context(conda_context) try: hash = hash_conda_packages(conda_packages) tempdir = tempfile.mkdtemp(prefix="jobdeps", suffix=hash) @@ -622,7 +600,7 @@ def build_isolated_environment( shutil.rmtree(tempdir) -def requirement_to_conda_targets(requirement, conda_context=None): +def requirement_to_conda_targets(requirement): conda_target = None if requirement.type == "package": conda_target = CondaTarget(requirement.name, @@ -630,19 +608,11 @@ def requirement_to_conda_targets(requirement, conda_context=None): return conda_target -def requirements_to_conda_targets(requirements, conda_context=None): - r_to_ct = functools.partial(requirement_to_conda_targets, - conda_context=conda_context) - conda_targets = (r_to_ct(_) for _ in requirements) +def requirements_to_conda_targets(requirements): + conda_targets = (requirement_to_conda_targets(_) for _ in requirements) return [c for c in conda_targets if c is not None] -def _ensure_conda_context(conda_context): - if conda_context is None: - conda_context = CondaContext() - return conda_context - - __all__ = ( 'CondaContext', 'CondaTarget', diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py index caa8dd5bbb0..ec63122a255 100644 --- a/lib/galaxy/tools/deps/resolvers/conda.py +++ b/lib/galaxy/tools/deps/resolvers/conda.py @@ -169,7 +169,7 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li def install_all(self, conda_targets): env = self.merged_environment_name(conda_targets) - return_code = install_conda_targets(conda_targets, env, conda_context=self.conda_context) + return_code = install_conda_targets(conda_targets, conda_context=self.conda_context, env_name=env) if return_code != 0: is_installed = False else: @@ -452,9 +452,9 @@ class CondaDependency(Dependency): def build_environment(self): env_path, exit_code = build_isolated_environment( CondaTarget(self.name, self.version), + conda_context=self.conda_context, path=self.environment_path, copy=self.conda_context.copy_dependencies, - conda_context=self.conda_context, ) if exit_code: if len(os.path.abspath(self.environment_path)) > 79: From 3551386a1ac667e0d2f7c6e82256d180aa5100f7 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 14 Aug 2017 14:47:40 +0100 Subject: [PATCH 4/4] Override channel list on the command line instead of modifying condarc The way we were modifying condarc in the ensure_channels_configured() method was broken because we were appending the missing channels at the end, so the final order may not be the one specified in conda_ensure_channels . Also: - not modifying the user condarc is a much better approach - overriding the channel list on the command line will make debugging easier --- lib/galaxy/tools/deps/conda_util.py | 61 ++++++----------------------- 1 file changed, 11 insertions(+), 50 deletions(-) diff --git a/lib/galaxy/tools/deps/conda_util.py b/lib/galaxy/tools/deps/conda_util.py index bf32cb03b20..d0a2396638f 100644 --- a/lib/galaxy/tools/deps/conda_util.py +++ b/lib/galaxy/tools/deps/conda_util.py @@ -11,7 +11,6 @@ from distutils.version import LooseVersion from sys import platform as _platform import six -import yaml from ..deps import commands from ..deps import installable @@ -95,7 +94,6 @@ class CondaContext(installable.InstallableContext): else: ensure_channels = None self.ensure_channels = ensure_channels - self.ensured_channels = False self._conda_version = None self._miniconda_version = None self._conda_build_available = None @@ -144,22 +142,14 @@ class CondaContext(installable.InstallableContext): def _conda_meta_path(self): return os.path.join(self.conda_prefix, "conda-meta") - def ensure_channels_configured(self): - if not self.ensured_channels: - self.ensured_channels = True - - changed = False - conda_conf = self.load_condarc() - if "channels" not in conda_conf: - conda_conf["channels"] = [] - channels = conda_conf["channels"] + @property + def _override_channels_args(self): + override_channels_args = [] + if self.ensure_channels: + override_channels_args.append("--override-channels") for channel in self.ensure_channels: - if channel not in channels: - changed = True - channels.append(channel) - - if changed: - self.save_condarc(conda_conf) + override_channels_args.extend(["--channel", channel]) + return override_channels_args def ensure_conda_build_installed_if_needed(self): if self.use_local and not self.conda_build_available: @@ -211,34 +201,6 @@ class CondaContext(installable.InstallableContext): self.conda_prefix, self.conda_exec) return False - def load_condarc(self): - condarc = self.condarc - if os.path.exists(condarc): - with open(condarc, "r") as f: - return yaml.safe_load(f) - else: - return {"channels": ["defaults"]} - - def save_condarc(self, conf): - condarc = self.condarc - try: - with open(condarc, "w") as f: - return yaml.safe_dump(conf, f) - except IOError: - template = ("Failed to update write to path [%s] while attempting to update conda configuration, " - "please update the configuration to override the condarc location or " - "grant this application write to the parent directory.") - message = template % condarc - raise Exception(message) - - @property - def condarc(self): - if self.condarc_override: - return self.condarc_override - else: - home = os.path.expanduser("~") - return os.path.join(home, ".condarc") - def command(self, operation, args): if isinstance(args, list): args = " ".join(args) @@ -269,6 +231,7 @@ class CondaContext(installable.InstallableContext): ] if allow_local and self.use_local: create_base_args.extend(["--use-local"]) + create_base_args.extend(self._override_channels_args) create_base_args.extend(args) return self.exec_command("create", create_base_args) @@ -288,6 +251,7 @@ class CondaContext(installable.InstallableContext): ] if allow_local and self.use_local: install_base_args.extend(["--use-local"]) + install_base_args.extend(self._override_channels_args) install_base_args.extend(args) return self.exec_command("install", install_base_args) @@ -449,7 +413,6 @@ def install_conda(conda_context, force_conda_build=False): def install_conda_targets(conda_targets, conda_context, env_name=None, allow_local=True): - conda_context.ensure_channels_configured() if env_name is not None: create_args = [ "--name", env_name, # environment for package @@ -464,7 +427,6 @@ def install_conda_targets(conda_targets, conda_context, env_name=None, allow_loc def install_conda_target(conda_target, conda_context, skip_environment=False): """ Install specified target into a its own environment. """ - conda_context.ensure_channels_configured() if not skip_environment: create_args = [ "--name", conda_target.install_environment, # environment for package @@ -489,14 +451,13 @@ def best_search_result(conda_target, conda_context, channels_override=None): Return ``None`` if no results match. """ - if not channels_override: - conda_context.ensure_channels_configured() - search_cmd = [conda_context.conda_exec, "search", "--full-name", "--json"] if channels_override: search_cmd.append("--override-channels") for channel in channels_override: search_cmd.extend(["--channel", channel]) + else: + search_cmd.extend(conda_context._override_channels_args) search_cmd.append(conda_target.package) res = commands.execute(search_cmd) hits = json.loads(res).get(conda_target.package, [])