Merge pull request #4731 from nsoranzo/release_17.05_backport_4409

[17.05] Backport #4409

config/galaxy.ini.sample

@@ -209,8 +209,7 @@ paste.app_factory = galaxy.web.buildapp:app_factory
 # Pass debug flag to conda commands.
 #conda_debug = False
 # conda channels to enable by default (http://conda.pydata.org/docs/custom-channels.html)
-# the recommended channel order is the one from BioConda (https://github.com/bioconda/bioconda-recipes/blob/master/config.yml#L8)
-#conda_ensure_channels = iuc,bioconda,r,defaults,conda-forge
+#conda_ensure_channels = iuc,bioconda,conda-forge,defaults,r
 # Set to True to instruct Galaxy to look for and install missing tool
 # dependencies before each job runs.
 #conda_auto_install = False

doc/source/admin/dependency_resolvers.rst

@@ -220,8 +220,8 @@ debug
 ensure_channels
     conda channels to enable by default. See
     http://conda.pydata.org/docs/custom-channels.html for more
-    information about channels. This defaults to ``iuc,bioconda,r,defaults,conda-forge``.
-    This order should be consistent with `Bioconda prescribed order <https://github.com/bioconda/bioconda-recipes/blob/master/config.yml#L8>`__
+    information about channels. This defaults to ``iuc,bioconda,conda-forge,defaults,r``.
+    This order should be consistent with the `Bioconda prescribed order <https://github.com/bioconda/bioconda-recipes/blob/master/config.yml>`__
     if it includes ``bioconda``.
 
 auto_install

lib/galaxy/tools/deps/conda_util.py

@@ -11,7 +11,6 @@ from distutils.version import LooseVersion
 from sys import platform as _platform
 
 import six
-import yaml
 
 from ..deps import commands
 from ..deps import installable
@@ -95,7 +94,6 @@ class CondaContext(installable.InstallableContext):
         else:
             ensure_channels = None
         self.ensure_channels = ensure_channels
-        self.ensured_channels = False
         self._conda_version = None
         self._miniconda_version = None
         self._conda_build_available = None
@@ -144,28 +142,20 @@ class CondaContext(installable.InstallableContext):
     def _conda_meta_path(self):
         return os.path.join(self.conda_prefix, "conda-meta")
 
-    def ensure_channels_configured(self):
-        if not self.ensured_channels:
-            self.ensured_channels = True
-
-            changed = False
-            conda_conf = self.load_condarc()
-            if "channels" not in conda_conf:
-                conda_conf["channels"] = []
-            channels = conda_conf["channels"]
+    @property
+    def _override_channels_args(self):
+        override_channels_args = []
+        if self.ensure_channels:
+            override_channels_args.append("--override-channels")
             for channel in self.ensure_channels:
-                if channel not in channels:
-                    changed = True
-                    channels.append(channel)
-
-            if changed:
-                self.save_condarc(conda_conf)
+                override_channels_args.extend(["--channel", channel])
+        return override_channels_args
 
     def ensure_conda_build_installed_if_needed(self):
         if self.use_local and not self.conda_build_available:
             conda_targets = [CondaTarget("conda-build", version=CONDA_BUILD_VERSION)]
             # Cannot use --use-local during installation fo conda-build.
-            return install_conda_targets(conda_targets, env_name=None, conda_context=self, allow_local=False)
+            return install_conda_targets(conda_targets, conda_context=self, env_name=None, allow_local=False)
         else:
             return 0
 
@@ -211,34 +201,6 @@ class CondaContext(installable.InstallableContext):
                         self.conda_prefix, self.conda_exec)
             return False
 
-    def load_condarc(self):
-        condarc = self.condarc
-        if os.path.exists(condarc):
-            with open(condarc, "r") as f:
-                return yaml.safe_load(f)
-        else:
-            return {"channels": ["defaults"]}
-
-    def save_condarc(self, conf):
-        condarc = self.condarc
-        try:
-            with open(condarc, "w") as f:
-                return yaml.safe_dump(conf, f)
-        except IOError:
-            template = ("Failed to update write to path [%s] while attempting to update conda configuration, "
-                        "please update the configuration to override the condarc location or "
-                        "grant this application write to the parent directory.")
-            message = template % condarc
-            raise Exception(message)
-
-    @property
-    def condarc(self):
-        if self.condarc_override:
-            return self.condarc_override
-        else:
-            home = os.path.expanduser("~")
-            return os.path.join(home, ".condarc")
-
     def command(self, operation, args):
         if isinstance(args, list):
             args = " ".join(args)
@@ -269,6 +231,7 @@ class CondaContext(installable.InstallableContext):
         ]
         if allow_local and self.use_local:
             create_base_args.extend(["--use-local"])
+        create_base_args.extend(self._override_channels_args)
         create_base_args.extend(args)
         return self.exec_command("create", create_base_args)
 
@@ -288,6 +251,7 @@ class CondaContext(installable.InstallableContext):
         ]
         if allow_local and self.use_local:
             install_base_args.extend(["--use-local"])
+        install_base_args.extend(self._override_channels_args)
         install_base_args.extend(args)
         return self.exec_command("install", install_base_args)
 
@@ -428,8 +392,7 @@ def hash_conda_packages(conda_packages, conda_target=None):
 
 # shell makes sense for planemo, in Galaxy this should just execute
 # these commands as Python
-def install_conda(conda_context=None, force_conda_build=False):
-    conda_context = _ensure_conda_context(conda_context)
+def install_conda(conda_context, force_conda_build=False):
     f, script_path = tempfile.mkstemp(suffix=".sh", prefix="conda_install")
     os.close(f)
     download_cmd = " ".join(commands.download_command(conda_link(), to=script_path, quote_url=True))
@@ -449,9 +412,7 @@ def install_conda(conda_context=None, force_conda_build=False):
             os.remove(script_path)
 
 
-def install_conda_targets(conda_targets, env_name=None, conda_context=None, allow_local=True):
-    conda_context = _ensure_conda_context(conda_context)
-    conda_context.ensure_channels_configured()
+def install_conda_targets(conda_targets, conda_context, env_name=None, allow_local=True):
     if env_name is not None:
         create_args = [
             "--name", env_name,  # environment for package
@@ -463,11 +424,9 @@ def install_conda_targets(conda_targets, env_name=None, conda_context=None, allo
         return conda_context.exec_install([t.package_specifier for t in conda_targets], allow_local=allow_local)
 
 
-def install_conda_target(conda_target, conda_context=None, skip_environment=False):
+def install_conda_target(conda_target, conda_context, skip_environment=False):
     """ Install specified target into a its own environment.
     """
-    conda_context = _ensure_conda_context(conda_context)
-    conda_context.ensure_channels_configured()
     if not skip_environment:
         create_args = [
             "--name", conda_target.install_environment,  # environment for package
@@ -478,8 +437,7 @@ def install_conda_target(conda_target, conda_context=None, skip_environment=Fals
         return conda_context.exec_install([conda_target.package_specifier])
 
 
-def cleanup_failed_install_of_environment(env, conda_context=None):
-    conda_context = _ensure_conda_context(conda_context)
+def cleanup_failed_install_of_environment(env, conda_context):
     if conda_context.has_env(env):
         conda_context.exec_remove([env])
 
@@ -488,20 +446,18 @@ def cleanup_failed_install(conda_target, conda_context=None):
     cleanup_failed_install_of_environment(conda_target.install_environment, conda_context=conda_context)
 
 
-def best_search_result(conda_target, conda_context=None, channels_override=None):
+def best_search_result(conda_target, conda_context, channels_override=None):
     """Find best "conda search" result for specified target.
 
     Return ``None`` if no results match.
     """
-    conda_context = _ensure_conda_context(conda_context)
-    if not channels_override:
-        conda_context.ensure_channels_configured()
-
     search_cmd = [conda_context.conda_exec, "search", "--full-name", "--json"]
     if channels_override:
         search_cmd.append("--override-channels")
         for channel in channels_override:
             search_cmd.extend(["--channel", channel])
+    else:
+        search_cmd.extend(conda_context._override_channels_args)
     search_cmd.append(conda_target.package)
     res = commands.execute(search_cmd)
     hits = json.loads(res).get(conda_target.package, [])
@@ -527,22 +483,7 @@ def is_search_hit_exact(conda_target, search_hit):
     return not target_version or search_hit['version'] == target_version
 
 
-def is_target_available(conda_target, conda_context=None, channels_override=None):
-    """Check if a specified target is available for installation.
-
-    If the package name exists return ``True`` (the ``bool``). If in addition
-    the version matches exactly return "exact" (a string). Otherwise return
-    ``False``.
-    """
-    (best_hit, exact) = best_search_result(conda_target, conda_context, channels_override)
-    if best_hit:
-        return 'exact' if exact else True
-    else:
-        return False
-
-
-def is_conda_target_installed(conda_target, conda_context=None):
-    conda_context = _ensure_conda_context(conda_context)
+def is_conda_target_installed(conda_target, conda_context):
     # fail by default
     if conda_context.has_env(conda_target.install_environment):
         return True
@@ -550,8 +491,7 @@ def is_conda_target_installed(conda_target, conda_context=None):
         return False
 
 
-def filter_installed_targets(conda_targets, conda_context=None):
-    conda_context = _ensure_conda_context(conda_context)
+def filter_installed_targets(conda_targets, conda_context):
     installed = functools.partial(is_conda_target_installed,
                                   conda_context=conda_context)
     return list(filter(installed, conda_targets))
@@ -559,9 +499,9 @@ def filter_installed_targets(conda_targets, conda_context=None):
 
 def build_isolated_environment(
     conda_packages,
+    conda_context,
     path=None,
     copy=False,
-    conda_context=None,
     quiet=False,
 ):
     """ Build a new environment (or reuse an existing one from hashes)
@@ -571,7 +511,6 @@ def build_isolated_environment(
         conda_packages = [conda_packages]
 
     # Lots we could do in here, hashing, checking revisions, etc...
-    conda_context = _ensure_conda_context(conda_context)
     try:
         hash = hash_conda_packages(conda_packages)
         tempdir = tempfile.mkdtemp(prefix="jobdeps", suffix=hash)
@@ -622,7 +561,7 @@ def build_isolated_environment(
         shutil.rmtree(tempdir)
 
 
-def requirement_to_conda_targets(requirement, conda_context=None):
+def requirement_to_conda_targets(requirement):
     conda_target = None
     if requirement.type == "package":
         conda_target = CondaTarget(requirement.name,
@@ -630,19 +569,11 @@ def requirement_to_conda_targets(requirement, conda_context=None):
     return conda_target
 
 
-def requirements_to_conda_targets(requirements, conda_context=None):
-    r_to_ct = functools.partial(requirement_to_conda_targets,
-                                conda_context=conda_context)
-    conda_targets = (r_to_ct(_) for _ in requirements)
+def requirements_to_conda_targets(requirements):
+    conda_targets = (requirement_to_conda_targets(_) for _ in requirements)
     return [c for c in conda_targets if c is not None]
 
 
-def _ensure_conda_context(conda_context):
-    if conda_context is None:
-        conda_context = CondaContext()
-    return conda_context
-
-
 __all__ = (
     'CondaContext',
     'CondaTarget',

lib/galaxy/tools/deps/resolvers/conda.py

@@ -39,7 +39,11 @@ from ..resolvers import (
 
 DEFAULT_BASE_PATH_DIRECTORY = "_conda"
 DEFAULT_CONDARC_OVERRIDE = "_condarc"
-DEFAULT_ENSURE_CHANNELS = "iuc,bioconda,r,defaults,conda-forge"
+# Conda channel order from highest to lowest, following the one used in
+# https://github.com/bioconda/bioconda-recipes/blob/master/config.yml , but
+# adding `iuc` as first channel (for Galaxy-specific packages) and `r` as last
+# (for old R packages)
+DEFAULT_ENSURE_CHANNELS = "iuc,bioconda,conda-forge,defaults,r"
 CONDA_SOURCE_CMD = """[ "$CONDA_DEFAULT_ENV" = "%s" ] ||
 MAX_TRIES=3
 COUNT=0
@@ -165,7 +169,7 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li
 
     def install_all(self, conda_targets):
         env = self.merged_environment_name(conda_targets)
-        return_code = install_conda_targets(conda_targets, env, conda_context=self.conda_context)
+        return_code = install_conda_targets(conda_targets, conda_context=self.conda_context, env_name=env)
         if return_code != 0:
             is_installed = False
         else:
@@ -448,9 +452,9 @@ class CondaDependency(Dependency):
     def build_environment(self):
         env_path, exit_code = build_isolated_environment(
             CondaTarget(self.name, self.version),
+            conda_context=self.conda_context,
             path=self.environment_path,
             copy=self.conda_context.copy_dependencies,
-            conda_context=self.conda_context,
         )
         if exit_code:
             if len(os.path.abspath(self.environment_path)) > 79: