Merge pull request #5397 from nsoranzo/release_18.01_conda_fixes

[18.01] Install latest conda and don't use shell=True

lib/galaxy/tools/deps/commands.py

@@ -1,12 +1,16 @@
 """Generic I/O and shell processing code used by Galaxy tool dependencies."""
+import logging
 import os
 import subprocess
 import sys as _sys
 
+import six
 from six.moves import shlex_quote
 
 from galaxy.util import which
 
+log = logging.getLogger(__name__)
+
 STDOUT_INDICATOR = "-"
 
 
@@ -50,9 +54,10 @@ def shell_process(cmds, env=None, **kwds):
     redirection.
     """
     sys = kwds.get("sys", _sys)
-    popen_kwds = dict(
-        shell=True,
-    )
+    popen_kwds = dict()
+    if isinstance(cmds, six.string_types):
+        log.warning("Passing program arguments as a string may be a security hazard if combined with untrusted input")
+        popen_kwds['shell'] = True
     if kwds.get("stdout", None) is None and redirecting_io(sys=sys):
         popen_kwds["stdout"] = subprocess.PIPE
     if kwds.get("stderr", None) is None and redirecting_io(sys=sys):
@@ -109,13 +114,13 @@ def download_command(url, to=STDOUT_INDICATOR, quote_url=False):
     if which("wget"):
         download_cmd = ["wget", "-q"]
         if to == STDOUT_INDICATOR:
-            download_cmd += ["-O", STDOUT_INDICATOR, url]
+            download_cmd.extend(["-O", STDOUT_INDICATOR, url])
         else:
-            download_cmd += ["--recursive", "-O", to, url]
+            download_cmd.extend(["--recursive", "-O", to, url])
     else:
         download_cmd = ["curl", "-L", url]
         if to != STDOUT_INDICATOR:
-            download_cmd += ["-o", to]
+            download_cmd.extend(["-o", to])
     return download_cmd
 
 

lib/galaxy/tools/deps/conda_util.py

@@ -5,14 +5,17 @@ import logging
 import os
 import re
 import shutil
+import sys
 import tempfile
 from distutils.version import LooseVersion
-from sys import platform as _platform
 
 import six
+from six.moves import shlex_quote
 
-from ..deps import commands
-from ..deps import installable
+from . import (
+    commands,
+    installable
+)
 
 log = logging.getLogger(__name__)
 
@@ -20,23 +23,26 @@ log = logging.getLogger(__name__)
 # break shell commands we are building.
 SHELL_UNSAFE_PATTERN = re.compile(r"[\s\"']")
 
-IS_OS_X = _platform == "darwin"
+IS_OS_X = sys.platform == "darwin"
 
 # BSD 3-clause
 CONDA_LICENSE = "http://docs.continuum.io/anaconda/eula"
 VERSIONED_ENV_DIR_NAME = re.compile(r"__(.*)@(.*)")
 UNVERSIONED_ENV_DIR_NAME = re.compile(r"__(.*)@_uv_")
 USE_PATH_EXEC_DEFAULT = False
-CONDA_VERSION = "4.3.24"
-CONDA_BUILD_VERSION = "2.1.17"
+CONDA_VERSION = "4.3.33"
+CONDA_BUILD_VERSION = "2.1.18"
 USE_LOCAL_DEFAULT = False
 
 
 def conda_link():
     if IS_OS_X:
-        url = "https://repo.continuum.io/miniconda/Miniconda3-4.2.12-MacOSX-x86_64.sh"
+        url = "https://repo.continuum.io/miniconda/Miniconda3-4.3.31-MacOSX-x86_64.sh"
     else:
-        url = "https://repo.continuum.io/miniconda/Miniconda3-4.2.12-Linux-x86_64.sh"
+        if sys.maxsize > 2**32:
+            url = "https://repo.continuum.io/miniconda/Miniconda3-4.3.31-Linux-x86_64.sh"
+        else:
+            url = "https://repo.continuum.io/miniconda/Miniconda3-4.3.31-Linux-x86.sh"
     return url
 
 
@@ -198,31 +204,35 @@ class CondaContext(installable.InstallableContext):
                         self.conda_prefix, self.conda_exec)
             return False
 
-    def command(self, operation, args):
-        if isinstance(args, list):
-            args = " ".join(args)
-        conda_prefix = self.conda_exec
-        if self.debug:
-            conda_prefix += " --debug"
-        return "%s %s %s" % (conda_prefix, operation, args)
-
     def exec_command(self, operation, args):
-        command = self.command(operation, args)
+        """
+        Execute the requested command.
+
+        Return the process exit code (i.e. 0 in case of success).
+        """
+        cmd = [self.conda_exec]
+        if self.debug:
+            cmd.append("--debug")
+        cmd.append(operation)
+        cmd.extend(args)
         env = {}
-        condarc_override = self.condarc_override
-        if condarc_override:
-            env["CONDARC"] = condarc_override
-        log.debug("Executing command: %s", command)
+        if self.condarc_override:
+            env["CONDARC"] = self.condarc_override
+        cmd_string = ' '.join(map(shlex_quote, cmd))
+        log.debug("Executing command: %s", cmd_string)
         conda_exec_home = env['HOME'] = tempfile.mkdtemp(prefix='conda_exec_home_')  # We don't want to pollute ~/.conda, which may not even be writable
         try:
-            return self.shell_exec(command, env=env)
-        except commands.CommandLineException as e:
-            log.warning(e)
-            return e.returncode
+            return self.shell_exec(cmd, env=env)
+        except Exception:
+            log.exception("Failed to execute command: %s", cmd_string)
+            return 1
         finally:
             shutil.rmtree(conda_exec_home, ignore_errors=True)
 
     def exec_create(self, args, allow_local=True):
+        """
+        Return the process exit code (i.e. 0 in case of success).
+        """
         create_base_args = [
             "-y"
         ]
@@ -233,7 +243,11 @@ class CondaContext(installable.InstallableContext):
         return self.exec_command("create", create_base_args)
 
     def exec_remove(self, args):
-        """Remove a conda environment using conda env remove -y --name `args`."""
+        """
+        Remove a conda environment using conda env remove -y --name `args`.
+
+        Return the process exit code (i.e. 0 in case of success).
+        """
         remove_base_args = [
             "remove",
             "-y",
@@ -243,11 +257,14 @@ class CondaContext(installable.InstallableContext):
         return self.exec_command("env", remove_base_args)
 
     def exec_install(self, args, allow_local=True):
+        """
+        Return the process exit code (i.e. 0 in case of success).
+        """
         install_base_args = [
             "-y"
         ]
         if allow_local and self.use_local:
-            install_base_args.extend(["--use-local"])
+            install_base_args.append("--use-local")
         install_base_args.extend(self._override_channels_args)
         install_base_args.extend(args)
         return self.exec_command("install", install_base_args)
@@ -255,6 +272,8 @@ class CondaContext(installable.InstallableContext):
     def exec_clean(self, args=[], quiet=False):
         """
         Clean up after conda installation.
+
+        Return the process exit code (i.e. 0 in case of success).
         """
         clean_base_args = [
             "--tarballs",
@@ -266,6 +285,9 @@ class CondaContext(installable.InstallableContext):
         return self.exec_command("clean", clean_args)
 
     def export_list(self, name, path):
+        """
+        Return the process exit code (i.e. 0 in case of success).
+        """
         return self.exec_command("list", [
             "--name", name,
             "--export", ">", path
@@ -392,24 +414,34 @@ def hash_conda_packages(conda_packages, conda_target=None):
 def install_conda(conda_context, force_conda_build=False):
     f, script_path = tempfile.mkstemp(suffix=".sh", prefix="conda_install")
     os.close(f)
-    download_cmd = " ".join(commands.download_command(conda_link(), to=script_path, quote_url=True))
-    install_cmd = "bash '%s' -b -p '%s'" % (script_path, conda_context.conda_prefix)
+    download_cmd = commands.download_command(conda_link(), to=script_path, quote_url=False)
+    install_cmd = ['bash', script_path, '-b', '-p', conda_context.conda_prefix]
     package_targets = [
         "conda=%s" % CONDA_VERSION,
     ]
     if force_conda_build or conda_context.use_local:
         package_targets.append("conda-build=%s" % CONDA_BUILD_VERSION)
-    fix_version_cmd = "%s install -y -q %s " % (os.path.join(conda_context.conda_prefix, 'bin/conda'), " ".join(package_targets))
-    full_command = "%s && %s && %s" % (download_cmd, install_cmd, fix_version_cmd)
+    log.info("Installing conda, this may take several minutes.")
     try:
-        log.info("Installing Conda, this may take several minutes.")
-        return conda_context.shell_exec(full_command)
+        exit_code = conda_context.shell_exec(download_cmd)
+        if exit_code:
+            return exit_code
+        exit_code = conda_context.shell_exec(install_cmd)
+    except Exception:
+        log.exception('Failed to install conda')
+        return 1
     finally:
         if os.path.exists(script_path):
             os.remove(script_path)
+    if exit_code:
+        return exit_code
+    return conda_context.exec_install(package_targets, allow_local=False)
 
 
 def install_conda_targets(conda_targets, conda_context, env_name=None, allow_local=True):
+    """
+    Return the process exit code (i.e. 0 in case of success).
+    """
     if env_name is not None:
         create_args = [
             "--name", env_name,  # environment for package
@@ -422,7 +454,10 @@ def install_conda_targets(conda_targets, conda_context, env_name=None, allow_loc
 
 
 def install_conda_target(conda_target, conda_context, skip_environment=False):
-    """ Install specified target into a its own environment.
+    """
+    Install specified target into a its own environment.
+
+    Return the process exit code (i.e. 0 in case of success).
     """
     if not skip_environment:
         create_args = [