From 889aa29d02aa54e04a1a31904f1b29b7453c49cf Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 26 Jan 2018 11:31:29 +0000 Subject: [PATCH 1/2] Conda: pass subprocess arguments as list --- lib/galaxy/tools/deps/commands.py | 17 ++++-- lib/galaxy/tools/deps/conda_util.py | 88 ++++++++++++++++++++--------- 2 files changed, 71 insertions(+), 34 deletions(-) diff --git a/lib/galaxy/tools/deps/commands.py b/lib/galaxy/tools/deps/commands.py index 02a3bf89116..af310c6ee25 100644 --- a/lib/galaxy/tools/deps/commands.py +++ b/lib/galaxy/tools/deps/commands.py @@ -1,12 +1,16 @@ """Generic I/O and shell processing code used by Galaxy tool dependencies.""" +import logging import os import subprocess import sys as _sys +import six from six.moves import shlex_quote from galaxy.util import which +log = logging.getLogger(__name__) + STDOUT_INDICATOR = "-" @@ -50,9 +54,10 @@ def shell_process(cmds, env=None, **kwds): redirection. """ sys = kwds.get("sys", _sys) - popen_kwds = dict( - shell=True, - ) + popen_kwds = dict() + if isinstance(cmds, six.string_types): + log.warning("Passing program arguments as a string may be a security hazard if combined with untrusted input") + popen_kwds['shell'] = True if kwds.get("stdout", None) is None and redirecting_io(sys=sys): popen_kwds["stdout"] = subprocess.PIPE if kwds.get("stderr", None) is None and redirecting_io(sys=sys): @@ -109,13 +114,13 @@ def download_command(url, to=STDOUT_INDICATOR, quote_url=False): if which("wget"): download_cmd = ["wget", "-q"] if to == STDOUT_INDICATOR: - download_cmd += ["-O", STDOUT_INDICATOR, url] + download_cmd.extend(["-O", STDOUT_INDICATOR, url]) else: - download_cmd += ["--recursive", "-O", to, url] + download_cmd.extend(["--recursive", "-O", to, url]) else: download_cmd = ["curl", "-L", url] if to != STDOUT_INDICATOR: - download_cmd += ["-o", to] + download_cmd.extend(["-o", to]) return download_cmd diff --git a/lib/galaxy/tools/deps/conda_util.py b/lib/galaxy/tools/deps/conda_util.py index c543246c53a..fe973535c5a 100644 --- a/lib/galaxy/tools/deps/conda_util.py +++ b/lib/galaxy/tools/deps/conda_util.py @@ -10,9 +10,12 @@ from distutils.version import LooseVersion from sys import platform as _platform import six +from six.moves import shlex_quote -from ..deps import commands -from ..deps import installable +from . import ( + commands, + installable +) log = logging.getLogger(__name__) @@ -198,31 +201,35 @@ class CondaContext(installable.InstallableContext): self.conda_prefix, self.conda_exec) return False - def command(self, operation, args): - if isinstance(args, list): - args = " ".join(args) - conda_prefix = self.conda_exec - if self.debug: - conda_prefix += " --debug" - return "%s %s %s" % (conda_prefix, operation, args) - def exec_command(self, operation, args): - command = self.command(operation, args) + """ + Execute the requested command. + + Return the process exit code (i.e. 0 in case of success). + """ + cmd = [self.conda_exec] + if self.debug: + cmd.append("--debug") + cmd.append(operation) + cmd.extend(args) env = {} - condarc_override = self.condarc_override - if condarc_override: - env["CONDARC"] = condarc_override - log.debug("Executing command: %s", command) + if self.condarc_override: + env["CONDARC"] = self.condarc_override + cmd_string = ' '.join(map(shlex_quote, cmd)) + log.debug("Executing command: %s", cmd_string) conda_exec_home = env['HOME'] = tempfile.mkdtemp(prefix='conda_exec_home_') # We don't want to pollute ~/.conda, which may not even be writable try: - return self.shell_exec(command, env=env) - except commands.CommandLineException as e: - log.warning(e) - return e.returncode + return self.shell_exec(cmd, env=env) + except Exception: + log.exception("Failed to execute command: %s", cmd_string) + return 1 finally: shutil.rmtree(conda_exec_home, ignore_errors=True) def exec_create(self, args, allow_local=True): + """ + Return the process exit code (i.e. 0 in case of success). + """ create_base_args = [ "-y" ] @@ -233,7 +240,11 @@ class CondaContext(installable.InstallableContext): return self.exec_command("create", create_base_args) def exec_remove(self, args): - """Remove a conda environment using conda env remove -y --name `args`.""" + """ + Remove a conda environment using conda env remove -y --name `args`. + + Return the process exit code (i.e. 0 in case of success). + """ remove_base_args = [ "remove", "-y", @@ -243,11 +254,14 @@ class CondaContext(installable.InstallableContext): return self.exec_command("env", remove_base_args) def exec_install(self, args, allow_local=True): + """ + Return the process exit code (i.e. 0 in case of success). + """ install_base_args = [ "-y" ] if allow_local and self.use_local: - install_base_args.extend(["--use-local"]) + install_base_args.append("--use-local") install_base_args.extend(self._override_channels_args) install_base_args.extend(args) return self.exec_command("install", install_base_args) @@ -255,6 +269,8 @@ class CondaContext(installable.InstallableContext): def exec_clean(self, args=[], quiet=False): """ Clean up after conda installation. + + Return the process exit code (i.e. 0 in case of success). """ clean_base_args = [ "--tarballs", @@ -266,6 +282,9 @@ class CondaContext(installable.InstallableContext): return self.exec_command("clean", clean_args) def export_list(self, name, path): + """ + Return the process exit code (i.e. 0 in case of success). + """ return self.exec_command("list", [ "--name", name, "--export", ">", path @@ -392,24 +411,34 @@ def hash_conda_packages(conda_packages, conda_target=None): def install_conda(conda_context, force_conda_build=False): f, script_path = tempfile.mkstemp(suffix=".sh", prefix="conda_install") os.close(f) - download_cmd = " ".join(commands.download_command(conda_link(), to=script_path, quote_url=True)) - install_cmd = "bash '%s' -b -p '%s'" % (script_path, conda_context.conda_prefix) + download_cmd = commands.download_command(conda_link(), to=script_path, quote_url=False) + install_cmd = ['bash', script_path, '-b', '-p', conda_context.conda_prefix] package_targets = [ "conda=%s" % CONDA_VERSION, ] if force_conda_build or conda_context.use_local: package_targets.append("conda-build=%s" % CONDA_BUILD_VERSION) - fix_version_cmd = "%s install -y -q %s " % (os.path.join(conda_context.conda_prefix, 'bin/conda'), " ".join(package_targets)) - full_command = "%s && %s && %s" % (download_cmd, install_cmd, fix_version_cmd) + log.info("Installing conda, this may take several minutes.") try: - log.info("Installing Conda, this may take several minutes.") - return conda_context.shell_exec(full_command) + exit_code = conda_context.shell_exec(download_cmd) + if exit_code: + return exit_code + exit_code = conda_context.shell_exec(install_cmd) + except Exception: + log.exception('Failed to install conda') + return 1 finally: if os.path.exists(script_path): os.remove(script_path) + if exit_code: + return exit_code + return conda_context.exec_install(package_targets, allow_local=False) def install_conda_targets(conda_targets, conda_context, env_name=None, allow_local=True): + """ + Return the process exit code (i.e. 0 in case of success). + """ if env_name is not None: create_args = [ "--name", env_name, # environment for package @@ -422,7 +451,10 @@ def install_conda_targets(conda_targets, conda_context, env_name=None, allow_loc def install_conda_target(conda_target, conda_context, skip_environment=False): - """ Install specified target into a its own environment. + """ + Install specified target into a its own environment. + + Return the process exit code (i.e. 0 in case of success). """ if not skip_environment: create_args = [ From b8873c65ad8c2b14bf00de2330586e606c232787 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 26 Jan 2018 11:34:50 +0000 Subject: [PATCH 2/2] Install the latest version of conda and conda-build Conda 4.3.33 contains a fix that should solve some conda env creation programs: https://github.com/conda/conda/pull/6766 Also support conda installation on x86 Linux architecture. --- lib/galaxy/tools/deps/conda_util.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/tools/deps/conda_util.py b/lib/galaxy/tools/deps/conda_util.py index fe973535c5a..fa4a09e3afb 100644 --- a/lib/galaxy/tools/deps/conda_util.py +++ b/lib/galaxy/tools/deps/conda_util.py @@ -5,9 +5,9 @@ import logging import os import re import shutil +import sys import tempfile from distutils.version import LooseVersion -from sys import platform as _platform import six from six.moves import shlex_quote @@ -23,23 +23,26 @@ log = logging.getLogger(__name__) # break shell commands we are building. SHELL_UNSAFE_PATTERN = re.compile(r"[\s\"']") -IS_OS_X = _platform == "darwin" +IS_OS_X = sys.platform == "darwin" # BSD 3-clause CONDA_LICENSE = "http://docs.continuum.io/anaconda/eula" VERSIONED_ENV_DIR_NAME = re.compile(r"__(.*)@(.*)") UNVERSIONED_ENV_DIR_NAME = re.compile(r"__(.*)@_uv_") USE_PATH_EXEC_DEFAULT = False -CONDA_VERSION = "4.3.24" -CONDA_BUILD_VERSION = "2.1.17" +CONDA_VERSION = "4.3.33" +CONDA_BUILD_VERSION = "2.1.18" USE_LOCAL_DEFAULT = False def conda_link(): if IS_OS_X: - url = "https://repo.continuum.io/miniconda/Miniconda3-4.2.12-MacOSX-x86_64.sh" + url = "https://repo.continuum.io/miniconda/Miniconda3-4.3.31-MacOSX-x86_64.sh" else: - url = "https://repo.continuum.io/miniconda/Miniconda3-4.2.12-Linux-x86_64.sh" + if sys.maxsize > 2**32: + url = "https://repo.continuum.io/miniconda/Miniconda3-4.3.31-Linux-x86_64.sh" + else: + url = "https://repo.continuum.io/miniconda/Miniconda3-4.3.31-Linux-x86.sh" return url