From 51050653b44c24328d9225b69d9b1b6971b0354c Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 5 Nov 2016 15:15:27 +0100 Subject: [PATCH] Rework dependency shell command handling The approach is to always log a warning with stdout and stderr when the exit code is not 0. With this change, one can expect logs like the follwing: ``` galaxy.tools.deps.conda_util DEBUG 2016-11-05 15:14:17,373 Executing command: /home/mvandenb/miniconda3/bin/conda create -y --name __bowtie@0.12.7 bowtie=0.12.7 127.0.0.1 - - [05/nov./2016:15:14:19 +0200] "POST /admin_toolshed/repository_installation_status_updates HTTP/1.1" 200 - "http://127.0.0.1:8080/admin_toolshed/prepare_for_install" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.90 Safari/537.36" 127.0.0.1 - - [05/nov./2016:15:14:22 +0200] "POST /admin_toolshed/repository_installation_status_updates HTTP/1.1" 200 - "http://127.0.0.1:8080/admin_toolshed/prepare_for_install" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.90 Safari/537.36" galaxy.tools.deps.conda_util WARNING 2016-11-05 15:14:24,029 Failed to execute command-line /home/mvandenb/miniconda3/bin/conda create -y --name __bowtie@0.12.7 bowtie=0.12.7, stderr was: -------->>begin stderr<<-------- PackageNotFoundError: Package not found: '' Package missing in current linux-64 channels: - bowtie 0.12.7* You can search for packages on anaconda.org with anaconda search -t conda bowtie You may need to install the anaconda-client command line client with conda install anaconda-client -------->>end stderr<<-------- -------->>begin stdout<<-------- Fetching package metadata ............... Solving package specifications: . -------->>end stdout<<-------- galaxy.tools.deps.resolvers.conda DEBUG 2016-11-05 15:14:24,030 Removing failed conda install of bowtie, version '0.12.7' ``` This should be helpful for the more obscure errors that can occur with conda. --- lib/galaxy/tools/deps/commands.py | 13 ++++++++++--- lib/galaxy/tools/deps/conda_util.py | 7 ++++++- lib/galaxy/tools/deps/resolvers/conda.py | 10 ++-------- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/tools/deps/commands.py b/lib/galaxy/tools/deps/commands.py index 3b131f8552d..31103b222bf 100644 --- a/lib/galaxy/tools/deps/commands.py +++ b/lib/galaxy/tools/deps/commands.py @@ -34,13 +34,19 @@ def shell(cmds, env=None, **kwds): """Run shell commands with `shell_process` and wait.""" sys = kwds.get("sys", _sys) assert sys is not None - p = shell_process(cmds, env, **kwds) if redirecting_io(sys=sys): + p = shell_process(cmds, env, **kwds) redirect_aware_commmunicate(p, sys=sys) exit = p.returncode return exit else: - return p.wait() + kwds['stdout'] = subprocess.PIPE + kwds['stderr'] = subprocess.PIPE + p = shell_process(cmds, env, **kwds) + stdout, stderr = p.communicate() + if p.returncode != 0: + raise CommandLineException(cmds, stdout, stderr, p.returncode) + return p.returncode def shell_process(cmds, env=None, **kwds): @@ -122,11 +128,12 @@ def download_command(url, to=STDOUT_INDICATOR, quote_url=False): class CommandLineException(Exception): """An exception indicating a non-zero command-line exit.""" - def __init__(self, command, stdout, stderr): + def __init__(self, command, stdout, stderr, returncode): """Construct a CommandLineException from command and standard I/O.""" self.command = command self.stdout = stdout self.stderr = stderr + self.returncode = returncode self.message = ("Failed to execute command-line %s, stderr was:\n" "-------->>begin stderr<<--------\n" "%s\n" diff --git a/lib/galaxy/tools/deps/conda_util.py b/lib/galaxy/tools/deps/conda_util.py index d63dd02055f..66d3c0dc7a5 100644 --- a/lib/galaxy/tools/deps/conda_util.py +++ b/lib/galaxy/tools/deps/conda_util.py @@ -183,7 +183,12 @@ class CondaContext(installable.InstallableContext): condarc_override = self.condarc_override if condarc_override: env["CONDARC"] = condarc_override - return self.shell_exec(command, env=env) + log.debug("Executing command: %s", command) + try: + return self.shell_exec(command, env=env) + except commands.CommandLineException as e: + log.warning(e) + return e.returncode def exec_create(self, args): create_base_args = [ diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py index 1ffe821527f..e55253c6d06 100644 --- a/lib/galaxy/tools/deps/resolvers/conda.py +++ b/lib/galaxy/tools/deps/resolvers/conda.py @@ -136,7 +136,7 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In if not is_installed: return NullDependency(version=version, name=name) - # Have installed conda_target and job_directory to send it too. + # Have installed conda_target and job_directory to send it to. # If dependency is for metadata generation, store environment in conda-metadata-env if kwds.get("metadata", False): conda_env = "conda-metadata-env" @@ -149,7 +149,6 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In copy=self.copy_dependencies, conda_context=self.conda_context, ) - if not exit_code: return CondaDependency( self.conda_context.activate, @@ -159,12 +158,7 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In version ) else: - if len(conda_environment) > 79: - # TODO: remove this once conda_build version 2 is released and packages have been rebuilt. - raise Exception("Conda dependency failed to build job environment. " - "This is most likely a limitation in conda. " - "You can try to shorten the path to the job_working_directory.") - raise Exception("Conda dependency seemingly installed but failed to build job environment.") + return NullDependency(version=version, name=name) def list_dependencies(self): for install_target in installed_conda_targets(self.conda_context):