From 82687795413520b5a23e86b17bc3198df3803e39 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 28 Aug 2017 12:01:04 +0200 Subject: [PATCH] Allow paramiko runner to recover from ssh timeout --- lib/galaxy/jobs/runners/util/cli/shell/rsh.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/jobs/runners/util/cli/shell/rsh.py b/lib/galaxy/jobs/runners/util/cli/shell/rsh.py index 204e44ac38a..deac36c3afc 100644 --- a/lib/galaxy/jobs/runners/util/cli/shell/rsh.py +++ b/lib/galaxy/jobs/runners/util/cli/shell/rsh.py @@ -1,4 +1,5 @@ import logging +import time import paramiko @@ -57,6 +58,9 @@ class ParamikoShell(object): self.timeout = int(timeout) if timeout else timeout self.ssh = paramiko.SSHClient() self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self.connect() + + def connect(self): self.ssh.connect(hostname=self.hostname, port=self.port, username=self.username, @@ -65,10 +69,19 @@ class ParamikoShell(object): timeout=self.timeout) def execute(self, cmd, timeout=60): - _, stdout, stderr = self.ssh.exec_command(cmd, timeout=timeout) + try: + _, stdout, stderr = self._execute(cmd, timeout) + except paramiko.SSHException as e: + log.error(e) + time.sleep(10) + self.connect() + _, stdout, stderr = self._execute(cmd, timeout) return_code = stdout.channel.recv_exit_status() return Bunch(stdout=stdout.read(), stderr=stderr.read(), returncode=return_code) + def _execute(self, cmd, timeout): + return self.ssh.exec_command(cmd, timeout=timeout) + class GlobusSecureShell(SecureShell):