mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
Restore improvements to job output checking code.
This commit is contained in:
John Chilton
+2
-152
@@ -25,6 +25,7 @@ from galaxy.jobs.runners import BaseJobRunner
|
||||
from galaxy.util.bunch import Bunch
|
||||
from galaxy.util.expressions import ExpressionContext
|
||||
from galaxy.util.json import from_json_string
|
||||
from .output_checker import check_output
|
||||
|
||||
log = logging.getLogger( __name__ )
|
||||
|
||||
@@ -1079,158 +1080,7 @@ class JobWrapper( object ):
|
||||
self.cleanup()
|
||||
|
||||
def check_tool_output( self, stdout, stderr, tool_exit_code, job ):
|
||||
"""
|
||||
Check the output of a tool - given the stdout, stderr, and the tool's
|
||||
exit code, return True if the tool exited succesfully and False
|
||||
otherwise. No exceptions should be thrown. If this code encounters
|
||||
an exception, it returns True so that the workflow can continue;
|
||||
otherwise, a bug in this code could halt workflow progress.
|
||||
Note that, if the tool did not define any exit code handling or
|
||||
any stdio/stderr handling, then it reverts back to previous behavior:
|
||||
if stderr contains anything, then False is returned.
|
||||
Note that the job id is just for messages.
|
||||
"""
|
||||
# By default, the tool succeeded. This covers the case where the code
|
||||
# has a bug but the tool was ok, and it lets a workflow continue.
|
||||
success = True
|
||||
|
||||
try:
|
||||
# Check exit codes and match regular expressions against stdout and
|
||||
# stderr if this tool was configured to do so.
|
||||
# If there is a regular expression for scanning stdout/stderr,
|
||||
# then we assume that the tool writer overwrote the default
|
||||
# behavior of just setting an error if there is *anything* on
|
||||
# stderr.
|
||||
if ( len( self.tool.stdio_regexes ) > 0 or
|
||||
len( self.tool.stdio_exit_codes ) > 0 ):
|
||||
# Check the exit code ranges in the order in which
|
||||
# they were specified. Each exit_code is a StdioExitCode
|
||||
# that includes an applicable range. If the exit code was in
|
||||
# that range, then apply the error level and add a message.
|
||||
# If we've reached a fatal error rule, then stop.
|
||||
max_error_level = galaxy.tools.StdioErrorLevel.NO_ERROR
|
||||
if tool_exit_code != None:
|
||||
for stdio_exit_code in self.tool.stdio_exit_codes:
|
||||
if ( tool_exit_code >= stdio_exit_code.range_start and
|
||||
tool_exit_code <= stdio_exit_code.range_end ):
|
||||
# Tack on a generic description of the code
|
||||
# plus a specific code description. For example,
|
||||
# this might prepend "Job 42: Warning (Out of Memory)\n".
|
||||
code_desc = stdio_exit_code.desc
|
||||
if ( None == code_desc ):
|
||||
code_desc = ""
|
||||
tool_msg = ( "%s: Exit code %d (%s)" % (
|
||||
galaxy.tools.StdioErrorLevel.desc( stdio_exit_code.error_level ),
|
||||
tool_exit_code,
|
||||
code_desc ) )
|
||||
log.info( "Job %s: %s" % (job.get_id_tag(), tool_msg) )
|
||||
stderr = tool_msg + "\n" + stderr
|
||||
max_error_level = max( max_error_level,
|
||||
stdio_exit_code.error_level )
|
||||
if ( max_error_level >=
|
||||
galaxy.tools.StdioErrorLevel.FATAL ):
|
||||
break
|
||||
|
||||
if max_error_level < galaxy.tools.StdioErrorLevel.FATAL:
|
||||
# We'll examine every regex. Each regex specifies whether
|
||||
# it is to be run on stdout, stderr, or both. (It is
|
||||
# possible for neither stdout nor stderr to be scanned,
|
||||
# but those regexes won't be used.) We record the highest
|
||||
# error level, which are currently "warning" and "fatal".
|
||||
# If fatal, then we set the job's state to ERROR.
|
||||
# If warning, then we still set the job's state to OK
|
||||
# but include a message. We'll do this if we haven't seen
|
||||
# a fatal error yet
|
||||
for regex in self.tool.stdio_regexes:
|
||||
# If ( this regex should be matched against stdout )
|
||||
# - Run the regex's match pattern against stdout
|
||||
# - If it matched, then determine the error level.
|
||||
# o If it was fatal, then we're done - break.
|
||||
# Repeat the stdout stuff for stderr.
|
||||
# TODO: Collapse this into a single function.
|
||||
if ( regex.stdout_match ):
|
||||
regex_match = re.search( regex.match, stdout,
|
||||
re.IGNORECASE )
|
||||
if ( regex_match ):
|
||||
rexmsg = self.regex_err_msg( regex_match, regex)
|
||||
log.info( "Job %s: %s"
|
||||
% ( job.get_id_tag(), rexmsg ) )
|
||||
stdout = rexmsg + "\n" + stdout
|
||||
max_error_level = max( max_error_level,
|
||||
regex.error_level )
|
||||
if ( max_error_level >=
|
||||
galaxy.tools.StdioErrorLevel.FATAL ):
|
||||
break
|
||||
|
||||
if ( regex.stderr_match ):
|
||||
regex_match = re.search( regex.match, stderr,
|
||||
re.IGNORECASE )
|
||||
if ( regex_match ):
|
||||
rexmsg = self.regex_err_msg( regex_match, regex)
|
||||
log.info( "Job %s: %s"
|
||||
% ( job.get_id_tag(), rexmsg ) )
|
||||
stderr = rexmsg + "\n" + stderr
|
||||
max_error_level = max( max_error_level,
|
||||
regex.error_level )
|
||||
if ( max_error_level >=
|
||||
galaxy.tools.StdioErrorLevel.FATAL ):
|
||||
break
|
||||
|
||||
# If we encountered a fatal error, then we'll need to set the
|
||||
# job state accordingly. Otherwise the job is ok:
|
||||
if max_error_level >= galaxy.tools.StdioErrorLevel.FATAL:
|
||||
success = False
|
||||
else:
|
||||
success = True
|
||||
|
||||
# When there are no regular expressions and no exit codes to check,
|
||||
# default to the previous behavior: when there's anything on stderr
|
||||
# the job has an error, and the job is ok otherwise.
|
||||
else:
|
||||
# TODO: Add in the tool and job id:
|
||||
# log.debug( "Tool did not define exit code or stdio handling; "
|
||||
# + "checking stderr for success" )
|
||||
if stderr:
|
||||
success = False
|
||||
else:
|
||||
success = True
|
||||
|
||||
# On any exception, return True.
|
||||
except:
|
||||
tb = traceback.format_exc()
|
||||
log.warning( "Tool check encountered unexpected exception; "
|
||||
+ "assuming tool was successful: " + tb )
|
||||
success = True
|
||||
|
||||
# Store the modified stdout and stderr in the job:
|
||||
if None != job:
|
||||
job.stdout = stdout
|
||||
job.stderr = stderr
|
||||
|
||||
return success
|
||||
|
||||
def regex_err_msg( self, match, regex ):
|
||||
"""
|
||||
Return a message about the match on tool output using the given
|
||||
ToolStdioRegex regex object. The regex_match is a MatchObject
|
||||
that will contain the string matched on.
|
||||
"""
|
||||
# Get the description for the error level:
|
||||
err_msg = galaxy.tools.StdioErrorLevel.desc( regex.error_level ) + ": "
|
||||
# If there's a description for the regular expression, then use it.
|
||||
# Otherwise, we'll take the first 256 characters of the match.
|
||||
if None != regex.desc:
|
||||
err_msg += regex.desc
|
||||
else:
|
||||
mstart = match.start()
|
||||
mend = match.end()
|
||||
err_msg += "Matched on "
|
||||
# TODO: Move the constant 256 somewhere else besides here.
|
||||
if mend - mstart > 256:
|
||||
err_msg += match.string[ mstart : mstart+256 ] + "..."
|
||||
else:
|
||||
err_msg += match.string[ mstart: mend ]
|
||||
return err_msg
|
||||
return check_output( self.tool, stdout, stderr, tool_exit_code, job )
|
||||
|
||||
def cleanup( self ):
|
||||
# remove temporary files
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
|
||||
|
||||
# These determine stdio-based error levels from matching on regular expressions
|
||||
# and exit codes. They are meant to be used comparatively, such as showing
|
||||
# that warning < fatal. This is really meant to just be an enum.
|
||||
class StdioErrorLevel( object ):
|
||||
NO_ERROR = 0
|
||||
LOG = 1
|
||||
WARNING = 2
|
||||
FATAL = 3
|
||||
MAX = 3
|
||||
descs = {
|
||||
NO_ERROR: 'No error',
|
||||
LOG: 'Log',
|
||||
WARNING: 'Warning',
|
||||
FATAL: 'Fatal error',
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def desc( error_level ):
|
||||
err_msg = "Unknown error"
|
||||
if ( error_level > 0 and
|
||||
error_level <= StdioErrorLevel.MAX ):
|
||||
err_msg = StdioErrorLevel.descs[ error_level ]
|
||||
return err_msg
|
||||
@@ -0,0 +1,164 @@
|
||||
import re
|
||||
from .error_level import StdioErrorLevel
|
||||
import traceback
|
||||
|
||||
from logging import getLogger
|
||||
log = getLogger( __name__ )
|
||||
|
||||
|
||||
def check_output( tool, stdout, stderr, tool_exit_code, job ):
|
||||
"""
|
||||
Check the output of a tool - given the stdout, stderr, and the tool's
|
||||
exit code, return True if the tool exited succesfully and False
|
||||
otherwise. No exceptions should be thrown. If this code encounters
|
||||
an exception, it returns True so that the workflow can continue;
|
||||
otherwise, a bug in this code could halt workflow progress.
|
||||
|
||||
Note that, if the tool did not define any exit code handling or
|
||||
any stdio/stderr handling, then it reverts back to previous behavior:
|
||||
if stderr contains anything, then False is returned.
|
||||
|
||||
Note that the job id is just for messages.
|
||||
"""
|
||||
# By default, the tool succeeded. This covers the case where the code
|
||||
# has a bug but the tool was ok, and it lets a workflow continue.
|
||||
success = True
|
||||
|
||||
try:
|
||||
# Check exit codes and match regular expressions against stdout and
|
||||
# stderr if this tool was configured to do so.
|
||||
# If there is a regular expression for scanning stdout/stderr,
|
||||
# then we assume that the tool writer overwrote the default
|
||||
# behavior of just setting an error if there is *anything* on
|
||||
# stderr.
|
||||
if ( len( tool.stdio_regexes ) > 0 or
|
||||
len( tool.stdio_exit_codes ) > 0 ):
|
||||
# Check the exit code ranges in the order in which
|
||||
# they were specified. Each exit_code is a StdioExitCode
|
||||
# that includes an applicable range. If the exit code was in
|
||||
# that range, then apply the error level and add a message.
|
||||
# If we've reached a fatal error rule, then stop.
|
||||
max_error_level = StdioErrorLevel.NO_ERROR
|
||||
if tool_exit_code != None:
|
||||
for stdio_exit_code in tool.stdio_exit_codes:
|
||||
if ( tool_exit_code >= stdio_exit_code.range_start and
|
||||
tool_exit_code <= stdio_exit_code.range_end ):
|
||||
# Tack on a generic description of the code
|
||||
# plus a specific code description. For example,
|
||||
# this might prepend "Job 42: Warning (Out of Memory)\n".
|
||||
code_desc = stdio_exit_code.desc
|
||||
if ( None == code_desc ):
|
||||
code_desc = ""
|
||||
tool_msg = ( "%s: Exit code %d (%s)" % (
|
||||
StdioErrorLevel.desc( stdio_exit_code.error_level ),
|
||||
tool_exit_code,
|
||||
code_desc ) )
|
||||
log.info( "Job %s: %s" % (job.get_id_tag(), tool_msg) )
|
||||
stderr = tool_msg + "\n" + stderr
|
||||
max_error_level = max( max_error_level,
|
||||
stdio_exit_code.error_level )
|
||||
if ( max_error_level >=
|
||||
StdioErrorLevel.FATAL ):
|
||||
break
|
||||
|
||||
if max_error_level < StdioErrorLevel.FATAL:
|
||||
# We'll examine every regex. Each regex specifies whether
|
||||
# it is to be run on stdout, stderr, or both. (It is
|
||||
# possible for neither stdout nor stderr to be scanned,
|
||||
# but those regexes won't be used.) We record the highest
|
||||
# error level, which are currently "warning" and "fatal".
|
||||
# If fatal, then we set the job's state to ERROR.
|
||||
# If warning, then we still set the job's state to OK
|
||||
# but include a message. We'll do this if we haven't seen
|
||||
# a fatal error yet
|
||||
for regex in tool.stdio_regexes:
|
||||
# If ( this regex should be matched against stdout )
|
||||
# - Run the regex's match pattern against stdout
|
||||
# - If it matched, then determine the error level.
|
||||
# o If it was fatal, then we're done - break.
|
||||
# Repeat the stdout stuff for stderr.
|
||||
# TODO: Collapse this into a single function.
|
||||
if ( regex.stdout_match ):
|
||||
regex_match = re.search( regex.match, stdout,
|
||||
re.IGNORECASE )
|
||||
if ( regex_match ):
|
||||
rexmsg = __regex_err_msg( regex_match, regex)
|
||||
log.info( "Job %s: %s"
|
||||
% ( job.get_id_tag(), rexmsg ) )
|
||||
stdout = rexmsg + "\n" + stdout
|
||||
max_error_level = max( max_error_level,
|
||||
regex.error_level )
|
||||
if ( max_error_level >=
|
||||
StdioErrorLevel.FATAL ):
|
||||
break
|
||||
|
||||
if ( regex.stderr_match ):
|
||||
regex_match = re.search( regex.match, stderr,
|
||||
re.IGNORECASE )
|
||||
if ( regex_match ):
|
||||
rexmsg = __regex_err_msg( regex_match, regex)
|
||||
log.info( "Job %s: %s"
|
||||
% ( job.get_id_tag(), rexmsg ) )
|
||||
stderr = rexmsg + "\n" + stderr
|
||||
max_error_level = max( max_error_level,
|
||||
regex.error_level )
|
||||
if ( max_error_level >=
|
||||
StdioErrorLevel.FATAL ):
|
||||
break
|
||||
|
||||
# If we encountered a fatal error, then we'll need to set the
|
||||
# job state accordingly. Otherwise the job is ok:
|
||||
if max_error_level >= StdioErrorLevel.FATAL:
|
||||
success = False
|
||||
else:
|
||||
success = True
|
||||
|
||||
# When there are no regular expressions and no exit codes to check,
|
||||
# default to the previous behavior: when there's anything on stderr
|
||||
# the job has an error, and the job is ok otherwise.
|
||||
else:
|
||||
# TODO: Add in the tool and job id:
|
||||
# log.debug( "Tool did not define exit code or stdio handling; "
|
||||
# + "checking stderr for success" )
|
||||
if stderr:
|
||||
success = False
|
||||
else:
|
||||
success = True
|
||||
|
||||
# On any exception, return True.
|
||||
except:
|
||||
tb = traceback.format_exc()
|
||||
log.warning( "Tool check encountered unexpected exception; "
|
||||
+ "assuming tool was successful: " + tb )
|
||||
success = True
|
||||
|
||||
# Store the modified stdout and stderr in the job:
|
||||
if None != job:
|
||||
job.stdout = stdout
|
||||
job.stderr = stderr
|
||||
|
||||
return success
|
||||
|
||||
|
||||
def __regex_err_msg( match, regex ):
|
||||
"""
|
||||
Return a message about the match on tool output using the given
|
||||
ToolStdioRegex regex object. The regex_match is a MatchObject
|
||||
that will contain the string matched on.
|
||||
"""
|
||||
# Get the description for the error level:
|
||||
err_msg = StdioErrorLevel.desc( regex.error_level ) + ": "
|
||||
# If there's a description for the regular expression, then use it.
|
||||
# Otherwise, we'll take the first 256 characters of the match.
|
||||
if None != regex.desc:
|
||||
err_msg += regex.desc
|
||||
else:
|
||||
mstart = match.start()
|
||||
mend = match.end()
|
||||
err_msg += "Matched on "
|
||||
# TODO: Move the constant 256 somewhere else besides here.
|
||||
if mend - mstart > 256:
|
||||
err_msg += match.string[ mstart : mstart + 256 ] + "..."
|
||||
else:
|
||||
err_msg += match.string[ mstart: mend ]
|
||||
return err_msg
|
||||
@@ -34,6 +34,7 @@ from paste import httpexceptions
|
||||
from sqlalchemy import and_
|
||||
|
||||
from galaxy import jobs, model
|
||||
from galaxy.jobs.error_level import StdioErrorLevel
|
||||
from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper
|
||||
from galaxy.jobs import ParallelismInfo
|
||||
from galaxy.tools.actions import DefaultToolAction
|
||||
@@ -64,33 +65,11 @@ from galaxy.model.item_attrs import Dictifiable
|
||||
from tool_shed.util import shed_util_common
|
||||
from .loader import load_tool, template_macro_params
|
||||
|
||||
|
||||
log = logging.getLogger( __name__ )
|
||||
|
||||
WORKFLOW_PARAMETER_REGULAR_EXPRESSION = re.compile( '''\$\{.+?\}''' )
|
||||
|
||||
# These determine stdio-based error levels from matching on regular expressions
|
||||
# and exit codes. They are meant to be used comparatively, such as showing
|
||||
# that warning < fatal. This is really meant to just be an enum.
|
||||
class StdioErrorLevel( object ):
|
||||
NO_ERROR = 0
|
||||
LOG = 1
|
||||
WARNING = 2
|
||||
FATAL = 3
|
||||
MAX = 3
|
||||
descs = {
|
||||
NO_ERROR : 'No error',
|
||||
LOG: 'Log',
|
||||
WARNING : 'Warning',
|
||||
FATAL : 'Fatal error'
|
||||
}
|
||||
@staticmethod
|
||||
def desc( error_level ):
|
||||
err_msg = "Unknown error"
|
||||
if ( error_level > 0 and
|
||||
error_level <= StdioErrorLevel.MAX ):
|
||||
err_msg = StdioErrorLevel.descs[ error_level ]
|
||||
return err_msg
|
||||
|
||||
class ToolNotFoundException( Exception ):
|
||||
pass
|
||||
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
from unittest import TestCase
|
||||
from galaxy.util.bunch import Bunch
|
||||
from galaxy.jobs.output_checker import check_output
|
||||
from galaxy.jobs.error_level import StdioErrorLevel
|
||||
|
||||
|
||||
class OutputCheckerTestCase( TestCase ):
|
||||
|
||||
def setUp( self ):
|
||||
self.tool = Bunch(
|
||||
stdio_regexes=[],
|
||||
stdio_exit_codes=[],
|
||||
)
|
||||
self.job = Bunch(
|
||||
stdout=None,
|
||||
stderr=None,
|
||||
get_id_tag=lambda: "test_id",
|
||||
)
|
||||
self.stdout = ''
|
||||
self.stderr = ''
|
||||
self.tool_exit_code = None
|
||||
|
||||
def test_default_no_stderr_success( self ):
|
||||
self.__assertSuccessful()
|
||||
|
||||
def test_default_stderr_failure( self ):
|
||||
self.stderr = 'foo'
|
||||
self.__assertNotSuccessful()
|
||||
|
||||
def test_exit_code_error( self ):
|
||||
mock_exit_code = Bunch( range_start=1, range_end=1, error_level=StdioErrorLevel.FATAL, desc=None )
|
||||
self.tool.stdio_exit_codes.append( mock_exit_code )
|
||||
self.tool_exit_code = 1
|
||||
self.__assertNotSuccessful()
|
||||
|
||||
def test_exit_code_success( self ):
|
||||
mock_exit_code = Bunch( range_start=1, range_end=1, error_level=StdioErrorLevel.FATAL, desc=None )
|
||||
self.tool.stdio_exit_codes.append( mock_exit_code )
|
||||
self.tool_exit_code = 0
|
||||
self.__assertSuccessful()
|
||||
|
||||
def test_problematic_strings( self ):
|
||||
problematic_str = '\x80abc'
|
||||
regex_rule = Bunch( match=r'.abc', stdout_match=False, stderr_match=True, error_level=StdioErrorLevel.FATAL, desc=None )
|
||||
self.tool.stdio_regexes = [ regex_rule ]
|
||||
self.stderr = problematic_str
|
||||
self.__assertNotSuccessful()
|
||||
|
||||
problematic_str = '\x80abc'
|
||||
regex_rule = Bunch( match=r'.abcd', stdout_match=False, stderr_match=True, error_level=StdioErrorLevel.FATAL, desc=None )
|
||||
self.tool.stdio_regexes = [ regex_rule ]
|
||||
self.stderr = problematic_str
|
||||
self.__assertSuccessful()
|
||||
|
||||
def __assertSuccessful( self ):
|
||||
self.assertTrue( self.__check_output() )
|
||||
|
||||
def __assertNotSuccessful( self ):
|
||||
self.assertFalse( self.__check_output() )
|
||||
|
||||
def __check_output( self ):
|
||||
return check_output( self.tool, self.stdout, self.stderr, self.tool_exit_code, self.job )
|
||||
Reference in New Issue
Block a user