Implement structured tool test data.

Rather than relying solely on exceptions back to nose/test framework - add option (--structured_data_report_file) to run_tests.sh that causes a bunch of detailed data to be dumped to the specified file in a very structured way. Includes full to_dict of the job from the API which in turn includes job metrics, command-line, job's standard error and outputs (instead of the test frameworks), as well as the tool inputs, and exceptions broken out for tool execution versus output checking.

Its all indexed in the file by the test id (without the actual test toolbox depending on knowing the test id) - so one could pair this information with the XUnit output to produce much more detailed breakdowns of the tests.

run_tests.sh

@@ -47,6 +47,7 @@ ensure_grunt() {
 test_script="./scripts/functional_tests.py"
 report_file="run_functional_tests.html"
 xunit_report_file=""
+structured_data_report_file=""
 with_framework_test_tools_arg=""
 
 driver="python"
@@ -156,6 +157,15 @@ do
               exit 1
           fi
           ;;
+      --structured_data_report_file)
+          if [ $# -gt 1 ]; then
+              structured_data_report_file=$2
+              shift 2
+          else
+              echo "--structured_data_report_file requires an argument" 1>&2
+              exit 1
+          fi
+          ;;
       -c|--coverage)
           # Must have coverage installed (try `which coverage`) - only valid with --unit
           # for now. Would be great to get this to work with functional tests though.
@@ -249,7 +259,12 @@ if [ "$driver" = "python" ]; then
     else
         xunit_args=""
     fi
-    python $test_script $coverage_arg -v --with-nosehtml --html-report-file $report_file $xunit_args $with_framework_test_tools_arg $extra_args
+    if [ -n "$structured_data_report_file" ]; then
+        structured_data_args="--with-structureddata --structured-data-file $structured_data_report_file"
+    else
+        structured_data_args=""
+    fi
+    python $test_script $coverage_arg -v --with-nosehtml --html-report-file $report_file $xunit_args $structured_data_args $with_framework_test_tools_arg $extra_args
 else
     ensure_grunt
     if [ -n "$watch" ]; then

scripts/functional_tests.py

@@ -55,6 +55,7 @@ from functional import database_contexts
 from base.api_util import get_master_api_key
 from base.api_util import get_user_api_key
 from base.nose_util import run
+from base.instrument import StructuredTestDataPlugin
 
 import nose.core
 import nose.config
@@ -439,6 +440,7 @@ def main():
                     user_api_key=get_user_api_key(),
                 )
             test_config = nose.config.Config( env=os.environ, ignoreFiles=ignore_files, plugins=nose.plugins.manager.DefaultPluginManager() )
+            test_config.plugins.addPlugin( StructuredTestDataPlugin() )
             test_config.configure( sys.argv )
             result = run_tests( test_config )
             success = result.wasSuccessful()

test/base/instrument.py

@@ -0,0 +1,89 @@
+""" Utilities to help instrument tool tests.
+
+Including structed data nose plugin that allows storing arbitrary structured
+data on a per test case basis - used by tool test to store inputs,
+output problems, job tests, etc... but could easily by used by other test
+types in a different way.
+"""
+
+import json
+import threading
+
+try:
+    from galaxy import eggs
+    eggs.require( "nose" )
+except ImportError:
+    pass
+
+
+from nose.plugins import Plugin
+
+NO_JOB_DATA = object()
+JOB_DATA = threading.local()
+JOB_DATA.new = True
+JOB_DATA.data = NO_JOB_DATA
+
+
+def register_job_data(data):
+    if not JOB_DATA.new:
+        return
+    JOB_DATA.data = data
+    JOB_DATA.new = False
+
+
+def fetch_job_data():
+    try:
+        if JOB_DATA.new:
+            return NO_JOB_DATA
+        else:
+            return JOB_DATA.data
+    finally:
+        JOB_DATA.new = True
+
+
+class StructuredTestDataPlugin( Plugin ):
+    name = 'structureddata'
+
+    def options(self, parser, env):
+        super(StructuredTestDataPlugin, self).options(parser, env=env)
+        parser.add_option(
+            '--structured-data-file', action='store',
+            dest='structured_data_file', metavar="FILE",
+            default=env.get('NOSE_STRUCTURED_DATA', 'structured_test_data.json'),
+            help=("Path to JSON file to store the Galaxy structured data report in."
+                  "Default is structured_test_data.json in the working directory "
+                  "[NOSE_STRUCTURED_DATA]"))
+
+    def configure(self, options, conf):
+        super(StructuredTestDataPlugin, self).configure(options, conf)
+        self.conf = conf
+        if not self.enabled:
+            return
+        self.tests = []
+        self.structured_data_report_file = open(options.structured_data_file, 'w')
+
+    def finalize(self, result):
+        pass
+
+    def _handle_result(self, test, *args, **kwds):
+        job_data = fetch_job_data()
+        id = test.id()
+        has_data = job_data is not NO_JOB_DATA
+        entry = {
+            'id': id,
+            'has_data': has_data,
+            'data': job_data if has_data else None,
+        }
+        self.tests.append(entry)
+
+    addError = _handle_result
+    addFailure = _handle_result
+    addSuccess = _handle_result
+
+    def report(self, stream):
+        report_obj = {
+            'version': '0.1',
+            'tests': self.tests,
+        }
+        json.dump(report_obj, self.structured_data_report_file)
+        self.structured_data_report_file.close()

test/base/interactor.py

@@ -6,6 +6,7 @@ from galaxy import eggs
 eggs.require( "requests" )
 from galaxy import util
 from galaxy.util.odict import odict
+from galaxy.util.bunch import Bunch
 from requests import get
 from requests import post
 from json import dumps
@@ -210,10 +211,14 @@ class GalaxyInteractorApi( object ):
         submit_response = self.__submit_tool( history_id, tool_id=testdef.tool.id, tool_input=inputs_tree )
         submit_response_object = submit_response.json()
         try:
-            return self.__dictify_outputs( submit_response_object ), submit_response_object[ 'jobs' ]
+            return Bunch(
+                inputs=inputs_tree,
+                outputs=self.__dictify_outputs( submit_response_object ),
+                jobs=submit_response_object[ 'jobs' ],
+            )
         except KeyError:
             message = "Error creating a job for these tool inputs - %s" % submit_response_object[ 'message' ]
-            raise Exception( message )
+            raise RunToolException( message, inputs_tree )
 
     def _create_collection( self, history_id, collection_def ):
         create_payload = dict(
@@ -404,6 +409,13 @@ class GalaxyInteractorApi( object ):
         return get( url, params=data )
 
 
+class RunToolException(Exception):
+
+    def __init__(self, message, inputs=None):
+        super(RunToolException, self).__init__(message)
+        self.inputs = inputs
+
+
 GALAXY_INTERACTORS = {
     'api': GalaxyInteractorApi,
 }

test/functional/test_toolbox.py

@@ -1,7 +1,8 @@
 import new
 import sys
 from base.twilltestcase import TwillTestCase
-from base.interactor import build_interactor, stage_data_in_history
+from base.interactor import build_interactor, stage_data_in_history, RunToolException
+from base.instrument import register_job_data
 from galaxy.tools import DataManagerTool
 from galaxy.util import bunch
 import logging
@@ -34,10 +35,48 @@ class ToolTestCase( TwillTestCase ):
 
         stage_data_in_history( galaxy_interactor, testdef.test_data(), test_history, shed_tool_id )
 
-        data_list, jobs = galaxy_interactor.run_tool( testdef, test_history )
-        self.assertTrue( data_list )
+        # Once data is ready, run the tool and check the outputs - record API
+        # input, job info, tool run exception, as well as exceptions related to
+        # job output checking and register they with the test plugin so it can
+        # record structured information.
+        tool_inputs = None
+        job_stdio = None
+        job_output_exceptions = None
+        tool_execution_exception = None
+        try:
+            try:
+                tool_response = galaxy_interactor.run_tool( testdef, test_history )
+                data_list, jobs, tool_inputs = tool_response.outputs, tool_response.jobs, tool_response.inputs
+            except RunToolException as e:
+                tool_inputs = e.inputs
+                tool_execution_exception = e
+                raise e
+            except Exception as e:
+                tool_execution_exception = e
+                raise e
 
-        self._verify_outputs( testdef, test_history, jobs, shed_tool_id, data_list, galaxy_interactor )
+            self.assertTrue( data_list )
+
+            try:
+                job_stdio = self._verify_outputs( testdef, test_history, jobs, shed_tool_id, data_list, galaxy_interactor )
+            except JobOutputsError as e:
+                job_stdio = e.job_stdio
+                job_output_exceptions = e.output_exceptions
+                raise e
+            except Exception as e:
+                job_output_exceptions = [e]
+                raise e
+        finally:
+            job_data = {}
+            if tool_inputs is not None:
+                job_data["inputs"] = tool_inputs
+            if job_stdio is not None:
+                job_data["job"] = job_stdio
+            if job_output_exceptions:
+                job_data["output_problems"] = map(str, job_output_exceptions)
+            if tool_execution_exception:
+                job_data["execution_problem"] = str(tool_execution_exception)
+            register_job_data(job_data)
 
         galaxy_interactor.delete_history( test_history )
 
@@ -63,6 +102,7 @@ class ToolTestCase( TwillTestCase ):
                 raise Exception( message )
         found_exceptions = []
 
+        job_stdio = None
         for output_index, output_tuple in enumerate(testdef.outputs):
             # Get the correct hid
             name, outfile, attributes = output_tuple
@@ -89,9 +129,22 @@ class ToolTestCase( TwillTestCase ):
                             if stream in job_stdio:
                                 print >>sys.stderr, self._format_stream( job_stdio[ stream ], stream=stream, format=True )
                 found_exceptions.append(e)
+        if job_stdio is None:
+            job_stdio = galaxy_interactor.get_job_stdio( jobs[0][ 'id' ] )
+
         if found_exceptions:
-            big_message = "\n".join(map(str, found_exceptions))
-            raise AssertionError(big_message)
+            raise JobOutputsError(found_exceptions, job_stdio)
+        else:
+            return job_stdio
+
+
+class JobOutputsError(AssertionError):
+
+    def __init__(self, output_exceptions, job_stdio):
+        big_message = "\n".join(map(str, output_exceptions))
+        super(JobOutputsError, self).__init__(big_message)
+        self.job_stdio = job_stdio
+        self.output_exceptions = output_exceptions
 
 
 @nottest