diff --git a/run_tests.sh b/run_tests.sh
index 217d7657bbd..f3592854f43 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -47,6 +47,7 @@ ensure_grunt() {
 test_script="./scripts/functional_tests.py"
 report_file="run_functional_tests.html"
 xunit_report_file=""
+structured_data_report_file=""
 with_framework_test_tools_arg=""
 
 driver="python"
@@ -156,6 +157,15 @@ do
               exit 1
           fi
           ;;
+      --structured_data_report_file)
+          if [ $# -gt 1 ]; then
+              structured_data_report_file=$2
+              shift 2
+          else
+              echo "--structured_data_report_file requires an argument" 1>&2
+              exit 1
+          fi
+          ;;
       -c|--coverage)
           # Must have coverage installed (try `which coverage`) - only valid with --unit
           # for now. Would be great to get this to work with functional tests though.
@@ -249,7 +259,12 @@ if [ "$driver" = "python" ]; then
     else
         xunit_args=""
     fi
-    python $test_script $coverage_arg -v --with-nosehtml --html-report-file $report_file $xunit_args $with_framework_test_tools_arg $extra_args
+    if [ -n "$structured_data_report_file" ]; then
+        structured_data_args="--with-structureddata --structured-data-file $structured_data_report_file"
+    else
+        structured_data_args=""
+    fi
+    python $test_script $coverage_arg -v --with-nosehtml --html-report-file $report_file $xunit_args $structured_data_args $with_framework_test_tools_arg $extra_args
 else
     ensure_grunt
     if [ -n "$watch" ]; then
diff --git a/scripts/functional_tests.py b/scripts/functional_tests.py
index 8e7c3ca6b7b..a7864894d4f 100644
--- a/scripts/functional_tests.py
+++ b/scripts/functional_tests.py
@@ -55,6 +55,7 @@ from functional import database_contexts
 from base.api_util import get_master_api_key
 from base.api_util import get_user_api_key
 from base.nose_util import run
+from base.instrument import StructuredTestDataPlugin
 
 import nose.core
 import nose.config
@@ -439,6 +440,7 @@ def main():
                     user_api_key=get_user_api_key(),
                 )
             test_config = nose.config.Config( env=os.environ, ignoreFiles=ignore_files, plugins=nose.plugins.manager.DefaultPluginManager() )
+            test_config.plugins.addPlugin( StructuredTestDataPlugin() )
             test_config.configure( sys.argv )
             result = run_tests( test_config )
             success = result.wasSuccessful()
diff --git a/test/base/instrument.py b/test/base/instrument.py
new file mode 100644
index 00000000000..dc7d92b9958
--- /dev/null
+++ b/test/base/instrument.py
@@ -0,0 +1,89 @@
+""" Utilities to help instrument tool tests.
+
+Including structed data nose plugin that allows storing arbitrary structured
+data on a per test case basis - used by tool test to store inputs,
+output problems, job tests, etc... but could easily by used by other test
+types in a different way.
+"""
+
+import json
+import threading
+
+try:
+    from galaxy import eggs
+    eggs.require( "nose" )
+except ImportError:
+    pass
+
+
+from nose.plugins import Plugin
+
+NO_JOB_DATA = object()
+JOB_DATA = threading.local()
+JOB_DATA.new = True
+JOB_DATA.data = NO_JOB_DATA
+
+
+def register_job_data(data):
+    if not JOB_DATA.new:
+        return
+    JOB_DATA.data = data
+    JOB_DATA.new = False
+
+
+def fetch_job_data():
+    try:
+        if JOB_DATA.new:
+            return NO_JOB_DATA
+        else:
+            return JOB_DATA.data
+    finally:
+        JOB_DATA.new = True
+
+
+class StructuredTestDataPlugin( Plugin ):
+    name = 'structureddata'
+
+    def options(self, parser, env):
+        super(StructuredTestDataPlugin, self).options(parser, env=env)
+        parser.add_option(
+            '--structured-data-file', action='store',
+            dest='structured_data_file', metavar="FILE",
+            default=env.get('NOSE_STRUCTURED_DATA', 'structured_test_data.json'),
+            help=("Path to JSON file to store the Galaxy structured data report in."
+                  "Default is structured_test_data.json in the working directory "
+                  "[NOSE_STRUCTURED_DATA]"))
+
+    def configure(self, options, conf):
+        super(StructuredTestDataPlugin, self).configure(options, conf)
+        self.conf = conf
+        if not self.enabled:
+            return
+        self.tests = []
+        self.structured_data_report_file = open(options.structured_data_file, 'w')
+
+    def finalize(self, result):
+        pass
+
+    def _handle_result(self, test, *args, **kwds):
+        job_data = fetch_job_data()
+        id = test.id()
+        has_data = job_data is not NO_JOB_DATA
+        entry = {
+            'id': id,
+            'has_data': has_data,
+            'data': job_data if has_data else None,
+        }
+        self.tests.append(entry)
+
+    addError = _handle_result
+    addFailure = _handle_result
+    addSuccess = _handle_result
+
+    def report(self, stream):
+        report_obj = {
+            'version': '0.1',
+            'tests': self.tests,
+        }
+        json.dump(report_obj, self.structured_data_report_file)
+        self.structured_data_report_file.close()
diff --git a/test/base/interactor.py b/test/base/interactor.py
index 52fdc45b8d2..7e409c8edad 100644
--- a/test/base/interactor.py
+++ b/test/base/interactor.py
@@ -6,6 +6,7 @@ from galaxy import eggs
 eggs.require( "requests" )
 from galaxy import util
 from galaxy.util.odict import odict
+from galaxy.util.bunch import Bunch
 from requests import get
 from requests import post
 from json import dumps
@@ -210,10 +211,14 @@ class GalaxyInteractorApi( object ):
         submit_response = self.__submit_tool( history_id, tool_id=testdef.tool.id, tool_input=inputs_tree )
         submit_response_object = submit_response.json()
         try:
-            return self.__dictify_outputs( submit_response_object ), submit_response_object[ 'jobs' ]
+            return Bunch(
+                inputs=inputs_tree,
+                outputs=self.__dictify_outputs( submit_response_object ),
+                jobs=submit_response_object[ 'jobs' ],
+            )
         except KeyError:
             message = "Error creating a job for these tool inputs - %s" % submit_response_object[ 'message' ]
-            raise Exception( message )
+            raise RunToolException( message, inputs_tree )
 
     def _create_collection( self, history_id, collection_def ):
         create_payload = dict(
@@ -404,6 +409,13 @@ class GalaxyInteractorApi( object ):
         return get( url, params=data )
 
 
+class RunToolException(Exception):
+
+    def __init__(self, message, inputs=None):
+        super(RunToolException, self).__init__(message)
+        self.inputs = inputs
+
+
 GALAXY_INTERACTORS = {
     'api': GalaxyInteractorApi,
 }
diff --git a/test/functional/test_toolbox.py b/test/functional/test_toolbox.py
index cc963560498..f2fb28e8867 100644
--- a/test/functional/test_toolbox.py
+++ b/test/functional/test_toolbox.py
@@ -1,7 +1,8 @@
 import new
 import sys
 from base.twilltestcase import TwillTestCase
-from base.interactor import build_interactor, stage_data_in_history
+from base.interactor import build_interactor, stage_data_in_history, RunToolException
+from base.instrument import register_job_data
 from galaxy.tools import DataManagerTool
 from galaxy.util import bunch
 import logging
@@ -34,10 +35,48 @@ class ToolTestCase( TwillTestCase ):
 
         stage_data_in_history( galaxy_interactor, testdef.test_data(), test_history, shed_tool_id )
 
-        data_list, jobs = galaxy_interactor.run_tool( testdef, test_history )
-        self.assertTrue( data_list )
+        # Once data is ready, run the tool and check the outputs - record API
+        # input, job info, tool run exception, as well as exceptions related to
+        # job output checking and register they with the test plugin so it can
+        # record structured information.
+        tool_inputs = None
+        job_stdio = None
+        job_output_exceptions = None
+        tool_execution_exception = None
+        try:
+            try:
+                tool_response = galaxy_interactor.run_tool( testdef, test_history )
+                data_list, jobs, tool_inputs = tool_response.outputs, tool_response.jobs, tool_response.inputs
+            except RunToolException as e:
+                tool_inputs = e.inputs
+                tool_execution_exception = e
+                raise e
+            except Exception as e:
+                tool_execution_exception = e
+                raise e
 
-        self._verify_outputs( testdef, test_history, jobs, shed_tool_id, data_list, galaxy_interactor )
+            self.assertTrue( data_list )
+
+            try:
+                job_stdio = self._verify_outputs( testdef, test_history, jobs, shed_tool_id, data_list, galaxy_interactor )
+            except JobOutputsError as e:
+                job_stdio = e.job_stdio
+                job_output_exceptions = e.output_exceptions
+                raise e
+            except Exception as e:
+                job_output_exceptions = [e]
+                raise e
+        finally:
+            job_data = {}
+            if tool_inputs is not None:
+                job_data["inputs"] = tool_inputs
+            if job_stdio is not None:
+                job_data["job"] = job_stdio
+            if job_output_exceptions:
+                job_data["output_problems"] = map(str, job_output_exceptions)
+            if tool_execution_exception:
+                job_data["execution_problem"] = str(tool_execution_exception)
+            register_job_data(job_data)
 
         galaxy_interactor.delete_history( test_history )
 
@@ -63,6 +102,7 @@ class ToolTestCase( TwillTestCase ):
                 raise Exception( message )
         found_exceptions = []
 
+        job_stdio = None
         for output_index, output_tuple in enumerate(testdef.outputs):
             # Get the correct hid
             name, outfile, attributes = output_tuple
@@ -89,9 +129,22 @@ class ToolTestCase( TwillTestCase ):
                             if stream in job_stdio:
                                 print >>sys.stderr, self._format_stream( job_stdio[ stream ], stream=stream, format=True )
                 found_exceptions.append(e)
+        if job_stdio is None:
+            job_stdio = galaxy_interactor.get_job_stdio( jobs[0][ 'id' ] )
+
         if found_exceptions:
-            big_message = "\n".join(map(str, found_exceptions))
-            raise AssertionError(big_message)
+            raise JobOutputsError(found_exceptions, job_stdio)
+        else:
+            return job_stdio
+
+
+class JobOutputsError(AssertionError):
+
+    def __init__(self, output_exceptions, job_stdio):
+        big_message = "\n".join(map(str, output_exceptions))
+        super(JobOutputsError, self).__init__(big_message)
+        self.job_stdio = job_stdio
+        self.output_exceptions = output_exceptions
 
 
 @nottest