Handle driver shutdown, show all test in pytest

This is pretty hackish as fixtures can't be used during test discovery.
To properly shut down the driver I've added a conftest.py file that will
be called when quitting pytest. We might be able to handle this a
little better if we wrote a full plugin, but for now this works.

Also now reports all test indexes, so you can select e.g test 2 of a
tool.

test/functional/conftest.py

@@ -0,0 +1,8 @@
+def pytest_unconfigure(config):
+    try:
+        # This needs to be run if no test were run.
+        from .test_toolbox_pytest import DRIVER
+        DRIVER.tear_down()
+        print("Galaxy test driver shutdown succesfull")
+    except Exception:
+        pass

test/functional/test_data_managers.py

@@ -37,7 +37,7 @@ def build_tests(tmp_dir=None, testing_shed_tools=False, master_api_key=None, use
                  master_api_key=master_api_key,
                  user_api_key=user_api_key,
                  name_prefix="TestForDataManagerTool_",
-                 baselass=DataManagerToolTestCase,
+                 baseclass=DataManagerToolTestCase,
                  user_email=user_email,
                  create_admin=create_admin,
                  G=G,

test/functional/test_toolbox.py

@@ -42,7 +42,7 @@ def build_tests(app=None,
                 master_api_key=None,
                 user_api_key=None,
                 name_prefix="TestForTool_",
-                baselass=ToolTestCase,
+                baseclass=ToolTestCase,
                 create_admin=False,
                 user_email=None,
                 G=None,
@@ -84,7 +84,7 @@ def build_tests(app=None,
         if contains and contains not in tool_id:
                 continue
         name = name_prefix + tool_id.replace(' ', '_')
-        baseclasses = (baselass, )
+        baseclasses = (baseclass, )
         namespace = dict()
 
         all_versions_test_count = 0
@@ -108,6 +108,7 @@ def build_tests(app=None,
                 namespace["galaxy_interactor"] = galaxy_interactor
                 namespace['master_api_key'] = master_api_key
                 namespace['user_api_key'] = user_api_key or galaxy_interactor.api_key
+                namespace['test_count'] = count
 
                 all_versions_test_count += 1
 

test/functional/test_toolbox_pytest.py

@@ -4,6 +4,7 @@ import sys
 galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir + '/' + os.path.pardir))
 sys.path[1:1] = [os.path.join(galaxy_root, "lib"), os.path.join(galaxy_root, "test")]
 
+import pytest
 from base import driver_util
 
 TEST_PREFIX = 'TestForTool_'
@@ -21,25 +22,38 @@ class DefaultGalaxyTestDriver(driver_util.GalaxyTestDriver):
         return self.build_tool_tests(return_test_classes=True)
 
 
-def __generate_testcases():
+def galaxy_driver():
     driver = DefaultGalaxyTestDriver()
     driver.setup()
-    tests = driver.build_tests()
+    return driver
+
+
+@pytest.fixture(scope='module')
+def driver(request):
+    request.addfinalizer(DRIVER.tear_down)
+    return DRIVER
+
+
+def cases():
+    # We setup a global driver, so that the driver fixture can tear down the driver
+    # Ideally `galaxy_driver` or `cases` would be fixtures and clean up after the yield,
+    # but that's not compatible with the use use of pytest.mark.parametrize
+    global DRIVER
+    DRIVER = galaxy_driver()
+    tests = DRIVER.build_tests()
     for test_name, test_class in tests.items():
         if test_name.startswith(TEST_PREFIX):
-            yield (test_name[len(TEST_PREFIX):], test_class)
-    driver.tear_down()
+            test_class.runTest = lambda : None
+            test_instance = test_class()
+            for index in range(test_instance.test_count):
+                yield (test_name[len(TEST_PREFIX):] + "|%d" % index, test_instance, index)
 
 
 def idfn(val):
     return val[0]
 
 
-def pytest_generate_tests(metafunc):
-    if 'tool_test' in metafunc.fixturenames:
-        metafunc.parametrize("tool_test", __generate_testcases(), ids=idfn)
-
-
-def test_tool(tool_test):
-    test = tool_test[1]
-    test.do_it(test)
+@pytest.mark.parametrize("testcases", cases(), ids=idfn)
+def test_tool(testcases, driver):
+    test = testcases[1]
+    test.do_it(test_index=testcases[2])