Merge pull request #2016 from jmchilton/test_driver_cleanup

Functional Test Drivers Overhaul
2026-09-24 16:30:27 +08:00 · 2016-03-29 14:36:09 -04:00
parent 2687002a27 b8ecf36f2a
commit d693aaf624
10 changed files with 1015 additions and 962 deletions
@@ -91,7 +91,6 @@ class Configuration( object ):
        self.tool_data_path = resolve_path( kwargs.get( "tool_data_path", "tool-data" ), os.getcwd() )
        self.builds_file_path = resolve_path( kwargs.get( "builds_file_path", os.path.join( self.tool_data_path, 'shared', 'ucsc', 'builds.txt') ), self.root )
        self.len_file_path = resolve_path( kwargs.get( "len_file_path", os.path.join( self.tool_data_path, 'shared', 'ucsc', 'chrom') ), self.root )
-        self.test_conf = resolve_path( kwargs.get( "test_conf", "" ), self.root )
        # The value of migrated_tools_config is the file reserved for containing only those tools that have been eliminated from the distribution
        # and moved to the tool shed.
        self.integrated_tool_panel_config = resolve_path( kwargs.get( 'integrated_tool_panel_config', 'integrated_tool_panel.xml' ), self.root )
@@ -1662,10 +1662,13 @@ class JobWrapper( object ):
                                                                         **kwds )
        if resolve_metadata_dependencies:
            metadata_tool = self.app.toolbox.get_tool("__SET_METADATA__")
-            dependency_shell_commands = metadata_tool.build_dependency_shell_commands(job_directory=self.working_directory)
-            if dependency_shell_commands:
-                dependency_shell_commands = "; ".join(dependency_shell_commands)
-                command = "%s; %s" % (dependency_shell_commands, command)
+            if metadata_tool is not None:
+                # Due to tool shed hacks for migrate and installed tool tests...
+                # see (``setup_shed_tools_for_test`` in test/base/driver_util.py).
+                dependency_shell_commands = metadata_tool.build_dependency_shell_commands(job_directory=self.working_directory)
+                if dependency_shell_commands:
+                    dependency_shell_commands = "; ".join(dependency_shell_commands)
+                    command = "%s; %s" % (dependency_shell_commands, command)
        return command

    @property
@@ -66,7 +66,6 @@ class Configuration( object ):
        self.new_file_path = resolve_path( kwargs.get( "new_file_path", "database/tmp" ), self.root )
        self.cookie_path = kwargs.get( "cookie_path", "/" )
        self.enable_quotas = string_as_bool( kwargs.get( 'enable_quotas', False ) )
-        self.test_conf = resolve_path( kwargs.get( "test_conf", "" ), self.root )
        self.id_secret = kwargs.get( "id_secret", "USING THE DEFAULT IS NOT SECURE!" )
        # Tool stuff
        self.tool_path = resolve_path( kwargs.get( "tool_path", "tools" ), self.root )
@@ -35,16 +35,117 @@ Run a specific API test:
    ./run_tests.sh -api test/api/test_tools.py:ToolsTestCase.test_map_over_with_output_format_actions


+External Tests:
+
+A small subset of tests can be run against an existing Galxy
+instance. The external Galaxy instance URL can be configured with
+--external_url. If this is set, either --external_master_key or
+--external_user_key must be set as well - more tests can be executed
+with --external_master_key than with a user key.
+
 Extra options:
+
 --verbose_errors      Force some tests produce more verbose error reporting.
- --no_cleanup          Do not delete temp files for Python functional tests (-toolshed, -framework, etc...)
- --debug               On python test error or failure invoke a pdb shell for interactive debugging of the test
- --report_file         Path of HTML report to produce (for Python Galaxy functional tests).
- --xunit_report_file   Path of XUnit report to produce (for Python Galaxy functional tests).
- --skip-venv           Do not create .venv (passes this flag to common_startup.sh)
- --dockerize           Run tests in a pre-configured Docker container (must be first argument if present).
- --db <type>           For use with --dockerize, run tests using partially migrated 'postgres', 'mysql',
-                       or 'sqlite' databases.
+ --no_cleanup          Do not delete temp files for Python functional tests
+                       (-toolshed, -framework, etc...)
+ --debug               On python test error or failure invoke a pdb shell for
+                       interactive debugging of the test
+ --report_file         Path of HTML report to produce (for Python Galaxy
+                       functional tests).
+ --xunit_report_file   Path of XUnit report to produce (for Python Galaxy
+                       functional tests).
+ --skip-venv           Do not create .venv (passes this flag to
+                       common_startup.sh)
+ --dockerize           Run tests in a pre-configured Docker container (must be
+                       first argument if present).
+ --db <type>           For use with --dockerize, run tests using partially
+                       migrated 'postgres', 'mysql', or 'sqlite' databases.
+ --external_url        External URL to use for Galaxy testing (only certain
+                       tests).
+ --external_master_key Master API key used to configure external tests.
+ --external_user_key   User API used for external tests - not required if
+                       external_master_key is specified.
+
+Environment Variables:
+
+In addition to the above command-line options, many environment variables
+can be used to control the Galaxy functional testing processing. Command-line
+options above like (--external_url) will set environment variables - in such
+cases the command line argument takes precedent over environment variables set
+at the time of running this script.
+
+Functional Test Environment Variables
+
+GALAXY_TEST_DBURI               Database connection string used for functional
+                                test database for Galaxy.
+GALAXY_TEST_INSTALL_DBURI       Database connection string used for functional
+                                test database for Galaxy's install framework.
+GALAXY_TEST_INSTALL_DB_MERGED   Set to use same database for Galaxy and install
+                                framework, this defaults to True for Galaxy
+                                tests an False for shed tests.
+GALAXY_TEST_DB_TEMPLATE         If GALAXY_TEST_DBURI is unset, this URL can be
+                                retrieved and should be an sqlite database that
+                                will be upgraded and tested against.
+GALAXY_TEST_TMP_DIR             Temp directory used for files required by
+                                Galaxy server setup for Galaxy functional tests.
+GALAXY_TEST_SAVE                Location to save certain test files (such as
+                                tool outputs).
+GALAXY_TEST_EXTERNAL            Target an external Galaxy as part of testing.
+GALAXY_CONFIG_MASTER_KEY        Master or admin API key to use as part of
+                                testing with GALAXY_TEST_EXTERNAL.
+GALAXY_TEST_USER_API_KEY        User API key to use as part of testing with
+                                GALAXY_TEST_EXTERNAL.
+GALAXY_TEST_HISTORY_ID          Point casperjs tests at specific external
+                                history for testing.
+GALAXY_TEST_WORKFLOW_FILE       Point casperjs tests at specific workflow
+                                file for testing.
+GALAXY_TEST_VERBOSE_ERRORS      Enable more verbose errors during API tests.
+GALAXY_TEST_UPLOAD_ASYNC        Upload tool test inputs asynchronously (may
+                                overwhelm sqlite database).
+GALAXY_TEST_RAW_DIFF            Don't slice up tool test diffs to keep output
+                                managable - print all output. (default off)
+GALAXY_TEST_DEFAULT_WAIT        Max time allowed for a tool test before Galaxy
+                                gives up (default 86400) - tools may define a
+                                maxseconds attribute to extend this.
+GALAXY_TEST_TOOL_DEPENDENCY_DIR tool dependency dir to use for Galaxy during
+                                functional tests.
+GALAXY_TEST_FILE_DIR            Test data sources (default to
+              test-data,https://github.com/galaxyproject/galaxy-test-data.git)
+GALAXY_TEST_DIRECTORY           $GALAXY_ROOT/test
+GALAXY_TEST_TOOL_DATA_PATH      Set to override tool data path during tool
+                                shed tests.
+GALAXY_TEST_FETCH_DATA          Fetch remote test data to
+                                GALAXY_TEST_DATA_REPO_CACHE as part of tool
+                                tests if it is not available locally (default
+                                to True). Requires git to be available on the
+                                command-line.
+GALAXY_TEST_DATA_REPO_CACHE     Where to cache remote test data to (default to
+                                test-data-cache).
+HTTP_ACCEPT_LANGUAGE            Defaults to 'en'
+GALAXY_TEST_NO_CLEANUP          Do not cleanup main test directory after tests,
+                                the deprecated option TOOL_SHED_TEST_NO_CLEANUP
+                                does the same thing.
+GALAXY_TEST_HOST                Host to use for Galaxy server setup for
+                                testing.
+GALAXY_TEST_PORT                Port to use for Galaxy server setup for
+                                testing.
+GALAXY_TEST_TOOL_PATH           Path defaulting to 'tools'.
+GALAXY_TEST_SHED_TOOL_CONF      Shed toolbox conf (defaults to
+                                config/shed_tool_conf.xml) used when testing
+                                installed to tools with -installed.
+TOOL_SHED_TEST_HOST             Host to use for shed server setup for testing.
+TOOL_SHED_TEST_PORT             Port to use for shed server setup for testing.
+TOOL_SHED_TEST_FILE_DIR         Defaults to test/tool_shed/test_data.
+TOOL_SHED_TEST_TMP_DIR          Defaults to random /tmp directory - place for
+                                tool shed test server files to be placed.
+TOOL_SHED_TEST_OMIT_GALAXY      Do not launch a Galaxy server for tool shed
+                                testing.
+
+Unit Test Environment Variables
+
+GALAXY_TEST_INCLUDE_SLOW - Used in unit tests to trigger slower tests that
+                           aren't included by default with --unit/-u.
+
 EOF
 }

@@ -160,6 +261,18 @@ do
          with_framework_test_tools_arg="-with_framework_test_tools"
          shift
          ;;
+      --external_url)
+          GALAXY_TEST_EXTERNAL=$2
+          shift 2
+          ;;
+      --external_master_key)
+          GALAXY_CONFIG_MASTER_KEY=$2
+          shift 2
+          ;;
+      --external_user_key)
+          GALAXY_TEST_USER_API_KEY=$2
+          shift 2
+          ;;
      -w|-workflow|--workflow)
          if [ $# -gt 1 ]; then
              workflow_file=$2
@@ -349,7 +462,8 @@ elif [ -n "$data_managers_test" ] ; then
    [ -n "$test_id" ] && class=":TestForDataManagerTool_$test_id" || class=""
    extra_args="functional.test_data_managers$class -data_managers"
 elif [ -n "$workflow_test" ]; then
-    extra_args="functional.workflow:WorkflowTestCase $workflow_file"
+    GALAXY_TEST_WORKFLOW_FILE="$workflow_file"
+    extra_args="functional.workflow:WorkflowTestCase"
 elif [ -n "$toolshed_script" ]; then
    extra_args="$toolshed_script"
 elif [ -n "$api_script" ]; then
@@ -387,7 +501,11 @@ if [ "$driver" = "python" ]; then
    else
        structured_data_args=""
    fi
-    python $test_script $coverage_arg -v --with-nosehtml --html-report-file $report_file $xunit_args $structured_data_args $with_framework_test_tools_arg $extra_args
+    if [ -n "$with_framework_test_tools_arg" ]; then
+        GALAXY_TEST_TOOL_CONF="config/tool_conf.xml.sample,test/functional/tools/samples_tool_conf.xml"
+        export GALAXY_TEST_TOOL_CONF
+    fi
+    python $test_script $coverage_arg -v --with-nosehtml --html-report-file $report_file $xunit_args $structured_data_args $extra_args
 else
    ensure_grunt
    if [ -n "$watch" ]; then
@@ -1,531 +1,121 @@
 #!/usr/bin/env python
-import httplib
-import logging
+"""Test driver for many Galaxy Python functional tests.
+
+Launch this script by running ``run_tests.sh`` from GALAXY_ROOT, see
+that script for a list of options.
+"""
+
 import os
 import os.path
-import random
-import re
-import shutil
-import socket
 import sys
-import tempfile
-import threading
-import time
-import urllib
-from ConfigParser import SafeConfigParser
-from json import dumps
-
-import nose.core
-import nose.config
-import nose.loader
-import nose.plugins.manager
-from paste import httpserver

 galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))
 sys.path[1:1] = [ os.path.join( galaxy_root, "lib" ), os.path.join( galaxy_root, "test" ) ]

+from base import driver_util
+log = driver_util.build_logger()
+
 from base.api_util import get_master_api_key, get_user_api_key
-from base.nose_util import run
-from base.instrument import StructuredTestDataPlugin
-from base.test_logging import logging_config_file
-from base.tool_shed_util import parse_tool_panel_config
-from functional import database_contexts
-from galaxy import tools
-from galaxy.app import UniverseApplication
-from galaxy.util.properties import load_app_properties
-from galaxy.web import buildapp
-
-log = logging.getLogger( "functional_tests.py" )
-
-default_galaxy_test_host = "localhost"
-default_galaxy_test_port_min = 8000
-default_galaxy_test_port_max = 9999
-default_galaxy_locales = 'en'
-default_galaxy_test_file_dir = "test-data,https://github.com/galaxyproject/galaxy-test-data.git"
-migrated_tool_panel_config = 'config/migrated_tools_conf.xml'
-installed_tool_panel_configs = [
-    os.environ.get('GALAXY_TEST_SHED_TOOL_CONF', 'config/shed_tool_conf.xml')
-]


-# should this serve static resources (scripts, images, styles, etc.)
-STATIC_ENABLED = True
+class MigratedToolsGalaxyTestDriver(driver_util.GalaxyTestDriver):
+    """Instantiate a Galaxy-style nose TestDriver for testing migrated Galaxy tools."""

-# Set up a job_conf.xml that explicitly limits jobs to 10 minutes.
-job_conf_xml = '''<?xml version="1.0"?>
-<!-- A test job config that explicitly configures job running the way it is configured by default (if there is no explicit config). -->
-<job_conf>
-    <plugins>
-        <plugin id="local" type="runner" load="galaxy.jobs.runners.local:LocalJobRunner" workers="4"/>
-    </plugins>
-    <handlers>
-        <handler id="main"/>
-    </handlers>
-    <destinations>
-        <destination id="local" runner="local"/>
-    </destinations>
-    <limits>
-        <limit type="walltime">00:10:00</limit>
-    </limits>
-</job_conf>
-'''
+    testing_shed_tools = True

-
-def get_static_settings():
-    """Returns dictionary of the settings necessary for a galaxy App
-    to be wrapped in the static middleware.
-
-    This mainly consists of the filesystem locations of url-mapped
-    static resources.
-    """
-    cwd = os.getcwd()
-    static_dir = os.path.join( cwd, 'static' )
-    # TODO: these should be copied from config/galaxy.ini
-    return dict(
-        # TODO: static_enabled needed here?
-        static_enabled=True,
-        static_cache_time=360,
-        static_dir=static_dir,
-        static_images_dir=os.path.join( static_dir, 'images', '' ),
-        static_favicon_dir=os.path.join( static_dir, 'favicon.ico' ),
-        static_scripts_dir=os.path.join( static_dir, 'scripts', '' ),
-        static_style_dir=os.path.join( static_dir, 'june_2007_style', 'blue' ),
-        static_robots_txt=os.path.join( static_dir, 'robots.txt' ),
-    )
-
-
-def get_webapp_global_conf():
-    """Get the global_conf dictionary sent as the first argument to app_factory.
-    """
-    # (was originally sent 'dict()') - nothing here for now except static settings
-    global_conf = dict()
-    if STATIC_ENABLED:
-        global_conf.update( get_static_settings() )
-    return global_conf
-
-
-def generate_config_file( input_filename, output_filename, config_items ):
-    '''
-    Generate a config file with the configuration that has been defined for the embedded web application.
-    This is mostly relevant when setting metadata externally, since the script for doing that does not
-    have access to app.config.
-    '''
-    cp = SafeConfigParser()
-    cp.read( input_filename )
-    config_items_by_section = []
-    for label, value in config_items:
-        found = False
-        # Attempt to determine the correct section for this configuration option.
-        for section in cp.sections():
-            if cp.has_option( section, label ):
-                config_tuple = section, label, value
-                config_items_by_section.append( config_tuple )
-                found = True
-                continue
-        # Default to app:main if no section was found.
-        if not found:
-            config_tuple = 'app:main', label, value
-            config_items_by_section.append( config_tuple )
-    print( config_items_by_section )
-
-    # Replace the default values with the provided configuration.
-    for section, label, value in config_items_by_section:
-        if cp.has_option( section, label ):
-            cp.remove_option( section, label )
-        cp.set( section, label, str( value ) )
-    fh = open( output_filename, 'w' )
-    cp.write( fh )
-    fh.close()
-
-
-def run_tests( test_config ):
-    return run( test_config )
-
-
-def __copy_database_template( source, db_path ):
-    """
-    Copy a 'clean' sqlite template database (from file or URL) to specified
-    database path.
-    """
-    os.makedirs( os.path.dirname( db_path ) )
-    if os.path.exists( source ):
-        shutil.copy( source, db_path )
-        assert os.path.exists( db_path )
-    elif source.lower().startswith( ( "http://", "https://", "ftp://" ) ):
-        urllib.urlretrieve( source, db_path )
-    else:
-        raise Exception( "Failed to copy database template from source %s" % source )
-
-
-def main():
-    # ---- Configuration ------------------------------------------------------
-    galaxy_test_host = os.environ.get( 'GALAXY_TEST_HOST', default_galaxy_test_host )
-    galaxy_test_port = os.environ.get( 'GALAXY_TEST_PORT', None )
-    galaxy_test_save = os.environ.get( 'GALAXY_TEST_SAVE', None)
-    tool_path = os.environ.get( 'GALAXY_TEST_TOOL_PATH', 'tools' )
-    if 'HTTP_ACCEPT_LANGUAGE' not in os.environ:
-        os.environ[ 'HTTP_ACCEPT_LANGUAGE' ] = default_galaxy_locales
-    testing_migrated_tools = __check_arg( '-migrated' )
-    testing_installed_tools = __check_arg( '-installed' )
-    datatypes_conf_override = None
-
-    if testing_migrated_tools or testing_installed_tools:
-        # Store a jsonified dictionary of tool_id : GALAXY_TEST_FILE_DIR pairs.
-        galaxy_tool_shed_test_file = 'shed_tools_dict'
-        # We need the upload tool for functional tests, so we'll create a temporary tool panel config that defines it.
-        fd, tmp_tool_panel_conf = tempfile.mkstemp()
-        os.write( fd, '<?xml version="1.0"?>\n' )
-        os.write( fd, '<toolbox>\n' )
-        os.write( fd, '<tool file="data_source/upload.xml"/>\n' )
-        os.write( fd, '</toolbox>\n' )
-        os.close( fd )
-        tool_config_file = tmp_tool_panel_conf
-        galaxy_test_file_dir = None
-        library_import_dir = None
-        user_library_import_dir = None
-        # Exclude all files except test_toolbox.py.
-        ignore_files = ( re.compile( r'^test_[adghlmsu]*' ), re.compile( r'^test_ta*' ) )
-    else:
-        framework_tool_dir = os.path.join('test', 'functional', 'tools')
-        framework_test = __check_arg( '-framework' )  # Run through suite of tests testing framework.
-        if framework_test:
-            tool_conf = os.path.join( framework_tool_dir, 'samples_tool_conf.xml' )
-            datatypes_conf_override = os.path.join( framework_tool_dir, 'sample_datatypes_conf.xml' )
-        else:
-            # Use tool_conf.xml toolbox.
-            tool_conf = None
-            if __check_arg( '-with_framework_test_tools' ):
-                tool_conf = "%s,%s" % ( 'config/tool_conf.xml.sample', os.path.join( framework_tool_dir, 'samples_tool_conf.xml' ) )
-        test_dir = default_galaxy_test_file_dir
-        tool_config_file = os.environ.get( 'GALAXY_TEST_TOOL_CONF', tool_conf )
-        galaxy_test_file_dir = os.environ.get( 'GALAXY_TEST_FILE_DIR', test_dir )
-        first_test_file_dir = galaxy_test_file_dir.split(",")[0]
-        if not os.path.isabs( first_test_file_dir ):
-            first_test_file_dir = os.path.join( os.getcwd(), first_test_file_dir )
-        library_import_dir = first_test_file_dir
-        import_dir = os.path.join( first_test_file_dir, 'users' )
-        if os.path.exists(import_dir):
-            user_library_import_dir = import_dir
-        else:
-            user_library_import_dir = None
-        ignore_files = ()
-
-    start_server = 'GALAXY_TEST_EXTERNAL' not in os.environ
-    tool_data_table_config_path = None
-    if os.path.exists( 'tool_data_table_conf.test.xml' ):
-        # If explicitly defined tables for test, use those.
-        tool_data_table_config_path = 'tool_data_table_conf.test.xml'
-    else:
-        # ... otherise find whatever Galaxy would use as the default and
-        # the sample data for fucntional tests to that.
-        default_tool_data_config = 'config/tool_data_table_conf.xml.sample'
-        for tool_data_config in ['config/tool_data_table_conf.xml', 'tool_data_table_conf.xml' ]:
-            if os.path.exists( tool_data_config ):
-                default_tool_data_config = tool_data_config
-        tool_data_table_config_path = '%s,test/functional/tool-data/sample_tool_data_tables.xml' % default_tool_data_config
-
-    default_data_manager_config = 'config/data_manager_conf.xml.sample'
-    for data_manager_config in ['config/data_manager_conf.xml', 'data_manager_conf.xml' ]:
-        if os.path.exists( data_manager_config ):
-            default_data_manager_config = data_manager_config
-    data_manager_config_file = "%s,test/functional/tools/sample_data_manager_conf.xml" % default_data_manager_config
-    shed_tool_data_table_config = 'config/shed_tool_data_table_conf.xml'
-    tool_dependency_dir = os.environ.get( 'GALAXY_TOOL_DEPENDENCY_DIR', None )
-    use_distributed_object_store = os.environ.get( 'GALAXY_USE_DISTRIBUTED_OBJECT_STORE', False )
-    galaxy_test_tmp_dir = os.environ.get( 'GALAXY_TEST_TMP_DIR', None )
-    if galaxy_test_tmp_dir is None:
-        galaxy_test_tmp_dir = tempfile.mkdtemp()
-
-    galaxy_job_conf_file = os.environ.get( 'GALAXY_TEST_JOB_CONF',
-                                           os.path.join( galaxy_test_tmp_dir, 'test_job_conf.xml' ) )
-    # Generate the job_conf.xml file.
-    file( galaxy_job_conf_file, 'w' ).write( job_conf_xml )
-
-    database_auto_migrate = False
-
-    galaxy_test_proxy_port = None
-    if start_server:
-        tempdir = tempfile.mkdtemp( dir=galaxy_test_tmp_dir )
-        # Configure the database path.
-        if 'GALAXY_TEST_DBPATH' in os.environ:
-            galaxy_db_path = os.environ[ 'GALAXY_TEST_DBPATH' ]
-        else:
-            galaxy_db_path = os.path.join( tempdir, 'database' )
-        # Configure the paths Galaxy needs to  test tools.
-        file_path = os.path.join( galaxy_db_path, 'files' )
-        template_cache_path = os.path.join( galaxy_db_path, 'compiled_templates' )
-        new_file_path = tempfile.mkdtemp( prefix='new_files_path_', dir=tempdir )
-        job_working_directory = tempfile.mkdtemp( prefix='job_working_directory_', dir=tempdir )
-        install_database_connection = os.environ.get( 'GALAXY_TEST_INSTALL_DBURI', None )
-        if 'GALAXY_TEST_DBURI' in os.environ:
-            database_connection = os.environ['GALAXY_TEST_DBURI']
-        else:
-            db_path = os.path.join( galaxy_db_path, 'universe.sqlite' )
-            if 'GALAXY_TEST_DB_TEMPLATE' in os.environ:
-                # Middle ground between recreating a completely new
-                # database and pointing at existing database with
-                # GALAXY_TEST_DBURI. The former requires a lot of setup
-                # time, the latter results in test failures in certain
-                # cases (namely tool shed tests expecting clean database).
-                log.debug( "Copying database template from %s.", os.environ['GALAXY_TEST_DB_TEMPLATE'] )
-                __copy_database_template(os.environ['GALAXY_TEST_DB_TEMPLATE'], db_path)
-                database_auto_migrate = True
-            database_connection = 'sqlite:///%s' % db_path
-        kwargs = {}
-        for dir in file_path, new_file_path, template_cache_path:
-            try:
-                if not os.path.exists( dir ):
-                    os.makedirs( dir )
-            except OSError:
-                pass
-
-    # Data Manager testing temp path
-    # For storing Data Manager outputs and .loc files so that real ones don't get clobbered
-    data_manager_test_tmp_path = tempfile.mkdtemp( prefix='data_manager_test_tmp', dir=galaxy_test_tmp_dir )
-    galaxy_data_manager_data_path = tempfile.mkdtemp( prefix='data_manager_tool-data', dir=data_manager_test_tmp_path )
-
-    # ---- Build Application --------------------------------------------------
-    master_api_key = get_master_api_key()
-    app = None
-    if start_server:
-        kwargs = dict( admin_users='test@bx.psu.edu',
-                       api_allow_run_as='test@bx.psu.edu',
-                       allow_library_path_paste=True,
-                       allow_user_creation=True,
-                       allow_user_deletion=True,
-                       database_connection=database_connection,
-                       database_auto_migrate=database_auto_migrate,
-                       datatype_converters_config_file="datatype_converters_conf.xml.sample",
-                       file_path=file_path,
-                       id_secret='changethisinproductiontoo',
-                       job_queue_workers=5,
-                       job_working_directory=job_working_directory,
-                       library_import_dir=library_import_dir,
-                       log_destination="stdout",
-                       new_file_path=new_file_path,
-                       template_cache_path=template_cache_path,
-                       running_functional_tests=True,
-                       shed_tool_data_table_config=shed_tool_data_table_config,
-                       template_path="templates",
-                       test_conf="test.conf",
-                       tool_config_file=tool_config_file,
-                       tool_data_table_config_path=tool_data_table_config_path,
-                       tool_path=tool_path,
-                       galaxy_data_manager_data_path=galaxy_data_manager_data_path,
-                       tool_parse_help=False,
-                       update_integrated_tool_panel=False,
-                       use_heartbeat=False,
-                       user_library_import_dir=user_library_import_dir,
-                       master_api_key=master_api_key,
-                       use_tasked_jobs=True,
-                       check_migrate_tools=False,
-                       cleanup_job='onsuccess',
-                       enable_beta_tool_formats=True,
-                       auto_configure_logging=logging_config_file is None,
-                       data_manager_config_file=data_manager_config_file )
-        if install_database_connection is not None:
-            kwargs[ 'install_database_connection' ] = install_database_connection
-        if not database_connection.startswith( 'sqlite://' ):
-            kwargs[ 'database_engine_option_max_overflow' ] = '20'
-            kwargs[ 'database_engine_option_pool_size' ] = '10'
-        if tool_dependency_dir is not None:
-            kwargs[ 'tool_dependency_dir' ] = tool_dependency_dir
-        if use_distributed_object_store:
-            kwargs[ 'object_store' ] = 'distributed'
-            kwargs[ 'distributed_object_store_config_file' ] = 'distributed_object_store_conf.xml.sample'
-        if datatypes_conf_override:
-            kwargs[ 'datatypes_config_file' ] = datatypes_conf_override
-        # If the user has passed in a path for the .ini file, do not overwrite it.
-        galaxy_config_file = os.environ.get( 'GALAXY_TEST_INI_FILE', None )
-        if not galaxy_config_file:
-            galaxy_config_file = os.path.join( galaxy_test_tmp_dir, 'functional_tests_wsgi.ini' )
-            config_items = []
-            for label in kwargs:
-                config_tuple = label, kwargs[ label ]
-                config_items.append( config_tuple )
-            # Write a temporary file, based on config/galaxy.ini.sample, using the configuration options defined above.
-            generate_config_file( 'config/galaxy.ini.sample', galaxy_config_file, config_items )
-        # Set the global_conf[ '__file__' ] option to the location of the temporary .ini file, which gets passed to set_metadata.sh.
-        kwargs[ 'global_conf' ] = get_webapp_global_conf()
-        kwargs[ 'global_conf' ][ '__file__' ] = galaxy_config_file
-        kwargs[ 'config_file' ] = galaxy_config_file
-        kwargs = load_app_properties(
-            kwds=kwargs
+    def build_tests(self):
+        """Build migrated tool test methods."""
+        self.setup_shed_tools(
+            testing_migrated_tools=True,
        )
-        # Build the Universe Application
-        app = UniverseApplication( **kwargs )
-        database_contexts.galaxy_context = app.model.context
-        log.info( "Embedded Universe application started" )
-
-    # ---- Run webserver ------------------------------------------------------
-    server = None
-
-    if start_server:
-        webapp = buildapp.app_factory( kwargs[ 'global_conf' ], app=app,
-            use_translogger=False, static_enabled=STATIC_ENABLED )
-        if galaxy_test_port is not None:
-            server = httpserver.serve( webapp, host=galaxy_test_host, port=galaxy_test_port, start_loop=False )
-        else:
-            random.seed()
-            for i in range( 0, 9 ):
-                try:
-                    galaxy_test_port = str( random.randint( default_galaxy_test_port_min, default_galaxy_test_port_max ) )
-                    log.debug( "Attempting to serve app on randomly chosen port: %s" % galaxy_test_port )
-                    server = httpserver.serve( webapp, host=galaxy_test_host, port=galaxy_test_port, start_loop=False )
-                    break
-                except socket.error, e:
-                    if e[0] == 98:
-                        continue
-                    raise
-            else:
-                raise Exception( "Unable to open a port between %s and %s to start Galaxy server" % ( default_galaxy_test_port_min, default_galaxy_test_port_max ) )
-        if galaxy_test_proxy_port:
-            os.environ['GALAXY_TEST_PORT'] = galaxy_test_proxy_port
-        else:
-            os.environ['GALAXY_TEST_PORT'] = galaxy_test_port
-
-        t = threading.Thread( target=server.serve_forever )
-        t.start()
-        # Test if the server is up
-        for i in range( 10 ):
-            conn = httplib.HTTPConnection( galaxy_test_host, galaxy_test_port )  # directly test the app, not the proxy
-            conn.request( "GET", "/" )
-            if conn.getresponse().status == 200:
-                break
-            time.sleep( 0.1 )
-        else:
-            raise Exception( "Test HTTP server did not return '200 OK' after 10 tries" )
-        log.info( "Embedded web server started" )
-
-    # ---- Find tests ---------------------------------------------------------
-    if galaxy_test_proxy_port:
-        log.info( "Functional tests will be run against %s:%s" % ( galaxy_test_host, galaxy_test_proxy_port ) )
-    else:
-        log.info( "Functional tests will be run against %s:%s" % ( galaxy_test_host, galaxy_test_port ) )
-    success = False
-    try:
-        tool_configs = app.config.tool_configs
-        # What requires these? Handy for (eg) functional tests to save outputs?
-        if galaxy_test_save:
-            os.environ[ 'GALAXY_TEST_SAVE' ] = galaxy_test_save
-        # Pass in through script setenv, will leave a copy of ALL test validate files
-        os.environ[ 'GALAXY_TEST_HOST' ] = galaxy_test_host
-
-        def _run_functional_test( testing_shed_tools=None ):
-            workflow_test = __check_arg( '-workflow', param=True )
-            if workflow_test:
-                import functional.workflow
-                functional.workflow.WorkflowTestCase.workflow_test_file = workflow_test
-                functional.workflow.WorkflowTestCase.master_api_key = master_api_key
-                functional.workflow.WorkflowTestCase.user_api_key = get_user_api_key()
-            data_manager_test = __check_arg( '-data_managers', param=False )
-            if data_manager_test:
-                import functional.test_data_managers
-                functional.test_data_managers.data_managers = app.data_managers  # seems like a hack...
-                functional.test_data_managers.build_tests(
-                    tmp_dir=data_manager_test_tmp_path,
-                    testing_shed_tools=testing_shed_tools,
-                    master_api_key=master_api_key,
-                    user_api_key=get_user_api_key(),
-                )
-            else:
-                # We must make sure that functional.test_toolbox is always imported after
-                # database_contexts.galaxy_content is set (which occurs in this method above).
-                # If functional.test_toolbox is imported before database_contexts.galaxy_content
-                # is set, sa_session will be None in all methods that use it.
-                import functional.test_toolbox
-                functional.test_toolbox.toolbox = app.toolbox
-                # When testing data managers, do not test toolbox.
-                functional.test_toolbox.build_tests(
-                    app=app,
-                    testing_shed_tools=testing_shed_tools,
-                    master_api_key=master_api_key,
-                    user_api_key=get_user_api_key(),
-                )
-            test_config = nose.config.Config( env=os.environ, ignoreFiles=ignore_files, plugins=nose.plugins.manager.DefaultPluginManager() )
-            test_config.plugins.addPlugin( StructuredTestDataPlugin() )
-            test_config.configure( sys.argv )
-            result = run_tests( test_config )
-            success = result.wasSuccessful()
-            return success
-
-        if testing_migrated_tools or testing_installed_tools:
-            shed_tools_dict = {}
-            if testing_migrated_tools:
-                has_test_data, shed_tools_dict = parse_tool_panel_config( migrated_tool_panel_config, shed_tools_dict )
-            elif testing_installed_tools:
-                for shed_tool_config in installed_tool_panel_configs:
-                    has_test_data, shed_tools_dict = parse_tool_panel_config( shed_tool_config, shed_tools_dict )
-            # Persist the shed_tools_dict to the galaxy_tool_shed_test_file.
-            shed_tools_file = open( galaxy_tool_shed_test_file, 'w' )
-            shed_tools_file.write( dumps( shed_tools_dict ) )
-            shed_tools_file.close()
-            if not os.path.isabs( galaxy_tool_shed_test_file ):
-                galaxy_tool_shed_test_file = os.path.join( os.getcwd(), galaxy_tool_shed_test_file )
-            os.environ[ 'GALAXY_TOOL_SHED_TEST_FILE' ] = galaxy_tool_shed_test_file
-            if testing_installed_tools:
-                # Eliminate the migrated_tool_panel_config from the app's tool_configs, append the list of installed_tool_panel_configs,
-                # and reload the app's toolbox.
-                relative_migrated_tool_panel_config = os.path.join( app.config.root, migrated_tool_panel_config )
-                if relative_migrated_tool_panel_config in tool_configs:
-                    tool_configs.remove( relative_migrated_tool_panel_config )
-                for installed_tool_panel_config in installed_tool_panel_configs:
-                    tool_configs.append( installed_tool_panel_config )
-                app.toolbox = tools.ToolBox( tool_configs, app.config.tool_path, app )
-            success = _run_functional_test( testing_shed_tools=True )
-            try:
-                os.unlink( tmp_tool_panel_conf )
-            except:
-                log.info( "Unable to remove temporary file: %s" % tmp_tool_panel_conf )
-            try:
-                os.unlink( galaxy_tool_shed_test_file )
-            except:
-                log.info( "Unable to remove file: %s" % galaxy_tool_shed_test_file )
-        else:
-            if galaxy_test_file_dir:
-                os.environ[ 'GALAXY_TEST_FILE_DIR' ] = galaxy_test_file_dir
-            success = _run_functional_test( )
-    except:
-        log.exception( "Failure running tests" )
-
-    log.info( "Shutting down" )
-    # ---- Tear down -----------------------------------------------------------
-    if server:
-        log.info( "Shutting down embedded web server" )
-        server.server_close()
-        server = None
-        log.info( "Embedded web server stopped" )
-    if app:
-        log.info( "Shutting down app" )
-        app.shutdown()
-        app = None
-        log.info( "Embedded Universe application stopped" )
-    try:
-        if os.path.exists( tempdir ) and 'GALAXY_TEST_NO_CLEANUP' not in os.environ:
-            log.info( "Cleaning up temporary files in %s" % tempdir )
-            shutil.rmtree( tempdir )
-        else:
-            log.info( "GALAXY_TEST_NO_CLEANUP is on. Temporary files in %s" % tempdir )
-    except:
-        pass
-    if success:
-        return 0
-    else:
-        return 1
+        self.build_tool_tests()


-def __check_arg( name, param=False ):
+class InstalledToolsGalaxyTestDriver(driver_util.GalaxyTestDriver):
+    """Galaxy-style nose TestDriver for testing installed Galaxy tools."""
+
+    testing_shed_tools = True
+
+    def build_tests(self):
+        """Build installed tool test methods."""
+        self.setup_shed_tools(
+            testing_installed_tools=True,
+        )
+        self.build_tool_tests()
+
+
+class DefaultGalaxyTestDriver(driver_util.GalaxyTestDriver):
+    """Default Galaxy-style nose test driver.
+
+    Just populate non-shed tool tests and run tests. Works
+    for tool tests, regular twill tests, and API testing.
+    """
+
+    def build_tests(self):
+        """Build framework tool test methods."""
+        self.build_tool_tests()
+
+
+class FrameworkToolsGalaxyTestDriver(DefaultGalaxyTestDriver):
+    """Galaxy-style nose TestDriver for testing framework Galaxy tools."""
+
+    default_tool_conf = driver_util.FRAMEWORK_SAMPLE_TOOLS_CONF
+    datatypes_conf_override = driver_util.FRAMEWORK_DATATYPES_CONF
+
+
+class DataManagersGalaxyTestDriver(driver_util.GalaxyTestDriver):
+    """Galaxy-style nose TestDriver for testing framework Galaxy tools."""
+
+    def build_tests(self):
+        """Build data manager test methods."""
+        import functional.test_data_managers
+        functional.test_data_managers.data_managers = self.app.data_managers
+        functional.test_data_managers.build_tests(
+            tmp_dir=self.galaxy_test_tmp_dir,
+            testing_shed_tools=self.testing_shed_tools,
+            master_api_key=get_master_api_key(),
+            user_api_key=get_user_api_key(),
+        )
+
+
+class WorkflowGalaxyTestDriver(driver_util.GalaxyTestDriver):
+    """Galaxy-style nose TestDriver for testing a Galaxy workflow."""
+
+    def build_tests(self):
+        """Setup WorkflowTestCase for test execution."""
+        import functional.workflow
+        functional.workflow.WorkflowTestCase.master_api_key = get_master_api_key()
+        functional.workflow.WorkflowTestCase.user_api_key = get_user_api_key()
+
+
+TEST_DRIVERS = {
+    '-migrated': MigratedToolsGalaxyTestDriver,
+    '-installed': InstalledToolsGalaxyTestDriver,
+    '-framework': FrameworkToolsGalaxyTestDriver,
+    '-data_managers': DataManagersGalaxyTestDriver,
+    '-workflow': WorkflowGalaxyTestDriver,
+}
+
+
+def find_test_driver():
+    """Look at command-line args and find the correct Galaxy test driver."""
+    test_driver = DefaultGalaxyTestDriver
+
+    for key in TEST_DRIVERS.keys():
+        if _check_arg(key):
+            test_driver = TEST_DRIVERS[key]
+
+    return test_driver
+
+
+def _check_arg( name ):
    try:
        index = sys.argv.index( name )
        del sys.argv[ index ]
-        if param:
-            ret_val = sys.argv[ index ]
-            del sys.argv[ index ]
-        else:
-            ret_val = True
+        ret_val = True
    except ValueError:
        ret_val = False
    return ret_val

 if __name__ == "__main__":
-    sys.exit( main() )
+    driver_util.drive_test(find_test_driver())
@@ -0,0 +1,663 @@
+"""Scripts for drivers of Galaxy functional tests."""
+
+import collections
+import httplib
+import json
+import logging
+import os
+import random
+import shutil
+import socket
+import sys
+import tempfile
+import threading
+import time
+
+from six.moves.urllib.request import urlretrieve
+
+import nose.config
+import nose.core
+import nose.loader
+import nose.plugins.manager
+
+from paste import httpserver
+
+from .api_util import get_master_api_key, get_user_api_key
+from .tool_shed_util import parse_tool_panel_config
+from .nose_util import run
+from .instrument import StructuredTestDataPlugin
+
+from functional import database_contexts
+
+from galaxy.app import UniverseApplication as GalaxyUniverseApplication
+from galaxy.web import buildapp
+from galaxy.webapps.tool_shed.app import UniverseApplication as ToolshedUniverseApplication
+from galaxy.util import asbool
+from galaxy.util.properties import load_app_properties
+
+from base.test_logging import logging_config_file
+
+galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir))
+DEFAULT_WEB_HOST = "localhost"
+GALAXY_TEST_DIRECTORY = os.path.join(galaxy_root, "test")
+GALAXY_TEST_FILE_DIR = "test-data,https://github.com/galaxyproject/galaxy-test-data.git"
+TOOL_SHED_TEST_DATA = os.path.join(GALAXY_TEST_DIRECTORY, "tool_shed", "test_data")
+FRAMEWORK_TOOLS_DIR = os.path.join(GALAXY_TEST_DIRECTORY, "functional", "tools")
+FRAMEWORK_UPLOAD_TOOL_CONF = os.path.join(FRAMEWORK_TOOLS_DIR, "upload_tool_conf.xml")
+FRAMEWORK_SAMPLE_TOOLS_CONF = os.path.join(FRAMEWORK_TOOLS_DIR, "samples_tool_conf.xml")
+FRAMEWORK_DATATYPES_CONF = os.path.join(FRAMEWORK_TOOLS_DIR, "sample_datatypes_conf.xml")
+MIGRATED_TOOL_PANEL_CONFIG = 'config/migrated_tools_conf.xml'
+INSTALLED_TOOL_PANEL_CONFIGS = [
+    os.environ.get('GALAXY_TEST_SHED_TOOL_CONF', 'config/shed_tool_conf.xml')
+]
+
+DEFAULT_LOCALES = "en"
+
+log = logging.getLogger("test_driver")
+
+
+def setup_tool_shed_tmp_dir():
+    tool_shed_test_tmp_dir = os.environ.get('TOOL_SHED_TEST_TMP_DIR', None)
+    if tool_shed_test_tmp_dir is None:
+        tool_shed_test_tmp_dir = tempfile.mkdtemp()
+    # Here's the directory where everything happens.  Temporary directories are created within this directory to contain
+    # the hgweb.config file, the database, new repositories, etc.  Since the tool shed browses repository contents via HTTP,
+    # the full path to the temporary directroy wher eht repositories are located cannot contain invalid url characters.
+    os.environ[ 'TOOL_SHED_TEST_TMP_DIR' ] = tool_shed_test_tmp_dir
+    return tool_shed_test_tmp_dir
+
+
+def get_galaxy_test_tmp_dir():
+    """Create test directory for use by Galaxy server being setup for testing."""
+    galaxy_test_tmp_dir = os.environ.get('GALAXY_TEST_TMP_DIR', None)
+    if galaxy_test_tmp_dir is None:
+        galaxy_test_tmp_dir = tempfile.mkdtemp()
+    return galaxy_test_tmp_dir
+
+
+def configure_environment():
+    """Hack up environment for test cases."""
+    # no op remove if unused
+    if 'HTTP_ACCEPT_LANGUAGE' not in os.environ:
+        os.environ[ 'HTTP_ACCEPT_LANGUAGE' ] = DEFAULT_LOCALES
+
+    # Used by get_filename in tool shed's twilltestcase.
+    if "TOOL_SHED_TEST_FILE_DIR" not in os.environ:
+        os.environ["TOOL_SHED_TEST_FILE_DIR"] = TOOL_SHED_TEST_DATA
+
+
+def build_logger():
+    """Build a logger for test driver script."""
+    return log
+
+
+def setup_galaxy_config(
+    tmpdir,
+    use_test_file_dir=False,
+    default_install_db_merged=True,
+    default_tool_data_table_config_path=None,
+    default_shed_tool_data_table_config=None,
+    enable_tool_shed_check=False,
+    default_tool_conf=None,
+    shed_tool_conf=None,
+    datatypes_conf=None,
+    update_integrated_tool_panel=False,
+):
+    """Setup environment and build config for test Galaxy instance."""
+    if not os.path.exists(tmpdir):
+        os.makedirs(tmpdir)
+    file_path = os.path.join(tmpdir, 'files')
+    template_cache_path = tempfile.mkdtemp(prefix='compiled_templates_', dir=tmpdir)
+    new_file_path = tempfile.mkdtemp(prefix='new_files_path_', dir=tmpdir )
+    job_working_directory = tempfile.mkdtemp(prefix='job_working_directory_', dir=tmpdir)
+
+    if use_test_file_dir:
+        galaxy_test_file_dir = os.environ.get('GALAXY_TEST_FILE_DIR', GALAXY_TEST_FILE_DIR)
+        os.environ['GALAXY_TEST_FILE_DIR'] = galaxy_test_file_dir
+        first_test_file_dir = galaxy_test_file_dir.split(",")[0]
+        if not os.path.isabs(first_test_file_dir):
+            first_test_file_dir = os.path.join(galaxy_root, first_test_file_dir)
+        library_import_dir = first_test_file_dir
+        import_dir = os.path.join(first_test_file_dir, 'users')
+        if os.path.exists(import_dir):
+            user_library_import_dir = import_dir
+        else:
+            user_library_import_dir = None
+    else:
+        user_library_import_dir = None
+        library_import_dir = None
+    tool_path = os.environ.get('GALAXY_TEST_TOOL_PATH', 'tools')
+    tool_dependency_dir = os.environ.get('GALAXY_TOOL_DEPENDENCY_DIR', None)
+    if tool_dependency_dir is None:
+        tool_dependency_dir = tempfile.mkdtemp(dir=tmpdir, prefix="tool_dependencies")
+    tool_data_table_config_path = _tool_data_table_config_path(default_tool_data_table_config_path)
+    default_data_manager_config = 'config/data_manager_conf.xml.sample'
+    for data_manager_config in ['config/data_manager_conf.xml', 'data_manager_conf.xml' ]:
+        if os.path.exists( data_manager_config ):
+            default_data_manager_config = data_manager_config
+    data_manager_config_file = "%s,test/functional/tools/sample_data_manager_conf.xml" % default_data_manager_config
+    master_api_key = get_master_api_key()
+
+    # Data Manager testing temp path
+    # For storing Data Manager outputs and .loc files so that real ones don't get clobbered
+    galaxy_data_manager_data_path = tempfile.mkdtemp(prefix='data_manager_tool-data', dir=tmpdir)
+
+    tool_conf = os.environ.get('GALAXY_TEST_TOOL_CONF', default_tool_conf)
+    if tool_conf is None:
+        # As a fallback always at least allow upload.
+        tool_conf = FRAMEWORK_UPLOAD_TOOL_CONF
+
+    if shed_tool_conf is not None:
+        tool_conf = "%s,%s" % (tool_conf, shed_tool_conf)
+
+    shed_tool_data_table_config = default_shed_tool_data_table_config
+    if shed_tool_data_table_config is None:
+        shed_tool_data_table_config = 'config/shed_tool_data_table_conf.xml'
+
+    config = dict(
+        admin_users='test@bx.psu.edu',
+        allow_library_path_paste=True,
+        allow_user_creation=True,
+        allow_user_deletion=True,
+        api_allow_run_as='test@bx.psu.edu',
+        auto_configure_logging=logging_config_file is None ,
+        check_migrate_tools=False,
+        cleanup_job='onsuccess',
+        data_manager_config_file=data_manager_config_file,
+        enable_beta_tool_formats=True,
+        file_path=file_path,
+        galaxy_data_manager_data_path=galaxy_data_manager_data_path,
+        id_secret='changethisinproductiontoo',
+        job_working_directory=job_working_directory,
+        job_queue_workers=5,
+        library_import_dir=library_import_dir,
+        log_destination="stdout",
+        new_file_path=new_file_path,
+        master_api_key=master_api_key,
+        running_functional_tests=True,
+        shed_tool_data_table_config=shed_tool_data_table_config,
+        template_cache_path=template_cache_path,
+        template_path='templates',
+        tool_config_file=tool_conf,
+        tool_data_table_config_path=tool_data_table_config_path,
+        tool_parse_help=False,
+        tool_path=tool_path,
+        update_integrated_tool_panel=update_integrated_tool_panel,
+        use_tasked_jobs=True,
+        use_heartbeat=False,
+        user_library_import_dir=user_library_import_dir,
+    )
+    config.update(database_conf(tmpdir))
+    config.update(install_database_conf(tmpdir, default_merged=default_install_db_merged))
+    if datatypes_conf is not None:
+        config['datatypes_config_file'] = datatypes_conf
+    if enable_tool_shed_check:
+        config["enable_tool_shed_check"] = enable_tool_shed_check
+        config["hours_between_check"] = 0.001
+    if tool_dependency_dir:
+        config["tool_dependency_dir"] = tool_dependency_dir
+        # Used by shed's twill dependency stuff - todo read from
+        # Galaxy's config API.
+        os.environ["GALAXY_TEST_TOOL_DEPENDENCY_DIR"] = tool_dependency_dir
+    return config
+
+
+def _tool_data_table_config_path(default_tool_data_table_config_path=None):
+    tool_data_table_config_path = os.environ.get('GALAXY_TEST_TOOL_DATA_TABLE_CONF', default_tool_data_table_config_path)
+    if tool_data_table_config_path is None:
+        # ... otherise find whatever Galaxy would use as the default and
+        # the sample data for fucntional tests to that.
+        default_tool_data_config = 'config/tool_data_table_conf.xml.sample'
+        for tool_data_config in ['config/tool_data_table_conf.xml', 'tool_data_table_conf.xml' ]:
+            if os.path.exists( tool_data_config ):
+                default_tool_data_config = tool_data_config
+        tool_data_table_config_path = '%s,test/functional/tool-data/sample_tool_data_tables.xml' % default_tool_data_config
+    return tool_data_table_config_path
+
+
+def nose_config_and_run( argv=None, env=None, ignore_files=[], plugins=None ):
+    """Setup a nose context and run tests.
+
+    Tests are specified by argv (defaulting to sys.argv).
+    """
+    if env is None:
+        env = os.environ
+    if plugins is None:
+        plugins = nose.plugins.manager.DefaultPluginManager()
+    if argv is None:
+        argv = sys.argv
+
+    test_config = nose.config.Config(
+        env=os.environ,
+        ignoreFiles=ignore_files,
+        plugins=plugins,
+    )
+
+    # Add custom plugin to produce JSON data used by planemo.
+    test_config.plugins.addPlugin( StructuredTestDataPlugin() )
+    test_config.configure( argv )
+
+    result = run( test_config )
+
+    success = result.wasSuccessful()
+    return success
+
+
+def copy_database_template( source, db_path ):
+    """Copy a 'clean' sqlite template database.
+
+    From file or URL to specified path for sqlite database.
+    """
+    db_path_dir = os.path.dirname(db_path)
+    if not os.path.exists(db_path_dir):
+        os.makedirs(db_path_dir)
+    if os.path.exists(source):
+        shutil.copy(source, db_path)
+        assert os.path.exists(db_path)
+    elif source.lower().startswith(("http://", "https://", "ftp://")):
+        urlretrieve(source, db_path)
+    else:
+        raise Exception( "Failed to copy database template from source %s" % source )
+
+
+def database_conf(db_path, prefix="GALAXY"):
+    """Find (and populate if needed) Galaxy database connection."""
+    database_auto_migrate = False
+    dburi_var = "%s_TEST_DBURI" % prefix
+    if dburi_var in os.environ:
+        database_connection = os.environ[dburi_var]
+    else:
+        default_db_filename = "%s.sqlite" % prefix.lower()
+        template_var = "%s_TEST_DB_TEMPLATE" % prefix
+        db_path = os.path.join(db_path, default_db_filename)
+        if template_var in os.environ:
+            # Middle ground between recreating a completely new
+            # database and pointing at existing database with
+            # GALAXY_TEST_DBURI. The former requires a lot of setup
+            # time, the latter results in test failures in certain
+            # cases (namely tool shed tests expecting clean database).
+            copy_database_template(os.environ[template_var], db_path)
+            database_auto_migrate = True
+        database_connection = 'sqlite:///%s' % db_path
+    config = {
+        "database_connection": database_connection,
+        "database_auto_migrate": database_auto_migrate
+    }
+    if not database_connection.startswith("sqlite://"):
+        config["database_engine_option_max_overflow"] = "20"
+        config["database_engine_option_pool_size"] = "10"
+    return config
+
+
+def install_database_conf(db_path, default_merged=False):
+    if 'GALAXY_TEST_INSTALL_DBURI' in os.environ:
+        install_galaxy_database_connection = os.environ['GALAXY_TEST_INSTALL_DBURI']
+    elif asbool(os.environ.get('GALAXY_TEST_INSTALL_DB_MERGED', default_merged)):
+        install_galaxy_database_connection = None
+    else:
+        install_galaxy_db_path = os.path.join(db_path, 'install.sqlite')
+        install_galaxy_database_connection = 'sqlite:///%s' % install_galaxy_db_path
+    conf = {}
+    if install_galaxy_database_connection is not None:
+        conf["install_database_connection"] = install_galaxy_database_connection
+    return conf
+
+
+def database_files_path(test_tmpdir, prefix="GALAXY"):
+    """Create a mock database/ directory like in GALAXY_ROOT.
+
+    Use prefix to default this if TOOL_SHED_TEST_DBPATH or
+    GALAXY_TEST_DBPATH is set in the environment.
+    """
+    environ_var = "%s_TEST_DBPATH" % prefix
+    if environ_var in os.environ:
+        db_path = os.environ[environ_var]
+    else:
+        tempdir = tempfile.mkdtemp(dir=test_tmpdir)
+        db_path = os.path.join(tempdir, 'database')
+    return db_path
+
+
+def _get_static_settings():
+    """Configuration required for Galaxy static middleware.
+
+    Returns dictionary of the settings necessary for a galaxy App
+    to be wrapped in the static middleware.
+
+    This mainly consists of the filesystem locations of url-mapped
+    static resources.
+    """
+    static_dir = os.path.join(galaxy_root, "static")
+
+    # TODO: these should be copied from config/galaxy.ini
+    return dict(
+        static_enabled=True,
+        static_cache_time=360,
+        static_dir=static_dir,
+        static_images_dir=os.path.join(static_dir, 'images', ''),
+        static_favicon_dir=os.path.join(static_dir, 'favicon.ico'),
+        static_scripts_dir=os.path.join(static_dir, 'scripts', ''),
+        static_style_dir=os.path.join(static_dir, 'june_2007_style', 'blue'),
+        static_robots_txt=os.path.join(static_dir, 'robots.txt'),
+    )
+
+
+def get_webapp_global_conf():
+    """Get the global_conf dictionary sent to ``app_factory``."""
+    # (was originally sent 'dict()') - nothing here for now except static settings
+    global_conf = dict()
+    global_conf.update( _get_static_settings() )
+    return global_conf
+
+
+def wait_for_http_server(host, port):
+    """Wait for an HTTP server to boot up."""
+    # Test if the server is up
+    for i in range( 10 ):
+        # directly test the app, not the proxy
+        conn = httplib.HTTPConnection(host, port)
+        conn.request( "GET", "/" )
+        if conn.getresponse().status == 200:
+            break
+        time.sleep( 0.1 )
+    else:
+        template = "Test HTTP server on host %s and port %s did not return '200 OK' after 10 tries"
+        message = template % (host, port)
+        raise Exception(message)
+
+
+def serve_webapp(webapp, port=None, host=None):
+    """Serve the webapp on a recommend port or a free one.
+
+    Return the port the webapp is running one.
+    """
+    server = None
+    if port is not None:
+        server = httpserver.serve( webapp, host=host, port=port, start_loop=False )
+    else:
+        random.seed()
+        for i in range( 0, 9 ):
+            try:
+                port = str( random.randint( 8000, 10000 ) )
+                server = httpserver.serve( webapp, host=host, port=port, start_loop=False )
+                break
+            except socket.error, e:
+                if e[0] == 98:
+                    continue
+                raise
+        else:
+            raise Exception( "Unable to open a port between %s and %s to start Galaxy server" % ( 8000, 1000 ) )
+
+    t = threading.Thread( target=server.serve_forever )
+    t.start()
+
+    return server, port
+
+
+def cleanup_directory(tempdir):
+    """Clean up temporary files used by test unless GALAXY_TEST_NO_CLEANUP is set.
+
+    Also respect TOOL_SHED_TEST_NO_CLEANUP for legacy reasons.
+    """
+    skip_cleanup = "GALAXY_TEST_NO_CLEANUP" in os.environ or "TOOL_SHED_TEST_NO_CLEANUP" in os.environ
+    if skip_cleanup:
+        log.info( "GALAXY_TEST_NO_CLEANUP is on. Temporary files in %s" % tempdir )
+        return
+    try:
+        if os.path.exists(tempdir) and skip_cleanup:
+            shutil.rmtree(tempdir)
+    except Exception:
+        pass
+
+
+def setup_shed_tools_for_test(app, tmpdir, testing_migrated_tools, testing_installed_tools):
+    """Modify Galaxy app's toolbox for migrated or installed tool tests."""
+    # Store a jsonified dictionary of tool_id : GALAXY_TEST_FILE_DIR pairs.
+    galaxy_tool_shed_test_file = os.path.join(tmpdir, 'shed_tools_dict')
+    shed_tools_dict = {}
+    if testing_migrated_tools:
+        has_test_data, shed_tools_dict = parse_tool_panel_config(MIGRATED_TOOL_PANEL_CONFIG, shed_tools_dict)
+    elif testing_installed_tools:
+        for shed_tool_config in INSTALLED_TOOL_PANEL_CONFIGS:
+            has_test_data, shed_tools_dict = parse_tool_panel_config(shed_tool_config, shed_tools_dict)
+    # Persist the shed_tools_dict to the galaxy_tool_shed_test_file.
+    with open(galaxy_tool_shed_test_file, 'w') as shed_tools_file:
+        shed_tools_file.write(json.dumps(shed_tools_dict))
+    if not os.path.isabs(galaxy_tool_shed_test_file):
+        galaxy_tool_shed_test_file = os.path.join(galaxy_root, galaxy_tool_shed_test_file)
+    os.environ['GALAXY_TOOL_SHED_TEST_FILE'] = galaxy_tool_shed_test_file
+    if testing_installed_tools:
+        # TODO: Do this without modifying app - that is a pretty violation
+        # of Galaxy's abstraction - we shouldn't require app at all let alone
+        # be modifying it.
+
+        tool_configs = app.config.tool_configs
+        # Eliminate the migrated_tool_panel_config from the app's tool_configs, append the list of installed_tool_panel_configs,
+        # and reload the app's toolbox.
+        relative_migrated_tool_panel_config = os.path.join(app.config.root, MIGRATED_TOOL_PANEL_CONFIG)
+        if relative_migrated_tool_panel_config in tool_configs:
+            tool_configs.remove(relative_migrated_tool_panel_config)
+        for installed_tool_panel_config in INSTALLED_TOOL_PANEL_CONFIGS:
+            tool_configs.append(installed_tool_panel_config)
+        from galaxy import tools  # noqa, delay import because this brings in so many modules for small tests
+        app.toolbox = tools.ToolBox(tool_configs, app.config.tool_path, app)
+
+
+def build_galaxy_app(simple_kwargs):
+    """Build a Galaxy app object from a simple keyword arguments.
+
+    Construct paste style complex dictionary and use load_app_properties so
+    Galaxy override variables are respected. Also setup "global" references
+    to sqlalchemy database context for Galaxy and install databases.
+    """
+    log.info("Galaxy database connection:", simple_kwargs["database_connection"])
+    simple_kwargs['global_conf'] = get_webapp_global_conf()
+    simple_kwargs['global_conf']['__file__'] = "config/galaxy.ini.sample"
+    simple_kwargs = load_app_properties(
+        kwds=simple_kwargs
+    )
+    # Build the Universe Application
+    app = GalaxyUniverseApplication( **simple_kwargs )
+    log.info( "Embedded Galaxy application started" )
+    database_contexts.galaxy_context = app.model.context
+    database_contexts.install_context = app.install_model.context
+    return app
+
+
+def build_shed_app(simple_kwargs):
+    """Build a Galaxy app object from a simple keyword arguments.
+
+    Construct paste style complex dictionary. Also setup "global" reference
+    to sqlalchemy database context for tool shed database.
+    """
+    log.info("Tool shed database connection:", simple_kwargs["database_connection"])
+    # TODO: Simplify global_conf to match Galaxy above...
+    simple_kwargs['__file__'] = 'tool_shed_wsgi.ini.sample'
+    simple_kwargs['global_conf'] = get_webapp_global_conf()
+
+    app = ToolshedUniverseApplication( **simple_kwargs )
+    database_contexts.tool_shed_context = app.model.context
+    log.info( "Embedded Toolshed application started" )
+    return app
+
+ServerWrapper = collections.namedtuple('ServerWrapper', ['app', 'server', 'name', 'host', 'port'])
+
+
+def _stop(self):
+    if self.server is not None:
+        log.info("Shutting down embedded %s web server" % self.name)
+        self.server.server_close()
+        log.info("Embedded web server %s stopped" % self.name)
+
+    if self.app is not None:
+        log.info("Stopping application %s" % self.name)
+        self.app.shutdown()
+        log.info("Application %s stopped." % self.name)
+
+ServerWrapper.stop = _stop
+
+
+def launch_server(app, webapp_factory, kwargs, prefix="GALAXY"):
+    """Launch a web server for a given app using supplied factory.
+
+    Consistently read either GALAXY_TEST_HOST and GALAXY_TEST_PORT or
+    TOOL_SHED_TEST_HOST and TOOL_SHED_TEST_PORT and ensure these are
+    all set after this method has been called.
+    """
+    name = prefix.lower()
+
+    host_env_key = "%s_TEST_HOST" % prefix
+    port_env_key = "%s_TEST_PORT" % prefix
+    host = os.environ.get(host_env_key, DEFAULT_WEB_HOST)
+    port = os.environ.get(port_env_key, None)
+
+    webapp = webapp_factory(
+        kwargs[ 'global_conf' ],
+        app=app,
+        use_translogger=False,
+        static_enabled=True
+    )
+    server, port = serve_webapp(
+        webapp,
+        host=host, port=port
+    )
+    os.environ[host_env_key] = host
+    os.environ[port_env_key] = port
+    wait_for_http_server(host, port)
+    log.info("Embedded web server for %s started" % name)
+    return ServerWrapper(
+        app, server, name, host, port
+    )
+
+
+class TestDriver(object):
+    """Responsible for the life-cycle of a Galaxy-style functional test.
+
+    Sets up servers, configures tests, runs nose, and tears things
+    down. This is somewhat like a Python TestCase - but different
+    because it is meant to provide a main() endpoint.
+    """
+
+    def __init__(self):
+        """Setup tracked resources."""
+        self.server_wrappers = []
+        self.temp_directories = []
+
+    def setup(self):
+        """Called before tests are built."""
+
+    def build_tests(self):
+        """After environment is setup, setup nose tests."""
+
+    def tear_down(self):
+        """Cleanup resources tracked by this object."""
+        for server_wrapper in self.server_wrappers:
+            server_wrapper.stop()
+        for temp_directory in self.temp_directories:
+            cleanup_directory(temp_directory)
+
+    def run(self):
+        """Driver whole test.
+
+        Setup environment, build tests (if needed), run test,
+        and finally cleanup resources.
+        """
+        configure_environment()
+        self.setup()
+        self.build_tests()
+        try:
+            success = nose_config_and_run()
+            return 0 if success else 1
+        except Exception as e:
+            log.info("Failure running tests")
+            raise e
+        finally:
+            log.info( "Shutting down")
+            self.tear_down()
+
+
+class GalaxyTestDriver(TestDriver):
+    """Instantial a Galaxy-style nose TestDriver for testing Galaxy."""
+
+    testing_shed_tools = False
+
+    def setup(self):
+        """Setup a Galaxy server for functional test (if needed)."""
+        self.external_galaxy = os.environ.get('GALAXY_TEST_EXTERNAL', None)
+        self.galaxy_test_tmp_dir = get_galaxy_test_tmp_dir()
+        self.temp_directories.append(self.galaxy_test_tmp_dir)
+
+        testing_shed_tools = getattr(self, "testing_shed_tools", False)
+        default_tool_conf = getattr(self, "default_tool_conf", None)
+        datatypes_conf_override = getattr(self, "datatypes_conf_override", None)
+
+        if self.external_galaxy is None:
+            tempdir = tempfile.mkdtemp(dir=self.galaxy_test_tmp_dir)
+            # Configure the database path.
+            galaxy_db_path = database_files_path(tempdir)
+            galaxy_config = setup_galaxy_config(
+                galaxy_db_path,
+                use_test_file_dir=not testing_shed_tools,
+                default_install_db_merged=True,
+                default_tool_conf=default_tool_conf,
+                datatypes_conf=datatypes_conf_override,
+            )
+
+            # ---- Build Application --------------------------------------------------
+            self.app = build_galaxy_app(galaxy_config)
+            server_wrapper = launch_server(
+                self.app,
+                buildapp.app_factory,
+                galaxy_config,
+            )
+            self.server_wrappers.append(server_wrapper)
+            log.info("Functional tests will be run against %s:%s" % (server_wrapper.host, server_wrapper.port))
+        else:
+            log.info("Functional tests will be run against %s" % self.external_galaxy)
+
+    def setup_shed_tools(self, testing_migrated_tools=False, testing_installed_tools=True):
+        setup_shed_tools_for_test(
+            self.app,
+            self.galaxy_test_tmp_dir,
+            testing_migrated_tools,
+            testing_installed_tools
+        )
+
+    def build_tool_tests(self):
+        if self.app is None:
+            return
+
+        # We must make sure that functional.test_toolbox is always imported after
+        # database_contexts.galaxy_content is set (which occurs in this method above).
+        # If functional.test_toolbox is imported before database_contexts.galaxy_content
+        # is set, sa_session will be None in all methods that use it.
+        import functional.test_toolbox
+        functional.test_toolbox.toolbox = self.app.toolbox
+        # When testing data managers, do not test toolbox.
+        functional.test_toolbox.build_tests(
+            app=self.app,
+            testing_shed_tools=self.testing_shed_tools,
+            master_api_key=get_master_api_key(),
+            user_api_key=get_user_api_key(),
+        )
+
+
+def drive_test(test_driver_class):
+    """Instantiate driver class, run, and exit appropriately."""
+    sys.exit(test_driver_class().run())
+
+
+__all__ = [
+    "copy_database_template",
+    "build_logger",
+    "drive_test",
+    "FRAMEWORK_UPLOAD_TOOL_CONF",
+    "FRAMEWORK_SAMPLE_TOOLS_CONF",
+    "FRAMEWORK_DATATYPES_CONF",
+    "database_conf",
+    "get_webapp_global_conf",
+    "nose_config_and_run",
+    "setup_galaxy_config",
+    "TestDriver",
+    "wait_for_http_server",
+]
@@ -48,7 +48,8 @@ class TwillTestCase( unittest.TestCase ):
        self.history_id = os.environ.get( 'GALAXY_TEST_HISTORY_ID', None )
        self.host = os.environ.get( 'GALAXY_TEST_HOST' )
        self.port = os.environ.get( 'GALAXY_TEST_PORT' )
-        self.url = "http://%s:%s" % ( self.host, self.port )
+        default_url = "http://%s:%s" % (self.host, self.port)
+        self.url = os.environ.get('GALAXY_TEST_EXTERNAL', default_url)
        self.test_data_resolver = TestDataResolver( )
        self.tool_shed_test_file = os.environ.get( 'GALAXY_TOOL_SHED_TEST_FILE', None )
        if self.tool_shed_test_file:
@@ -0,0 +1,6 @@
+<?xml version="1.0"?>
+<toolbox tool_path="${tool_conf_dir}">
+  <section id="getext" name="Get Data">
+    <tool file="upload.xml"/>
+  </section>
+</toolbox>
@@ -17,7 +17,7 @@ class WorkflowTestCase( TwillTestCase ):
    Kind of a shell of a test case for running workflow tests. Probably
    needs to look more like test_toolbox.
    """
-    workflow_test_file = None
+    workflow_test_file = os.environ.get("GALAXY_TEST_WORKFLOW_FILE", None)
    user_api_key = None
    master_api_key = None

@@ -1,96 +1,26 @@
 #!/usr/bin/env python
+"""Test driver for tool shed functional tests.
+
+Launch this script by running ``run_tests.sh -t`` from GALAXY_ROOT.
+"""
 from __future__ import absolute_import

-import httplib
-import logging
 import os
-import random
-import shutil
-import socket
 import string
 import sys
 import tempfile
-import threading
-import time
-import urllib

 galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir))
-tool_shed_home_directory = os.path.join( galaxy_root, 'test', 'tool_shed' )
-default_tool_shed_test_file_dir = os.path.join( tool_shed_home_directory, 'test_data' )
-# Here's the directory where everything happens.  Temporary directories are created within this directory to contain
-# the hgweb.config file, the database, new repositories, etc.  Since the tool shed browses repository contents via HTTP,
-# the full path to the temporary directroy wher eht repositories are located cannot contain invalid url characters.
-tool_shed_test_tmp_dir = os.path.join( tool_shed_home_directory, 'tmp' )
-os.environ[ 'TOOL_SHED_TEST_TMP_DIR' ] = tool_shed_test_tmp_dir
 # Need to remove this directory from sys.path
 sys.path[0:1] = [ os.path.join( galaxy_root, "lib" ), os.path.join( galaxy_root, "test" ) ]

-from paste import httpserver
-import nose.config
-import nose.core
-import nose.loader
-import nose.plugins.manager
+from base import driver_util
+log = driver_util.build_logger()

 # This is for the tool shed application.
 from galaxy.webapps.tool_shed import buildapp as toolshedbuildapp
-from galaxy.webapps.tool_shed.app import UniverseApplication as ToolshedUniverseApplication
-# This is for the galaxy application.
-from galaxy.app import UniverseApplication as GalaxyUniverseApplication
-from galaxy.util import asbool
 from galaxy.web import buildapp as galaxybuildapp

-from base import nose_util
-from functional import database_contexts
-
-log = logging.getLogger( "tool_shed_functional_tests.py" )
-
-default_tool_shed_test_host = "localhost"
-default_tool_shed_test_port_min = 8000
-default_tool_shed_test_port_max = 8999
-default_tool_shed_locales = 'en'
-default_galaxy_test_port_min = 9000
-default_galaxy_test_port_max = 9999
-default_galaxy_test_host = 'localhost'
-
-# Use separate databases for Galaxy and tool shed install info by default,
-# set GALAXY_TEST_INSTALL_DB_MERGED to True to revert to merged databases
-# behavior.
-default_install_db_merged = False
-
-# should this serve static resources (scripts, images, styles, etc.)
-STATIC_ENABLED = True
-
-
-def get_static_settings():
-    """Returns dictionary of the settings necessary for a galaxy App
-    to be wrapped in the static middleware.
-
-    This mainly consists of the filesystem locations of url-mapped
-    static resources.
-    """
-    static_dir = os.path.join( galaxy_root, 'static' )
-    # TODO: these should be copied from galaxy.ini
-    return dict(
-        # TODO: static_enabled needed here?
-        static_enabled=True,
-        static_cache_time=360,
-        static_dir=static_dir,
-        static_images_dir=os.path.join( static_dir, 'images', '' ),
-        static_favicon_dir=os.path.join( static_dir, 'favicon.ico' ),
-        static_scripts_dir=os.path.join( static_dir, 'scripts', '' ),
-        static_style_dir=os.path.join( static_dir, 'june_2007_style', 'blue' ),
-        static_robots_txt=os.path.join( static_dir, 'robots.txt' ),
-    )
-
-
-def get_webapp_global_conf():
-    """Get the global_conf dictionary sent as the first argument to app_factory.
-    """
-    # (was originally sent 'dict()') - nothing here for now except static settings
-    global_conf = dict()
-    if STATIC_ENABLED:
-        global_conf.update( get_static_settings() )
-    return global_conf

 tool_sheds_conf_xml_template = '''<?xml version="1.0"?>
 <tool_sheds>
@@ -105,14 +35,6 @@ shed_tool_conf_xml_template = '''<?xml version="1.0"?>
 </toolbox>
 '''

-tool_conf_xml = '''<?xml version="1.0"?>
-<toolbox>
-    <section name="Get Data" id="getext">
-        <tool file="data_source/upload.xml"/>
-    </section>
-</toolbox>
-'''
-
 tool_data_table_conf_xml_template = '''<?xml version="1.0"?>
 <tables>
 </tables>
@@ -124,372 +46,124 @@ shed_data_manager_conf_xml_template = '''<?xml version="1.0"?>
 '''


-def run_tests( test_config ):
-    return nose_util.run( test_config )
+class ToolShedTestDriver(driver_util.TestDriver):
+    """Instantial a Galaxy-style nose TestDriver for testing the tool shed."""

-
-def main():
-    # ---- Configuration ------------------------------------------------------
-    tool_shed_test_host = os.environ.get( 'TOOL_SHED_TEST_HOST', default_tool_shed_test_host )
-    tool_shed_test_port = os.environ.get( 'TOOL_SHED_TEST_PORT', None )
-    galaxy_test_host = os.environ.get( 'GALAXY_TEST_HOST', default_galaxy_test_host )
-    galaxy_test_port = os.environ.get( 'GALAXY_TEST_PORT', None )
-    tool_path = os.environ.get( 'TOOL_SHED_TEST_TOOL_PATH', 'tools' )
-    if 'HTTP_ACCEPT_LANGUAGE' not in os.environ:
-        os.environ[ 'HTTP_ACCEPT_LANGUAGE' ] = default_tool_shed_locales
-    tool_shed_test_file_dir = os.environ.get( 'TOOL_SHED_TEST_FILE_DIR', default_tool_shed_test_file_dir )
-    if not os.path.isabs( tool_shed_test_file_dir ):
-        tool_shed_test_file_dir = tool_shed_test_file_dir
-    ignore_files = ()
-    tool_dependency_dir = os.environ.get( 'TOOL_SHED_TOOL_DEPENDENCY_DIR', None )
-    use_distributed_object_store = os.environ.get( 'TOOL_SHED_USE_DISTRIBUTED_OBJECT_STORE', False )
-    if not os.path.isdir( tool_shed_test_tmp_dir ):
-        os.mkdir( tool_shed_test_tmp_dir )
-    tool_shed_test_proxy_port = None
-    galaxy_test_proxy_port = None
-    if 'TOOL_SHED_TEST_DBPATH' in os.environ:
-        shed_db_path = os.environ[ 'TOOL_SHED_TEST_DBPATH' ]
-    else:
-        tempdir = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir )
-        shed_db_path = os.path.join( tempdir, 'database' )
-    shed_tool_data_table_conf_file = os.environ.get( 'TOOL_SHED_TEST_TOOL_DATA_TABLE_CONF', os.path.join( tool_shed_test_tmp_dir, 'shed_tool_data_table_conf.xml' ) )
-    galaxy_shed_data_manager_conf_file = os.environ.get( 'GALAXY_SHED_DATA_MANAGER_CONF', os.path.join( tool_shed_test_tmp_dir, 'test_shed_data_manager_conf.xml' ) )
-    galaxy_tool_data_table_conf_file = os.environ.get( 'GALAXY_TEST_TOOL_DATA_TABLE_CONF', os.path.join( tool_shed_test_tmp_dir, 'tool_data_table_conf.xml' ) )
-    galaxy_tool_conf_file = os.environ.get( 'GALAXY_TEST_TOOL_CONF', os.path.join( tool_shed_test_tmp_dir, 'test_tool_conf.xml' ) )
-    galaxy_shed_tool_conf_file = os.environ.get( 'GALAXY_TEST_SHED_TOOL_CONF', os.path.join( tool_shed_test_tmp_dir, 'test_shed_tool_conf.xml' ) )
-    galaxy_migrated_tool_conf_file = os.environ.get( 'GALAXY_TEST_MIGRATED_TOOL_CONF', os.path.join( tool_shed_test_tmp_dir, 'test_migrated_tool_conf.xml' ) )
-    galaxy_tool_sheds_conf_file = os.environ.get( 'GALAXY_TEST_TOOL_SHEDS_CONF', os.path.join( tool_shed_test_tmp_dir, 'test_sheds_conf.xml' ) )
-    if 'GALAXY_TEST_TOOL_DATA_PATH' in os.environ:
-        tool_data_path = os.environ.get( 'GALAXY_TEST_TOOL_DATA_PATH' )
-    else:
-        tool_data_path = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir )
-        os.environ[ 'GALAXY_TEST_TOOL_DATA_PATH' ] = tool_data_path
-    if 'GALAXY_TEST_DBPATH' in os.environ:
-        galaxy_db_path = os.environ[ 'GALAXY_TEST_DBPATH' ]
-    else:
-        tempdir = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir )
-        galaxy_db_path = os.path.join( tempdir, 'database' )
-    shed_file_path = os.path.join( shed_db_path, 'files' )
-    galaxy_file_path = os.path.join( galaxy_db_path, 'files' )
-    hgweb_config_file_path = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir )
-    new_repos_path = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir )
-    galaxy_tempfiles = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir )
-    galaxy_shed_tool_path = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir )
-    galaxy_migrated_tool_path = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir )
-    galaxy_tool_dependency_dir = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir )
-    os.environ[ 'GALAXY_TEST_TOOL_DEPENDENCY_DIR' ] = galaxy_tool_dependency_dir
-    hgweb_config_dir = hgweb_config_file_path
-    os.environ[ 'TEST_HG_WEB_CONFIG_DIR' ] = hgweb_config_dir
-    print "Directory location for hgweb.config:", hgweb_config_dir
-    if 'TOOL_SHED_TEST_DBURI' in os.environ:
-        toolshed_database_connection = os.environ[ 'TOOL_SHED_TEST_DBURI' ]
-    else:
-        toolshed_database_connection = 'sqlite:///' + os.path.join( shed_db_path, 'community_test.sqlite' )
-    galaxy_database_auto_migrate = False
-    if 'GALAXY_TEST_DBURI' in os.environ:
-        galaxy_database_connection = os.environ[ 'GALAXY_TEST_DBURI' ]
-    else:
-        db_path = os.path.join( galaxy_db_path, 'universe.sqlite' )
-        if 'GALAXY_TEST_DB_TEMPLATE' in os.environ:
-            # Middle ground between recreating a completely new
-            # database and pointing at existing database with
-            # GALAXY_TEST_DBURI. The former requires a lot of setup
-            # time, the latter results in test failures in certain
-            # cases (namely tool shed tests expecting clean database).
-            __copy_database_template(os.environ['GALAXY_TEST_DB_TEMPLATE'], db_path)
-            galaxy_database_auto_migrate = True
-        if not os.path.exists(galaxy_db_path):
-            os.makedirs(galaxy_db_path)
-        galaxy_database_connection = 'sqlite:///%s' % db_path
-    if 'GALAXY_TEST_INSTALL_DBURI' in os.environ:
-        install_galaxy_database_connection = os.environ[ 'GALAXY_TEST_INSTALL_DBURI' ]
-    elif asbool( os.environ.get( 'GALAXY_TEST_INSTALL_DB_MERGED', default_install_db_merged ) ):
-        install_galaxy_database_connection = galaxy_database_connection
-    else:
-        install_galaxy_db_path = os.path.join( galaxy_db_path, 'install.sqlite' )
-        install_galaxy_database_connection = 'sqlite:///%s' % install_galaxy_db_path
-    tool_shed_global_conf = get_webapp_global_conf()
-    tool_shed_global_conf[ '__file__' ] = 'tool_shed_wsgi.ini.sample'
-    kwargs = dict( admin_users='test@bx.psu.edu',
-                   allow_user_creation=True,
-                   allow_user_deletion=True,
-                   database_connection=toolshed_database_connection,
-                   datatype_converters_config_file='datatype_converters_conf.xml.sample',
-                   file_path=shed_file_path,
-                   global_conf=tool_shed_global_conf,
-                   hgweb_config_dir=hgweb_config_dir,
-                   job_queue_workers=5,
-                   id_secret='changethisinproductiontoo',
-                   log_destination="stdout",
-                   new_file_path=new_repos_path,
-                   running_functional_tests=True,
-                   shed_tool_data_table_config=shed_tool_data_table_conf_file,
-                   smtp_server='smtp.dummy.string.tld',
-                   email_from='functional@localhost',
-                   template_path='templates',
-                   tool_path=tool_path,
-                   tool_parse_help=False,
-                   tool_data_table_config_path=galaxy_tool_data_table_conf_file,
-                   use_heartbeat=False )
-    for dir in [ tool_shed_test_tmp_dir ]:
-        try:
-            os.makedirs( dir )
-        except OSError:
-            pass
-
-    print "Tool shed database connection:", toolshed_database_connection
-    print "Galaxy database connection:", galaxy_database_connection
-
-    # Generate the tool_data_table_conf.xml file.
-    file( galaxy_tool_data_table_conf_file, 'w' ).write( tool_data_table_conf_xml_template )
-    # Generate the shed_tool_data_table_conf.xml file.
-    file( shed_tool_data_table_conf_file, 'w' ).write( tool_data_table_conf_xml_template )
-    os.environ[ 'TOOL_SHED_TEST_TOOL_DATA_TABLE_CONF' ] = shed_tool_data_table_conf_file
-    # ---- Build Tool Shed Application --------------------------------------------------
-    toolshedapp = None
-#    if not toolshed_database_connection.startswith( 'sqlite://' ):
-#        kwargs[ 'database_engine_option_max_overflow' ] = '20'
-    if tool_dependency_dir is not None:
-        kwargs[ 'tool_dependency_dir' ] = tool_dependency_dir
-    if use_distributed_object_store:
-        kwargs[ 'object_store' ] = 'distributed'
-        kwargs[ 'distributed_object_store_config_file' ] = 'distributed_object_store_conf.xml.sample'
-
-    kwargs[ 'global_conf' ] = tool_shed_global_conf
-
-    if not toolshed_database_connection.startswith( 'sqlite://' ):
-            kwargs[ 'database_engine_option_pool_size' ] = '10'
-
-    toolshedapp = ToolshedUniverseApplication( **kwargs )
-    database_contexts.tool_shed_context = toolshedapp.model.context
-    log.info( "Embedded Toolshed application started" )
-
-    # ---- Run tool shed webserver ------------------------------------------------------
-    tool_shed_server = None
-    tool_shed_global_conf[ 'database_connection' ] = toolshed_database_connection
-    toolshedwebapp = toolshedbuildapp.app_factory( tool_shed_global_conf,
-                                                   use_translogger=False,
-                                                   static_enabled=True,
-                                                   app=toolshedapp )
-    if tool_shed_test_port is not None:
-        tool_shed_server = httpserver.serve( toolshedwebapp, host=tool_shed_test_host, port=tool_shed_test_port, start_loop=False )
-    else:
-        random.seed()
-        for i in range( 0, 9 ):
-            try:
-                tool_shed_test_port = str( random.randint( default_tool_shed_test_port_min, default_tool_shed_test_port_max ) )
-                log.debug( "Attempting to serve app on randomly chosen port: %s" % tool_shed_test_port )
-                tool_shed_server = httpserver.serve( toolshedwebapp, host=tool_shed_test_host, port=tool_shed_test_port, start_loop=False )
-                break
-            except socket.error, e:
-                if e[0] == 98:
-                    continue
-                raise
+    def setup(self):
+        """Entry point for test driver script."""
+        # ---- Configuration ------------------------------------------------------
+        tool_shed_test_tmp_dir = driver_util.setup_tool_shed_tmp_dir()
+        if not os.path.isdir( tool_shed_test_tmp_dir ):
+            os.mkdir( tool_shed_test_tmp_dir )
+        self.temp_directories.append(tool_shed_test_tmp_dir)
+        shed_db_path = driver_util.database_files_path(tool_shed_test_tmp_dir, prefix="TOOL_SHED")
+        shed_tool_data_table_conf_file = os.environ.get( 'TOOL_SHED_TEST_TOOL_DATA_TABLE_CONF', os.path.join( tool_shed_test_tmp_dir, 'shed_tool_data_table_conf.xml' ) )
+        galaxy_shed_data_manager_conf_file = os.environ.get( 'GALAXY_SHED_DATA_MANAGER_CONF', os.path.join( tool_shed_test_tmp_dir, 'test_shed_data_manager_conf.xml' ) )
+        default_tool_data_table_config_path = os.path.join( tool_shed_test_tmp_dir, 'tool_data_table_conf.xml' )
+        galaxy_shed_tool_conf_file = os.environ.get( 'GALAXY_TEST_SHED_TOOL_CONF', os.path.join( tool_shed_test_tmp_dir, 'test_shed_tool_conf.xml' ) )
+        galaxy_migrated_tool_conf_file = os.environ.get( 'GALAXY_TEST_MIGRATED_TOOL_CONF', os.path.join( tool_shed_test_tmp_dir, 'test_migrated_tool_conf.xml' ) )
+        galaxy_tool_sheds_conf_file = os.environ.get( 'GALAXY_TEST_TOOL_SHEDS_CONF', os.path.join( tool_shed_test_tmp_dir, 'test_sheds_conf.xml' ) )
+        if 'GALAXY_TEST_TOOL_DATA_PATH' in os.environ:
+            tool_data_path = os.environ.get( 'GALAXY_TEST_TOOL_DATA_PATH' )
        else:
-            raise Exception( "Unable to open a port between %s and %s to start Galaxy server" % ( default_tool_shed_test_port_min, default_tool_shed_test_port_max ) )
-    if tool_shed_test_proxy_port:
-        os.environ[ 'TOOL_SHED_TEST_PORT' ] = tool_shed_test_proxy_port
-    else:
-        os.environ[ 'TOOL_SHED_TEST_PORT' ] = tool_shed_test_port
-    t = threading.Thread( target=tool_shed_server.serve_forever )
-    t.start()
-    # Test if the server is up
-    for i in range( 10 ):
-        # Directly test the app, not the proxy.
-        conn = httplib.HTTPConnection( tool_shed_test_host, tool_shed_test_port )
-        conn.request( "GET", "/" )
-        if conn.getresponse().status == 200:
-            break
-        time.sleep( 0.1 )
-    else:
-        raise Exception( "Test HTTP server did not return '200 OK' after 10 tries" )
-    log.info( "Embedded web server started" )
-
-    # ---- Optionally start up a Galaxy instance ------------------------------------------------------
-    if 'TOOL_SHED_TEST_OMIT_GALAXY' not in os.environ:
-        # Generate the tool_conf.xml file.
-        file( galaxy_tool_conf_file, 'w' ).write( tool_conf_xml )
-        # Generate the shed_tool_conf.xml file.
-        tool_sheds_conf_template_parser = string.Template( tool_sheds_conf_xml_template )
-        tool_sheds_conf_xml = tool_sheds_conf_template_parser.safe_substitute( shed_url=tool_shed_test_host, shed_port=tool_shed_test_port )
-        file( galaxy_tool_sheds_conf_file, 'w' ).write( tool_sheds_conf_xml )
-        # Generate the tool_sheds_conf.xml file.
-        shed_tool_conf_template_parser = string.Template( shed_tool_conf_xml_template )
-        shed_tool_conf_xml = shed_tool_conf_template_parser.safe_substitute( shed_tool_path=galaxy_shed_tool_path )
-        file( galaxy_shed_tool_conf_file, 'w' ).write( shed_tool_conf_xml )
-        # Generate the migrated_tool_conf.xml file.
-        migrated_tool_conf_xml = shed_tool_conf_template_parser.safe_substitute( shed_tool_path=galaxy_migrated_tool_path )
-        file( galaxy_migrated_tool_conf_file, 'w' ).write( migrated_tool_conf_xml )
-        os.environ[ 'GALAXY_TEST_SHED_TOOL_CONF' ] = galaxy_shed_tool_conf_file
-        # Generate shed_data_manager_conf.xml
-        if not os.environ.get( 'GALAXY_SHED_DATA_MANAGER_CONF' ):
-            open( galaxy_shed_data_manager_conf_file, 'wb' ).write( shed_data_manager_conf_xml_template )
-        galaxy_global_conf = get_webapp_global_conf()
-        galaxy_global_conf[ '__file__' ] = 'config/galaxy.ini.sample'
-
-        kwargs = dict( allow_user_creation=True,
+            tool_data_path = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir )
+            os.environ[ 'GALAXY_TEST_TOOL_DATA_PATH' ] = tool_data_path
+        galaxy_db_path = driver_util.database_files_path(tool_shed_test_tmp_dir)
+        shed_file_path = os.path.join( shed_db_path, 'files' )
+        hgweb_config_file_path = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir )
+        new_repos_path = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir )
+        galaxy_shed_tool_path = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir )
+        galaxy_migrated_tool_path = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir )
+        hgweb_config_dir = hgweb_config_file_path
+        os.environ[ 'TEST_HG_WEB_CONFIG_DIR' ] = hgweb_config_dir
+        print "Directory location for hgweb.config:", hgweb_config_dir
+        toolshed_database_conf = driver_util.database_conf(shed_db_path, prefix="TOOL_SHED")
+        kwargs = dict( admin_users='test@bx.psu.edu',
+                       allow_user_creation=True,
                       allow_user_deletion=True,
-                       admin_users='test@bx.psu.edu',
-                       allow_library_path_paste=True,
-                       install_database_connection=install_galaxy_database_connection,
-                       database_connection=galaxy_database_connection,
-                       database_auto_migrate=galaxy_database_auto_migrate,
-                       datatype_converters_config_file="datatype_converters_conf.xml.sample",
-                       check_migrate_tools=False,
-                       enable_tool_shed_check=True,
-                       file_path=galaxy_file_path,
-                       global_conf=galaxy_global_conf,
-                       hours_between_check=0.001,
-                       id_secret='changethisinproductiontoo',
+                       datatype_converters_config_file='datatype_converters_conf.xml.sample',
+                       file_path=shed_file_path,
+                       hgweb_config_dir=hgweb_config_dir,
                       job_queue_workers=5,
+                       id_secret='changethisinproductiontoo',
                       log_destination="stdout",
-                       migrated_tools_config=galaxy_migrated_tool_conf_file,
-                       new_file_path=galaxy_tempfiles,
+                       new_file_path=new_repos_path,
                       running_functional_tests=True,
-                       shed_data_manager_config_file=galaxy_shed_data_manager_conf_file,
                       shed_tool_data_table_config=shed_tool_data_table_conf_file,
-                       shed_tool_path=galaxy_shed_tool_path,
-                       template_path="templates",
-                       tool_data_path=tool_data_path,
-                       tool_dependency_dir=galaxy_tool_dependency_dir,
-                       tool_path=tool_path,
-                       tool_config_file=[ galaxy_tool_conf_file, galaxy_shed_tool_conf_file ],
-                       tool_sheds_config_file=galaxy_tool_sheds_conf_file,
+                       smtp_server='smtp.dummy.string.tld',
+                       email_from='functional@localhost',
+                       template_path='templates',
                       tool_parse_help=False,
-                       tool_data_table_config_path=galaxy_tool_data_table_conf_file,
-                       update_integrated_tool_panel=False,
                       use_heartbeat=False )
+        kwargs.update(toolshed_database_conf)
+        # Generate the tool_data_table_conf.xml file.
+        file( default_tool_data_table_config_path, 'w' ).write( tool_data_table_conf_xml_template )
+        # Generate the shed_tool_data_table_conf.xml file.
+        file( shed_tool_data_table_conf_file, 'w' ).write( tool_data_table_conf_xml_template )
+        os.environ[ 'TOOL_SHED_TEST_TOOL_DATA_TABLE_CONF' ] = shed_tool_data_table_conf_file
+        # ---- Build Tool Shed Application --------------------------------------------------
+        toolshedapp = driver_util.build_shed_app(kwargs)

-        # ---- Build Galaxy Application --------------------------------------------------
-        if not galaxy_database_connection.startswith( 'sqlite://' ) and not install_galaxy_database_connection.startswith( 'sqlite://' ):
-            kwargs[ 'database_engine_option_pool_size' ] = '10'
-            kwargs[ 'database_engine_option_max_overflow' ] = '20'
-        galaxyapp = GalaxyUniverseApplication( **kwargs )
-
-        log.info( "Embedded Galaxy application started" )
-
-        # ---- Run galaxy webserver ------------------------------------------------------
-        galaxy_server = None
-        galaxy_global_conf[ 'database_file' ] = galaxy_database_connection
-        galaxywebapp = galaxybuildapp.app_factory( galaxy_global_conf,
-                                                   use_translogger=False,
-                                                   static_enabled=True,
-                                                   app=galaxyapp )
-        database_contexts.galaxy_context = galaxyapp.model.context
-        database_contexts.install_context = galaxyapp.install_model.context
-        if galaxy_test_port is not None:
-            galaxy_server = httpserver.serve( galaxywebapp, host=galaxy_test_host, port=galaxy_test_port, start_loop=False )
-        else:
-            random.seed()
-            for i in range( 0, 9 ):
-                try:
-                    galaxy_test_port = str( random.randint( default_galaxy_test_port_min, default_galaxy_test_port_max ) )
-                    log.debug( "Attempting to serve app on randomly chosen port: %s" % galaxy_test_port )
-                    galaxy_server = httpserver.serve( galaxywebapp, host=galaxy_test_host, port=galaxy_test_port, start_loop=False )
-                    break
-                except socket.error, e:
-                    if e[0] == 98:
-                        continue
-                    raise
-            else:
-                raise Exception( "Unable to open a port between %s and %s to start Galaxy server" %
-                                 ( default_galaxy_test_port_min, default_galaxy_test_port_max ) )
-        if galaxy_test_proxy_port:
-            os.environ[ 'GALAXY_TEST_PORT' ] = galaxy_test_proxy_port
-        else:
-            os.environ[ 'GALAXY_TEST_PORT' ] = galaxy_test_port
-        t = threading.Thread( target=galaxy_server.serve_forever )
-        t.start()
-        # Test if the server is up
-        for i in range( 10 ):
-            # Directly test the app, not the proxy.
-            conn = httplib.HTTPConnection( galaxy_test_host, galaxy_test_port )
-            conn.request( "GET", "/" )
-            if conn.getresponse().status == 200:
-                break
-            time.sleep( 0.1 )
-        else:
-            raise Exception( "Test HTTP server did not return '200 OK' after 10 tries" )
-        log.info( "Embedded galaxy web server started" )
-    # ---- Find tests ---------------------------------------------------------
-    if tool_shed_test_proxy_port:
-        log.info( "Functional tests will be run against %s:%s" % ( tool_shed_test_host, tool_shed_test_proxy_port ) )
-    else:
+        # ---- Run tool shed webserver ------------------------------------------------------
+        # TODO: Needed for hg middleware ('lib/galaxy/webapps/tool_shed/framework/middleware/hg.py')
+        kwargs['global_conf']['database_connection'] = kwargs["database_connection"]
+        tool_shed_server_wrapper = driver_util.launch_server(
+            toolshedapp,
+            toolshedbuildapp.app_factory,
+            kwargs,
+            prefix="TOOL_SHED",
+        )
+        self.server_wrappers.append(tool_shed_server_wrapper)
+        tool_shed_test_host = tool_shed_server_wrapper.host
+        tool_shed_test_port = tool_shed_server_wrapper.port
        log.info( "Functional tests will be run against %s:%s" % ( tool_shed_test_host, tool_shed_test_port ) )
-    if galaxy_test_proxy_port:
-        log.info( "Galaxy tests will be run against %s:%s" % ( galaxy_test_host, galaxy_test_proxy_port ) )
-    else:
-        log.info( "Galaxy tests will be run against %s:%s" % ( galaxy_test_host, galaxy_test_port ) )
-    success = False
-    try:
-        # Pass in through script set env, will leave a copy of ALL test validate files.
-        os.environ[ 'TOOL_SHED_TEST_HOST' ] = tool_shed_test_host
-        os.environ[ 'GALAXY_TEST_HOST' ] = galaxy_test_host
-        if tool_shed_test_file_dir:
-            os.environ[ 'TOOL_SHED_TEST_FILE_DIR' ] = tool_shed_test_file_dir
-        test_config = nose.config.Config( env=os.environ, ignoreFiles=ignore_files, plugins=nose.plugins.manager.DefaultPluginManager() )
-        test_config.configure( sys.argv )
-        # Run the tests.
-        result = run_tests( test_config )
-        success = result.wasSuccessful()
-    except:
-        log.exception( "Failure running tests" )

-    log.info( "Shutting down" )
-    # ---- Tear down -----------------------------------------------------------
-    if tool_shed_server:
-        log.info( "Shutting down embedded web server" )
-        tool_shed_server.server_close()
-        tool_shed_server = None
-        log.info( "Embedded web server stopped" )
-    if toolshedapp:
-        log.info( "Shutting down tool shed app" )
-        toolshedapp.shutdown()
-        toolshedapp = None
-        log.info( "Embedded tool shed application stopped" )
-    if 'TOOL_SHED_TEST_OMIT_GALAXY' not in os.environ:
-        if galaxy_server:
-            log.info( "Shutting down galaxy web server" )
-            galaxy_server.server_close()
-            galaxy_server = None
-            log.info( "Embedded galaxy server stopped" )
-        if galaxyapp:
-            log.info( "Shutting down galaxy app" )
-            galaxyapp.shutdown()
-            galaxyapp = None
-            log.info( "Embedded galaxy application stopped" )
-    if 'TOOL_SHED_TEST_NO_CLEANUP' not in os.environ:
-        try:
-            for dir in [ tool_shed_test_tmp_dir ]:
-                if os.path.exists( dir ):
-                    log.info( "Cleaning up temporary files in %s" % dir )
-                    shutil.rmtree( dir )
-        except:
-            pass
-    if success:
-        return 0
-    else:
-        return 1
+        # ---- Optionally start up a Galaxy instance ------------------------------------------------------
+        if 'TOOL_SHED_TEST_OMIT_GALAXY' not in os.environ:
+            # Generate the shed_tool_conf.xml file.
+            tool_sheds_conf_template_parser = string.Template( tool_sheds_conf_xml_template )
+            tool_sheds_conf_xml = tool_sheds_conf_template_parser.safe_substitute( shed_url=tool_shed_test_host, shed_port=tool_shed_test_port )
+            file( galaxy_tool_sheds_conf_file, 'w' ).write( tool_sheds_conf_xml )
+            # Generate the tool_sheds_conf.xml file.
+            shed_tool_conf_template_parser = string.Template( shed_tool_conf_xml_template )
+            shed_tool_conf_xml = shed_tool_conf_template_parser.safe_substitute( shed_tool_path=galaxy_shed_tool_path )
+            file( galaxy_shed_tool_conf_file, 'w' ).write( shed_tool_conf_xml )
+            # Generate the migrated_tool_conf.xml file.
+            migrated_tool_conf_xml = shed_tool_conf_template_parser.safe_substitute( shed_tool_path=galaxy_migrated_tool_path )
+            file( galaxy_migrated_tool_conf_file, 'w' ).write( migrated_tool_conf_xml )
+            os.environ[ 'GALAXY_TEST_SHED_TOOL_CONF' ] = galaxy_shed_tool_conf_file
+            # Generate shed_data_manager_conf.xml
+            if not os.environ.get( 'GALAXY_SHED_DATA_MANAGER_CONF' ):
+                open( galaxy_shed_data_manager_conf_file, 'wb' ).write( shed_data_manager_conf_xml_template )
+            kwargs = dict( migrated_tools_config=galaxy_migrated_tool_conf_file,
+                           shed_data_manager_config_file=galaxy_shed_data_manager_conf_file,
+                           shed_tool_path=galaxy_shed_tool_path,
+                           tool_data_path=tool_data_path,
+                           tool_sheds_config_file=galaxy_tool_sheds_conf_file )
+            kwargs.update(
+                driver_util.setup_galaxy_config(
+                    galaxy_db_path,
+                    use_test_file_dir=False,
+                    default_install_db_merged=False,
+                    default_tool_data_table_config_path=default_tool_data_table_config_path,
+                    default_shed_tool_data_table_config=shed_tool_data_table_conf_file,
+                    enable_tool_shed_check=True,
+                    shed_tool_conf=galaxy_shed_tool_conf_file,
+                    update_integrated_tool_panel=True,
+                )
+            )
+            print "Galaxy database connection:", kwargs["database_connection"]

-
-def __copy_database_template( source, db_path ):
-    """
-    Copy a 'clean' sqlite template database (from file or URL) to specified
-    database path.
-    """
-    os.makedirs( os.path.dirname( db_path ) )
-    if os.path.exists( source ):
-        shutil.copy( source, db_path )
-        assert os.path.exists( db_path )
-    elif source.startswith("http"):
-        urllib.urlretrieve( source, db_path )
-    else:
-        raise Exception( "Failed to copy database template from source %s" % source )
+            # ---- Run galaxy webserver ------------------------------------------------------
+            galaxyapp = driver_util.build_galaxy_app(kwargs)
+            galaxy_server_wrapper = driver_util.launch_server(
+                galaxyapp,
+                galaxybuildapp.app_factory,
+                kwargs,
+            )
+            log.info("Galaxy tests will be run against %s:%s" % (galaxy_server_wrapper.host, galaxy_server_wrapper.port))
+            self.server_wrappers.append(galaxy_server_wrapper)


 if __name__ == "__main__":
-    try:
-        sys.exit( main() )
-    except Exception, e:
-        log.exception( str( e ) )
-        exit(1)
+    driver_util.drive_test(ToolShedTestDriver)