From e7b5293534d6cf9a704b9d295271abffda0f3c5d Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 28 Mar 2016 12:46:43 -0400 Subject: [PATCH] Refactor toolbox hacks used by installed and migration tool tests out... into driver_util. This greatly simplifies the code structure of scripts/functional_tests.py's main method. Add description of GALAXY_TEST_SHED_TOOL_CONF and some documentation to all of this. --- run_tests.sh | 3 ++- scripts/functional_tests.py | 50 ++++++++++--------------------------- test/base/driver_util.py | 38 ++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 38 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 7c5173ed98b..2b8b7d8ee8b 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -69,7 +69,8 @@ GALAXY_TEST_NO_CLEANUP Do not cleanup main test directory after tests, GALAXY_TEST_HOST Host to use for Galaxy server setup for testing. GALAXY_TEST_PORT Port to use for Galaxy server setup for testing. GALAXY_TEST_TOOL_PATH Defaults to 'tools' - +GALAXY_TEST_SHED_TOOL_CONF Shed toolbox conf (defaults to config/shed_tool_conf.xml) + used when testing installed to tools with -installed. TOOL_SHED_TEST_HOST Host to use for shed server setup for testing. TOOL_SHED_TEST_PORT Port to use for shed server setup for testing. TOOL_SHED_TEST_FILE_DIR Defaults to test/tool_shed/test_data. diff --git a/scripts/functional_tests.py b/scripts/functional_tests.py index 858d3fb1594..674785863b3 100644 --- a/scripts/functional_tests.py +++ b/scripts/functional_tests.py @@ -10,7 +10,6 @@ import os.path import sys import tempfile from ConfigParser import SafeConfigParser -from json import dumps galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir)) sys.path[1:1] = [ os.path.join( galaxy_root, "lib" ), os.path.join( galaxy_root, "test" ) ] @@ -21,18 +20,12 @@ log = driver_util.build_logger() from base.api_util import get_master_api_key, get_user_api_key from base.test_logging import logging_config_file -from base.tool_shed_util import parse_tool_panel_config from functional import database_contexts -from galaxy import tools from galaxy.app import UniverseApplication from galaxy.util.properties import load_app_properties from galaxy.web import buildapp default_galaxy_test_host = "localhost" -migrated_tool_panel_config = 'config/migrated_tools_conf.xml' -installed_tool_panel_configs = [ - os.environ.get('GALAXY_TEST_SHED_TOOL_CONF', 'config/shed_tool_conf.xml') -] # Set up a job_conf.xml that explicitly limits jobs to 10 minutes. @@ -98,13 +91,14 @@ def main(): testing_installed_tools = _check_arg( '-installed' ) datatypes_conf_override = None - use_test_file_dir = not (testing_migrated_tools or testing_installed_tools) - if not use_test_file_dir: + testing_shed_tools = testing_migrated_tools or testing_installed_tools + if testing_shed_tools: # Store a jsonified dictionary of tool_id : GALAXY_TEST_FILE_DIR pairs. galaxy_tool_shed_test_file = 'shed_tools_dict' # We need the upload tool for functional tests, so we'll create a temporary tool panel config that defines it. tool_config_file = driver_util.FRAMEWORK_UPLOAD_TOOL_CONF else: + galaxy_tool_shed_test_file = None framework_test = _check_arg( '-framework' ) # Run through suite of tests testing framework. if framework_test: tool_conf = driver_util.FRAMEWORK_SAMPLE_TOOLS_CONF @@ -153,7 +147,7 @@ def main(): galaxy_db_path = driver_util.database_files_path(tempdir) galaxy_config = driver_util.setup_galaxy_config( galaxy_db_path, - use_test_file_dir=use_test_file_dir + use_test_file_dir=not testing_shed_tools, ) database_connection, database_auto_migrate = driver_util.database_conf(galaxy_db_path) @@ -225,7 +219,6 @@ def main(): log.info( "Functional tests will be run against %s:%s" % ( galaxy_test_host, galaxy_test_port ) ) success = False try: - tool_configs = app.config.tool_configs # What requires these? Handy for (eg) functional tests to save outputs? # Pass in through script setenv, will leave a copy of ALL test validate files os.environ[ 'GALAXY_TEST_HOST' ] = galaxy_test_host @@ -263,36 +256,19 @@ def main(): ) return driver_util.nose_config_and_run() - if not use_test_file_dir: - shed_tools_dict = {} - if testing_migrated_tools: - has_test_data, shed_tools_dict = parse_tool_panel_config( migrated_tool_panel_config, shed_tools_dict ) - elif testing_installed_tools: - for shed_tool_config in installed_tool_panel_configs: - has_test_data, shed_tools_dict = parse_tool_panel_config( shed_tool_config, shed_tools_dict ) - # Persist the shed_tools_dict to the galaxy_tool_shed_test_file. - shed_tools_file = open( galaxy_tool_shed_test_file, 'w' ) - shed_tools_file.write( dumps( shed_tools_dict ) ) - shed_tools_file.close() - if not os.path.isabs( galaxy_tool_shed_test_file ): - galaxy_tool_shed_test_file = os.path.join( os.getcwd(), galaxy_tool_shed_test_file ) - os.environ[ 'GALAXY_TOOL_SHED_TEST_FILE' ] = galaxy_tool_shed_test_file - if testing_installed_tools: - # Eliminate the migrated_tool_panel_config from the app's tool_configs, append the list of installed_tool_panel_configs, - # and reload the app's toolbox. - relative_migrated_tool_panel_config = os.path.join( app.config.root, migrated_tool_panel_config ) - if relative_migrated_tool_panel_config in tool_configs: - tool_configs.remove( relative_migrated_tool_panel_config ) - for installed_tool_panel_config in installed_tool_panel_configs: - tool_configs.append( installed_tool_panel_config ) - app.toolbox = tools.ToolBox( tool_configs, app.config.tool_path, app ) - success = _run_functional_test( testing_shed_tools=True ) + if testing_shed_tools: + driver_util.setup_shed_tools_for_test( + app, + galaxy_tool_shed_test_file, + testing_migrated_tools, + testing_installed_tools, + ) + success = _run_functional_test( testing_shed_tools=True ) + if galaxy_tool_shed_test_file is not None: try: os.unlink( galaxy_tool_shed_test_file ) except: log.info( "Unable to remove file: %s" % galaxy_tool_shed_test_file ) - else: - success = _run_functional_test( ) except: log.exception( "Failure running tests" ) diff --git a/test/base/driver_util.py b/test/base/driver_util.py index 49537b7fbac..8f215554ee2 100644 --- a/test/base/driver_util.py +++ b/test/base/driver_util.py @@ -1,6 +1,7 @@ """Scripts for drivers of Galaxy functional tests.""" import httplib +import json import logging import os import random @@ -20,6 +21,7 @@ import nose.plugins.manager from paste import httpserver +from .tool_shed_util import parse_tool_panel_config from .nose_util import run from .instrument import StructuredTestDataPlugin @@ -33,6 +35,10 @@ FRAMEWORK_TOOLS_DIR = os.path.join(GALAXY_TEST_DIRECTORY, "functional", "tools") FRAMEWORK_UPLOAD_TOOL_CONF = os.path.join(FRAMEWORK_TOOLS_DIR, "upload_tool_conf.xml") FRAMEWORK_SAMPLE_TOOLS_CONF = os.path.join(FRAMEWORK_TOOLS_DIR, "samples_tool_conf.xml") FRAMEWORK_DATATYPES_CONF = os.path.join(FRAMEWORK_TOOLS_DIR, "sample_datatypes_conf.xml") +MIGRATED_TOOL_PANEL_CONFIG = 'config/migrated_tools_conf.xml' +INSTALLED_TOOL_PANEL_CONFIGS = [ + os.environ.get('GALAXY_TEST_SHED_TOOL_CONF', 'config/shed_tool_conf.xml') +] DEFAULT_LOCALES = "en" @@ -314,6 +320,37 @@ def cleanup_directory(tempdir): pass +def setup_shed_tools_for_test(app, galaxy_tool_shed_test_file, testing_migrated_tools, testing_installed_tools): + """Modify Galaxy app's toolbox for migrated or installed tool tests.""" + shed_tools_dict = {} + if testing_migrated_tools: + has_test_data, shed_tools_dict = parse_tool_panel_config(MIGRATED_TOOL_PANEL_CONFIG, shed_tools_dict) + elif testing_installed_tools: + for shed_tool_config in INSTALLED_TOOL_PANEL_CONFIGS: + has_test_data, shed_tools_dict = parse_tool_panel_config(shed_tool_config, shed_tools_dict) + # Persist the shed_tools_dict to the galaxy_tool_shed_test_file. + with open(galaxy_tool_shed_test_file, 'w') as shed_tools_file: + shed_tools_file.write(json.dumps(shed_tools_dict)) + if not os.path.isabs(galaxy_tool_shed_test_file): + galaxy_tool_shed_test_file = os.path.join(galaxy_root, galaxy_tool_shed_test_file) + os.environ['GALAXY_TOOL_SHED_TEST_FILE'] = galaxy_tool_shed_test_file + if testing_installed_tools: + # TODO: Do this without modifying app - that is a pretty violation + # of Galaxy's abstraction - we shouldn't require app at all let alone + # be modifying it. + + tool_configs = app.config.tool_configs + # Eliminate the migrated_tool_panel_config from the app's tool_configs, append the list of installed_tool_panel_configs, + # and reload the app's toolbox. + relative_migrated_tool_panel_config = os.path.join(app.config.root, MIGRATED_TOOL_PANEL_CONFIG) + if relative_migrated_tool_panel_config in tool_configs: + tool_configs.remove(relative_migrated_tool_panel_config) + for installed_tool_panel_config in INSTALLED_TOOL_PANEL_CONFIGS: + tool_configs.append(installed_tool_panel_config) + from galaxy import tools # noqa, delay import because this brings in so many modules for small tests + app.toolbox = tools.ToolBox(tool_configs, app.config.tool_path, app) + + __all__ = [ "cleanup_directory", "configure_environment", @@ -326,5 +363,6 @@ __all__ = [ "get_webapp_global_conf", "nose_config_and_run", "setup_galaxy_config", + "setup_shed_tools_for_test", "wait_for_http_server", ]