diff --git a/.ci/flake8_blacklist.txt b/.ci/flake8_blacklist.txt index 6b1ffa12a84..7904194e7b7 100644 --- a/.ci/flake8_blacklist.txt +++ b/.ci/flake8_blacklist.txt @@ -19,7 +19,4 @@ scripts/scramble/ scripts/tool_shed/ scripts/tools/ scripts/transfer.py -test/base/ -test/casperjs/ -test/install_and_test_tool_shed_repositories/ tools/ diff --git a/.ci/flake8_wrapper.sh b/.ci/flake8_wrapper.sh index 4f725cd4845..3d6036ee3fc 100755 --- a/.ci/flake8_wrapper.sh +++ b/.ci/flake8_wrapper.sh @@ -5,4 +5,4 @@ set -e flake8 --exclude `paste -sd, .ci/flake8_blacklist.txt` . # Look for obviously broken stuff lots more places. -flake8 --select=E901,E902,F821,F822,F823,F831 --exclude lib/galaxy/util/pastescript/serve.py,lib/pkg_resources.py lib/ test/{api,unit} +flake8 --select=E901,E902,F821,F822,F823,F831 --exclude lib/pkg_resources.py contrib/ lib/ diff --git a/test/base/asserts/__init__.py b/test/base/asserts/__init__.py index fb95dd6742d..04a18e08733 100644 --- a/test/base/asserts/__init__.py +++ b/test/base/asserts/__init__.py @@ -1,6 +1,7 @@ import inspect import logging import sys + log = logging.getLogger( __name__ ) assertion_module_names = ['text', 'tabular', 'xml'] @@ -14,7 +15,7 @@ for assertion_module_name in assertion_module_names: full_assertion_module_name = 'base.asserts.' + assertion_module_name log.debug(full_assertion_module_name) try: - #Dynamically import module + # Dynamically import module __import__(full_assertion_module_name) assertion_module = sys.modules[full_assertion_module_name] assertion_modules.append(assertion_module) @@ -40,7 +41,7 @@ def verify_assertion(data, assertion_description): if assert_function is None: errmsg = "Unable to find test function associated with XML tag '%s'. Check your tool file syntax." % tag raise AssertionError(errmsg) - + assert_function_args = inspect.getargspec(assert_function).args args = {} for attribute, value in assertion_description["attributes"].iteritems(): @@ -72,6 +73,6 @@ def verify_assertion(data, assertion_description): if "children" in assert_function_args: args["children"] = assertion_description["children"] - + # TODO: Verify all needed function arguments are specified. assert_function(**args) diff --git a/test/base/interactor.py b/test/base/interactor.py index b70bf4fc5d4..535e6446c96 100644 --- a/test/base/interactor.py +++ b/test/base/interactor.py @@ -1,17 +1,18 @@ import os import re +from json import dumps +from logging import getLogger from StringIO import StringIO -from galaxy.tools.parser.interface import TestCollectionDef + from galaxy import eggs eggs.require( "requests" ) +from requests import get, post + from galaxy import util +from galaxy.tools.parser.interface import TestCollectionDef +from galaxy.util.bunch import Bunch from galaxy.util.odict import odict -from galaxy.util.bunch import Bunch -from requests import get -from requests import post -from json import dumps -from logging import getLogger log = getLogger( __name__ ) # Off by default because it can pound the database pretty heavily @@ -57,7 +58,7 @@ class GalaxyInteractorApi( object ): name = output_testdef.name self.wait_for_jobs( history_id, jobs, maxseconds ) hid = self.__output_id( output_data ) - ## TODO: Twill version verifys dataset is 'ok' in here. + # TODO: Twill version verifys dataset is 'ok' in here. self.verify_output_dataset( history_id=history_id, hda_id=hid, outfile=outfile, attributes=attributes, shed_tool_id=shed_tool_id ) primary_datasets = attributes.get( 'primary_datasets', {} ) @@ -155,10 +156,10 @@ class GalaxyInteractorApi( object ): file_name = self.twill_test_case.get_filename( composite_file.get( 'value' ), shed_tool_id=shed_tool_id ) files["files_%s|file_data" % i] = open( file_name, 'rb' ) tool_input.update({ - #"files_%d|NAME" % i: name, + # "files_%d|NAME" % i: name, "files_%d|type" % i: "upload_dataset", - ## TODO: - #"files_%d|space_to_tab" % i: composite_file.get( 'space_to_tab', False ) + # TODO: + # "files_%d|space_to_tab" % i: composite_file.get( 'space_to_tab', False ) }) name = test_data[ 'name' ] else: @@ -180,7 +181,7 @@ class GalaxyInteractorApi( object ): dataset = submit_response["outputs"][0] except KeyError: raise Exception(submit_response) - #raise Exception(str(dataset)) + # raise Exception(str(dataset)) hid = dataset['id'] self.uploads[ os.path.basename(fname) ] = self.uploads[ fname ] = self.uploads[ name ] = {"src": "hda", "id": hid} return self.__wait_for_history( history_id ) @@ -203,7 +204,7 @@ class GalaxyInteractorApi( object ): new_values.append( value ) inputs_tree[ key ] = new_values - # # HACK: Flatten single-value lists. Required when using expand_grouping + # HACK: Flatten single-value lists. Required when using expand_grouping for key, value in inputs_tree.iteritems(): if isinstance(value, list) and len(value) == 1: inputs_tree[key] = value[0] @@ -255,9 +256,9 @@ class GalaxyInteractorApi( object ): return output_collections_dict def __dictify_outputs( self, datasets_object ): - ## Convert outputs list to a dictionary that can be accessed by - ## output_name so can be more flexiable about ordering of outputs - ## but also allows fallback to legacy access as list mode. + # Convert outputs list to a dictionary that can be accessed by + # output_name so can be more flexiable about ordering of outputs + # but also allows fallback to legacy access as list mode. outputs_dict = odict() index = 0 for output in datasets_object[ 'outputs' ]: diff --git a/test/base/twilltestcase.py b/test/base/twilltestcase.py index 5e29c4cd7eb..88fdae40c6b 100644 --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -1,4 +1,3 @@ -import StringIO import difflib import filecmp import logging @@ -6,6 +5,7 @@ import os import pprint import re import shutil +import StringIO import subprocess import tarfile import tempfile @@ -13,9 +13,10 @@ import time import unittest import urllib import zipfile +from json import loads from urlparse import urlparse - from xml.etree import ElementTree + from galaxy import eggs eggs.require( "MarkupSafe" ) from markupsafe import escape @@ -24,14 +25,13 @@ import twill import twill.commands as tc from twill.other_packages._mechanize_dist import ClientForm -from base.asserts import verify_assertions -from base.test_data import TestDataResolver -from galaxy.util import asbool -from galaxy.util.json import loads from galaxy.web import security from galaxy.web.framework.helpers import iff -#Force twill to log to a buffer -- FIXME: Should this go to stdout and be captured by nose? +from base.asserts import verify_assertions +from base.test_data import TestDataResolver + +# Force twill to log to a buffer -- FIXME: Should this go to stdout and be captured by nose? buffer = StringIO.StringIO() twill.set_output( buffer ) tc.config( 'use_tidy', 0 ) @@ -69,8 +69,8 @@ class TwillTestCase( unittest.TestCase ): def act_on_multiple_datasets( self, cntrller, library_id, do_action, ldda_ids='', strings_displayed=[] ): # Can't use the ~/library_admin/libraries form as twill barfs on it so we'll simulate the form submission # by going directly to the form action - self.visit_url( '%s/library_common/act_on_multiple_datasets?cntrller=%s&library_id=%s&ldda_ids=%s&do_action=%s' \ - % ( self.url, cntrller, library_id, ldda_ids, do_action ) ) + self.visit_url( '%s/library_common/act_on_multiple_datasets?cntrller=%s&library_id=%s&ldda_ids=%s&do_action=%s' % + ( self.url, cntrller, library_id, ldda_ids, do_action ) ) for check_str in strings_displayed: self.check_page_for_string( check_str ) @@ -139,7 +139,7 @@ class TwillTestCase( unittest.TestCase ): self.check_page_for_string( check_str ) def add_tag( self, item_id, item_class, context, new_tag ): - self.visit_url( "%s/tag/add_tag_async?item_id=%s&item_class=%s&context=%s&new_tag=%s" % \ + self.visit_url( "%s/tag/add_tag_async?item_id=%s&item_class=%s&context=%s&new_tag=%s" % ( self.url, item_id, item_class, context, new_tag ) ) def add_template( self, cntrller, item_type, form_type, form_id, form_name, @@ -156,7 +156,7 @@ class TwillTestCase( unittest.TestCase ): elif item_type == 'ldda': params[ 'ldda_id' ] = ldda_id self.visit_url( url, params ) - self.check_page_for_string ( "Select a template for the" ) + self.check_page_for_string( "Select a template for the" ) self.refresh_form( "form_id", form_id ) # For some unknown reason, twill barfs if the form number ( 1 ) is used in the following # rather than the form anme ( select_template ), so we have to use the form name. @@ -308,6 +308,7 @@ class TwillTestCase( unittest.TestCase ): parent_folder = parent_folder.parent path += ldda.name return path + def mkdir( file ): dir = os.path.join( tmpd, os.path.dirname( file ) ) if not os.path.exists( dir ): @@ -361,7 +362,7 @@ class TwillTestCase( unittest.TestCase ): (3) The contents of that key match the provided value. If use_string_contains=True, this will perform a substring match, otherwise an exact match. """ - #TODO: multi key, value + # TODO: multi key, value hda = dict() for history_item in self.get_history_from_api(): if history_item[ 'id' ] == hda_id: @@ -458,7 +459,7 @@ class TwillTestCase( unittest.TestCase ): def check_request_grid( self, cntrller, state, deleted=False, strings_displayed=[] ): params = { 'f-state': state, 'f-deleted': deleted, 'sort': 'create_time' } - self.visit_url( '/%s/browse_requests' % cntrller ) + self.visit_url( '/%s/browse_requests' % cntrller, params ) for check_str in strings_displayed: self.check_page_for_string( check_str ) @@ -603,7 +604,7 @@ class TwillTestCase( unittest.TestCase ): url = "/admin/groups" params = dict( operation='create', create_group_button='Save', name=name ) if in_user_ids: - params [ 'in_users' ] = ','.join( in_user_ids ) + params[ 'in_users' ] = ','.join( in_user_ids ) if in_role_ids: params[ 'in_roles' ] = ','.join( in_role_ids ) if create_role_for_group: @@ -743,7 +744,6 @@ class TwillTestCase( unittest.TestCase ): tc.fv( "1", "name", name ) tc.fv( "1", "desc", desc ) for index, field_value_tuple in enumerate( field_value_tuples ): - field_index = index + 1 field_name, field_value, refresh_on_change = field_value_tuple if refresh_on_change: # Only the AddressField type has a refresh on change setup on selecting an option @@ -754,7 +754,7 @@ class TwillTestCase( unittest.TestCase ): # handle new address self.check_page_for_string( 'Short address description' ) for address_field, value in address_value.items(): - tc.fv( "1", field_name+'_'+address_field, value ) + tc.fv( "1", field_name + '_' + address_field, value ) else: # existing address tc.fv( "1", field_name, address_value ) @@ -783,7 +783,7 @@ class TwillTestCase( unittest.TestCase ): url_params[ 'create_group_for_role' ] = [ 'yes', 'yes' ] doseq = True else: - doseq=False + doseq = False self.visit_url( url, params=url_params, doseq=doseq ) for check_str in strings_displayed: self.check_page_for_string( check_str ) @@ -869,11 +869,11 @@ class TwillTestCase( unittest.TestCase ): # page from an external URL. By "barfs", I mean that twill somehow loses hod on the # cntrller param. We'll just simulate the form submission by building the URL manually. # Here's the old, better approach... - #self.visit_url( "%s/library_common/browse_library?cntrller=%s&id=%s" % ( self.url, cntrller, library_id ) ) - #for ldda_id in ldda_ids: + # self.visit_url( "%s/library_common/browse_library?cntrller=%s&id=%s" % ( self.url, cntrller, library_id ) ) + # for ldda_id in ldda_ids: # tc.fv( "1", "ldda_ids", ldda_id ) - #tc.fv( "1", "do_action", format ) - #tc.submit( "action_on_datasets_button" ) + # tc.fv( "1", "do_action", format ) + # tc.submit( "action_on_datasets_button" ) # Here's the new approach... params = dict( cntrller=cntrller, library_id=library_id, do_action=format, ldda_ids=ldda_ids ) url = "/library_common/act_on_multiple_datasets" @@ -893,7 +893,6 @@ class TwillTestCase( unittest.TestCase ): if new_desc: tc.fv( "1", "desc", new_desc ) for index, ( field_name, field_value ) in enumerate( new_fields ): - field_name_index = index + 1 tc.fv( "1", field_name, field_value ) tc.submit( "edit_basic_request_info_button" ) for check_str in strings_displayed_after_submit: @@ -1071,7 +1070,7 @@ class TwillTestCase( unittest.TestCase ): attributes = {} if attributes.get( 'sort', False ): history_data.sort() - ##Why even bother with the check loop below, why not just use the diff output? This seems wasteful. + # Why even bother with the check loop below, why not just use the diff output? This seems wasteful. if len( local_file ) == len( history_data ): for i in range( len( history_data ) ): if local_file[i].rstrip( '\r\n' ) != history_data[i].rstrip( '\r\n' ): @@ -1091,11 +1090,11 @@ class TwillTestCase( unittest.TestCase ): diff_slice = diff[0:40] else: diff_slice = diff[:25] + ["********\n", "*SNIP *\n", "********\n"] + diff[-25:] - #FIXME: This pdf stuff is rather special cased and has not been updated to consider lines_diff - #due to unknown desired behavior when used in conjunction with a non-zero lines_diff - #PDF forgiveness can probably be handled better by not special casing by __extension__ here - #and instead using lines_diff or a regular expression matching - #or by creating and using a specialized pdf comparison function + # FIXME: This pdf stuff is rather special cased and has not been updated to consider lines_diff + # due to unknown desired behavior when used in conjunction with a non-zero lines_diff + # PDF forgiveness can probably be handled better by not special casing by __extension__ here + # and instead using lines_diff or a regular expression matching + # or by creating and using a specialized pdf comparison function if file1.endswith( '.pdf' ) or file2.endswith( '.pdf' ): # PDF files contain creation dates, modification dates, ids and descriptions that change with each # new file, so we need to handle these differences. As long as the rest of the PDF file does @@ -1145,7 +1144,7 @@ class TwillTestCase( unittest.TestCase ): for i in range( len( history_data ) ): if not re.match( local_file[i].rstrip( '\r\n' ), history_data[i].rstrip( '\r\n' ) ): line_diff_count += 1 - diffs.append( 'Regular Expression: %s\nData file : %s' % ( local_file[i].rstrip( '\r\n' ), history_data[i].rstrip( '\r\n' ) ) ) + diffs.append( 'Regular Expression: %s\nData file : %s' % ( local_file[i].rstrip( '\r\n' ), history_data[i].rstrip( '\r\n' ) ) ) if line_diff_count > lines_diff: raise AssertionError( "Regular expression did not match data file (allowed variants=%i):\n%s" % ( lines_diff, "".join( diffs ) ) ) @@ -1160,13 +1159,13 @@ class TwillTestCase( unittest.TestCase ): history_data = ''.join( history_data ) else: history_data = open( file2, 'U' ).read() - #lines_diff not applicable to multiline matching + # lines_diff not applicable to multiline matching assert re.match( local_file, history_data, re.MULTILINE ), "Multiline Regular expression did not match data file" def files_contains( self, file1, file2, attributes=None ): """Checks the contents of file2 for substrings found in file1, on a per-line basis""" local_file = open( file1, 'U' ).readlines() # regex file - #TODO: allow forcing ordering of contains + # TODO: allow forcing ordering of contains history_data = open( file2, 'U' ).read() lines_diff = int( attributes.get( 'lines_diff', 0 ) ) line_diff_count = 0 @@ -1188,7 +1187,7 @@ class TwillTestCase( unittest.TestCase ): template_refresh_field_contents='', template_fields=[], strings_displayed=[], strings_not_displayed=[], strings_displayed_after_submit=[], strings_not_displayed_after_submit=[] ): """Add information to a library using an existing template with 2 elements""" - self.visit_url( "%s/library_common/folder_info?cntrller=%s&id=%s&library_id=%s" % \ + self.visit_url( "%s/library_common/folder_info?cntrller=%s&id=%s&library_id=%s" % ( self.url, cntrller, folder_id, library_id ) ) if name and new_name and description: tc.fv( '1', "name", new_name ) @@ -1300,7 +1299,7 @@ class TwillTestCase( unittest.TestCase ): return jsondata[ 'state' ] in [ 'queued', 'running' ] def get_tags( self, item_id, item_class ): - self.visit_url( "%s/tag/get_tagging_elt_async?item_id=%s&item_class=%s" % \ + self.visit_url( "%s/tag/get_tagging_elt_async?item_id=%s&item_class=%s" % ( self.url, item_id, item_class ) ) def history_as_xml_tree( self, show_deleted=False ): @@ -1353,8 +1352,8 @@ class TwillTestCase( unittest.TestCase ): def import_datasets_to_histories( self, cntrller, library_id, ldda_ids='', new_history_name='Unnamed history', strings_displayed=[] ): # Can't use the ~/library_admin/libraries form as twill barfs on it so we'll simulate the form submission # by going directly to the form action - self.visit_url( '%s/library_common/import_datasets_to_histories?cntrller=%s&library_id=%s&ldda_ids=%s&new_history_name=%s&import_datasets_to_histories_button=Import+library+datasets' \ - % ( self.url, cntrller, library_id, ldda_ids, new_history_name ) ) + self.visit_url( '%s/library_common/import_datasets_to_histories?cntrller=%s&library_id=%s&ldda_ids=%s&new_history_name=%s&import_datasets_to_histories_button=Import+library+datasets' % + ( self.url, cntrller, library_id, ldda_ids, new_history_name ) ) for check_str in strings_displayed: self.check_page_for_string( check_str ) @@ -1417,7 +1416,7 @@ class TwillTestCase( unittest.TestCase ): def ldda_info( self, cntrller, library_id, folder_id, ldda_id, strings_displayed=[], strings_not_displayed=[] ): """View library_dataset_dataset_association information""" - self.visit_url( "%s/library_common/ldda_info?cntrller=%s&library_id=%s&folder_id=%s&id=%s" % \ + self.visit_url( "%s/library_common/ldda_info?cntrller=%s&library_id=%s&folder_id=%s&id=%s" % ( self.url, cntrller, library_id, folder_id, ldda_id ) ) for check_str in strings_displayed: self.check_page_for_string( check_str ) @@ -1431,7 +1430,7 @@ class TwillTestCase( unittest.TestCase ): def ldda_edit_info( self, cntrller, library_id, folder_id, ldda_id, ldda_name, new_ldda_name='', template_refresh_field_name='1_field_name', template_refresh_field_contents='', template_fields=[], strings_displayed=[], strings_not_displayed=[] ): """Edit library_dataset_dataset_association information, optionally template element information""" - self.visit_url( "%s/library_common/ldda_edit_info?cntrller=%s&library_id=%s&folder_id=%s&id=%s" % \ + self.visit_url( "%s/library_common/ldda_edit_info?cntrller=%s&library_id=%s&folder_id=%s&id=%s" % ( self.url, cntrller, library_id, folder_id, ldda_id ) ) check_str = 'Edit attributes of %s' % ldda_name self.check_page_for_string( check_str ) @@ -1554,13 +1553,13 @@ class TwillTestCase( unittest.TestCase ): # If inheritable is True, the item is currently inheritable. if item_type == 'library': url = "%s/library_common/manage_template_inheritance?cntrller=%s&item_type=%s&library_id=%s" % \ - ( self.url, cntrller, item_type, library_id ) + ( self.url, cntrller, item_type, library_id ) elif item_type == 'folder': url = "%s/library_common/manage_template_inheritance?cntrller=%s&item_type=%s&library_id=%s&folder_id=%s" % \ - ( self.url, cntrller, item_type, library_id, folder_id ) + ( self.url, cntrller, item_type, library_id, folder_id ) elif item_type == 'ldda': url = "%s/library_common/manage_template_inheritance?cntrller=%s&item_type=%s&library_id=%s&folder_id=%s&ldda_id=%s" % \ - ( self.url, cntrller, item_type, library_id, folder_id, ldda_id ) + ( self.url, cntrller, item_type, library_id, folder_id, ldda_id ) self.visit_url( url ) if inheritable: self.check_page_for_string = 'will no longer be inherited to contained folders and datasets' @@ -1868,12 +1867,12 @@ class TwillTestCase( unittest.TestCase ): formcontrols = self.get_form_controls( f ) hc_prefix = ' repository_installation_timeout: - raise AssertionError( 'Repository installation timed out after %d seconds, repository state is %s.' % \ - ( timeout_counter, repository.status ) ) + raise AssertionError( 'Repository installation timed out after %d seconds, repository state is %s.' % + ( timeout_counter, galaxy_repository.status ) ) break time.sleep( 1 ) # Set all metadata on each installed repository. @@ -162,7 +158,7 @@ class InstallTestRepository( TwillTestCase ): log.debug( 'Waited %d seconds for repository %s.' % ( timeout_counter, str( galaxy_repository.name ) ) ) # This timeout currently defaults to 10 minutes. if timeout_counter > repository_installation_timeout: - raise AssertionError( 'Repository installation timed out after %d seconds, repository state is %s.' % \ + raise AssertionError( 'Repository installation timed out after %d seconds, repository state is %s.' % ( timeout_counter, galaxy_repository.status ) ) break time.sleep( 1 ) diff --git a/test/install_and_test_tool_shed_repositories/base/util.py b/test/install_and_test_tool_shed_repositories/base/util.py index 21acb15b8d0..a52e2537039 100644 --- a/test/install_and_test_tool_shed_repositories/base/util.py +++ b/test/install_and_test_tool_shed_repositories/base/util.py @@ -1,5 +1,12 @@ +import json +import logging import os +import platform +import string +import subprocess import sys +import time +from datetime import datetime, timedelta cwd = os.getcwd() sys.path.append( cwd ) @@ -12,45 +19,22 @@ sys.path = new_path from galaxy import eggs eggs.require( 'mercurial' ) eggs.require( "nose" ) +import nose +from mercurial import hg, ui +from nose.plugins import Plugin -import json -import logging +import galaxy.webapps.tool_shed.model.mapping import install_and_test_tool_shed_repositories.base.test_db_util as test_db_util import install_and_test_tool_shed_repositories.functional.test_install_repositories as test_install_repositories -import nose -import platform -import string -import subprocess -import time import tool_shed.repository_types.util as rt_util import tool_shed.util.shed_util_common as suc -import urllib - -from datetime import datetime -from datetime import timedelta - from common import get_api_url from common import get_latest_downloadable_changeset_revision_via_api -from common import get_repository_dict -from common import json_from_url -from common import submit -from common import update - -from galaxy.util import asbool -from galaxy.util import listify -from galaxy.util import unicodify -import galaxy.webapps.tool_shed.model.mapping - -from nose.plugins import Plugin -from tool_shed.util import common_util -from tool_shed.util import hg_util -from tool_shed.util import tool_dependency_util - +from common import get_repository_dict, json_from_url, submit, update +from galaxy.util import asbool, listify, unicodify +from tool_shed.util import common_util, hg_util from tool_shed.util.xml_util import parse_xml -from mercurial import hg -from mercurial import ui - log = logging.getLogger(__name__) # Set up a job_conf.xml that explicitly limits jobs to 10 minutes. @@ -307,6 +291,7 @@ def get_missing_repository_dependencies( repository, all_missing_repository_depe all_missing_repository_dependencies ) ) return all_missing_repository_dependencies + def get_missing_tool_dependencies( repository, all_missing_tool_dependencies=None ): """ Return the entire list of missing tool dependencies for the received repository. The entire @@ -326,6 +311,7 @@ def get_missing_tool_dependencies( repository, all_missing_tool_dependencies=Non all_missing_tool_dependencies ) ) return all_missing_tool_dependencies + def get_repositories_to_install( tool_shed_url, test_framework ): """ Get a list of repository info dicts to install. This method expects a json list of dicts with the following structure: @@ -374,7 +360,7 @@ def get_repositories_to_install( tool_shed_url, test_framework ): if changeset_revision != hg_util.INITIAL_CHANGELOG_HASH: # If testing repositories of type tool_dependency_definition, filter accordingly. if test_framework == TOOL_DEPENDENCY_DEFINITIONS and \ - repository_dict.get( 'type', None ) != rt_util.TOOL_DEPENDENCY_DEFINITION: + repository_dict.get( 'type', None ) != rt_util.TOOL_DEPENDENCY_DEFINITION: continue # Merge the dictionary returned from /api/repository_revisions with the detailed repository_dict and # append it to the list of repository_dicts to install and test. @@ -415,6 +401,7 @@ def get_repositories_to_install( tool_shed_url, test_framework ): ( changeset_revision, name, owner, repository_id, repository_metadata_id ) return repository_dicts, error_message + def get_repository( name, owner, changeset_revision ): """Return a repository record associated with the received name, owner, changeset_revision if one exists.""" repository = None @@ -425,6 +412,7 @@ def get_repository( name, owner, changeset_revision ): pass return repository + def get_repository_current_revision( repo_path ): """This method uses the python mercurial API to get the current working directory's mercurial changeset hash.""" # Initialize a mercurial repo object from the provided path. @@ -439,6 +427,7 @@ def get_repository_current_revision( repo_path ): hg_id = '%d:%s' % ( ctx_rev, str( changectx ) ) return hg_id + def get_repository_dependencies_dicts( tool_shed_url, encoded_repository_metadata_id ): """ Return the list of dictionaries that define all repository dependencies of the repository_metadata @@ -452,6 +441,7 @@ def get_repository_dependencies_dicts( tool_shed_url, encoded_repository_metadat return None, error_message return repository_dependencies_dicts, error_message + def get_repository_tuple_from_elem( elem ): attributes = elem.attrib name = attributes.get( 'name', None ) @@ -459,6 +449,7 @@ def get_repository_tuple_from_elem( elem ): changeset_revision = attributes.get( 'changeset_revision', None ) return ( name, owner, changeset_revision ) + def get_static_settings(): """ Return a dictionary of the settings necessary for a Galaxy application to be wrapped in the static @@ -466,16 +457,17 @@ def get_static_settings(): """ cwd = os.getcwd() static_dir = os.path.join( cwd, 'static' ) - #TODO: these should be copied from galaxy.ini - #TODO: static_enabled needed here? - return dict( static_enabled = True, - static_cache_time = 360, - static_dir = static_dir, - static_images_dir = os.path.join( static_dir, 'images', '' ), - static_favicon_dir = os.path.join( static_dir, 'favicon.ico' ), - static_scripts_dir = os.path.join( static_dir, 'scripts', '' ), - static_style_dir = os.path.join( static_dir, 'june_2007_style', 'blue' ), - static_robots_txt = os.path.join( static_dir, 'robots.txt' ) ) + # TODO: these should be copied from galaxy.ini + # TODO: static_enabled needed here? + return dict( static_enabled=True, + static_cache_time=360, + static_dir=static_dir, + static_images_dir=os.path.join( static_dir, 'images', '' ), + static_favicon_dir=os.path.join( static_dir, 'favicon.ico' ), + static_scripts_dir=os.path.join( static_dir, 'scripts', '' ), + static_style_dir=os.path.join( static_dir, 'june_2007_style', 'blue' ), + static_robots_txt=os.path.join( static_dir, 'robots.txt' ) ) + def get_time_last_tested( tool_shed_url, encoded_repository_metadata_id ): """ @@ -497,6 +489,7 @@ def get_time_last_tested( tool_shed_url, encoded_repository_metadata_id ): error_message = 'The url %s returned the invalid repository_metadata_dict %s' % ( str( api_url ), str( repository_metadata_dict ) ) return None, error_message + def get_tool_test_results_dict( tool_test_results_dicts ): if tool_test_results_dicts: # Inspect the tool_test_results_dict for the last test run to make sure it contains only a test_environment @@ -509,7 +502,7 @@ def get_tool_test_results_dict( tool_test_results_dicts ): # since it will be re-inserted later. tool_test_results_dict = tool_test_results_dicts.pop( 0 ) elif len( tool_test_results_dict ) == 2 and \ - 'test_environment' in tool_test_results_dict and 'missing_test_components' in tool_test_results_dict: + 'test_environment' in tool_test_results_dict and 'missing_test_components' in tool_test_results_dict: # We can re-use tool_test_results_dict if its only entries are "test_environment" and "missing_test_components". # In this case, some tools are missing tests components while others are not. tool_test_results_dict = tool_test_results_dicts.pop( 0 ) @@ -517,10 +510,11 @@ def get_tool_test_results_dict( tool_test_results_dicts ): # The latest tool_test_results_dict has been populated with the results of a test run, so it cannot be used. tool_test_results_dict = {} else: - # Create a new dictionary for this test test run, + # Create a new dictionary for this test test run, tool_test_results_dict = {} return tool_test_results_dict + def get_tool_test_results_dicts( tool_shed_url, encoded_repository_metadata_id ): """ Return the list of dictionaries contained in the Tool Shed's repository_metadata.tool_test_results @@ -541,6 +535,7 @@ def get_tool_test_results_dicts( tool_shed_url, encoded_repository_metadata_id ) error_message = 'The url %s returned the invalid repository_metadata_dict %s' % ( str( api_url ), str( repository_metadata_dict ) ) return None, error_message + def get_webapp_global_conf(): """Return the global_conf dictionary sent as the first argument to app_factory.""" global_conf = {} @@ -548,6 +543,7 @@ def get_webapp_global_conf(): global_conf.update( get_static_settings() ) return global_conf + def initialize_install_and_test_statistics_dict(): # Initialize a dictionary for the summary that will be printed to stdout. install_and_test_statistics_dict = {} @@ -560,6 +556,7 @@ def initialize_install_and_test_statistics_dict(): install_and_test_statistics_dict[ 'at_least_one_test_failed' ] = [] return install_and_test_statistics_dict + def initialize_tool_tests_results_dict( app, tool_test_results_dict ): test_environment_dict = tool_test_results_dict.get( 'test_environment', {} ) if len( test_environment_dict ) == 0: @@ -593,6 +590,7 @@ def initialize_tool_tests_results_dict( app, tool_test_results_dict ): tool_dependencies=[] ) return tool_test_results_dict + def install_repository( app, repository_dict ): """Install a repository defined by the received repository_dict from the tool shed into Galaxy.""" name = str( repository_dict.get( 'name', None ) ) @@ -617,6 +615,7 @@ def install_repository( app, repository_dict ): log.error( error_message ) return repository, error_message + def is_excluded( exclude_list_dicts, name, owner, changeset_revision, encoded_repository_metadata_id ): """ Return True if the repository defined by the received name, owner, changeset_revision should @@ -645,7 +644,7 @@ def is_excluded( exclude_list_dicts, name, owner, changeset_revision, encoded_re rd_owner = repository_dependency_dict.get( 'owner', '' ) rd_changeset_revision = repository_dependency_dict.get( 'changeset_revision', '' ) if ( rd_name, rd_owner, rd_changeset_revision ) in exclude_repositories or \ - ( rd_name, rd_owner, None ) in exclude_repositories: + ( rd_name, rd_owner, None ) in exclude_repositories: print 'Revision %s of repository %s owned by %s is excluded from testing because ' % \ ( changeset_revision, name, owner, reason ) print 'it requires revision %s of repository %s owned by %s (which is excluded from testing).' % \ @@ -656,6 +655,7 @@ def is_excluded( exclude_list_dicts, name, owner, changeset_revision, encoded_re break return False, None + def is_latest_downloadable_revision( url, repository_dict ): """ Return True if the changeset_revision defined in the received repository_dict is the latest @@ -675,6 +675,7 @@ def is_latest_downloadable_revision( url, repository_dict ): is_latest_downloadable = changeset_revision == str( latest_revision ) return is_latest_downloadable, error_message + def parse_exclude_list( xml_filename ): """Return a list of repositories to exclude from testing.""" # This method expects an xml document that looks something like this: @@ -731,6 +732,7 @@ def parse_exclude_list( xml_filename ): print 'The exclude file %s defines no repositories to be excluded from testing.' % xml_filename return exclude_list + def populate_dependency_install_containers( app, repository, repository_identifier_tup, install_and_test_statistics_dict, tool_test_results_dict ): """ @@ -839,6 +841,7 @@ def populate_dependency_install_containers( app, repository, repository_identifi tool_test_results_dict[ 'successful_installations' ][ 'tool_dependencies' ].append( tool_dependency_info_dict ) return params, install_and_test_statistics_dict, tool_test_results_dict + def populate_install_containers_for_repository_dependencies( app, repository, repository_metadata_id, install_and_test_statistics_dict, can_update_tool_shed ): """ @@ -905,7 +908,7 @@ def populate_install_containers_for_repository_dependencies( app, repository, re # This is counter intuitive because the following check is on strings like this: '2014-01-21T19:46:06.953741', # so if "time_last_tested > twenty_hours_ago" is True, then it implies that the time_last_tested column # was actually updated less than 20 hours ago, and should not be updated again because we're likely processing - # another dependent repository, many of which can have the same repository dependency. + # another dependent repository, many of which can have the same repository dependency. try: # Be very conservative here. Our default behavior will be to assume containers have not been populated # during the current test run. @@ -930,8 +933,8 @@ def populate_install_containers_for_repository_dependencies( app, repository, re if len( tool_test_results_dict ) <= 1: tool_test_results_dict = tool_test_results_dicts.pop( 0 ) elif len( tool_test_results_dict ) == 2 and \ - 'test_environment' in tool_test_results_dict and \ - 'missing_test_components' in tool_test_results_dict: + 'test_environment' in tool_test_results_dict and \ + 'missing_test_components' in tool_test_results_dict: tool_test_results_dict = tool_test_results_dicts.pop( 0 ) else: tool_test_results_dict = {} @@ -986,6 +989,7 @@ def populate_install_containers_for_repository_dependencies( app, repository, re print 'tool_shed: %s name: %s owner: %s changeset_revision: %s' % \ ( cleaned_tool_shed_url, name, owner, changeset_revision ) + def populate_shed_conf_file( shed_conf_file, tool_path, xml_elems=None ): """Populate the file defined by shed_conf_file with xml_elems or initialize it with a template string.""" if xml_elems is None: @@ -993,6 +997,7 @@ def populate_shed_conf_file( shed_conf_file, tool_path, xml_elems=None ): xml_elems = tool_conf_template_parser.safe_substitute( shed_tool_path=tool_path ) file( shed_conf_file, 'w' ).write( xml_elems ) + def populate_galaxy_shed_tools_dict_file( galaxy_shed_tools_dict_file, shed_tools_dict=None ): """ Populate the file defined by galaxy_shed_tools_dict_file with the contents of the shed_tools_dict @@ -1002,12 +1007,13 @@ def populate_galaxy_shed_tools_dict_file( galaxy_shed_tools_dict_file, shed_tool shed_tools_dict = {} file( galaxy_shed_tools_dict_file, 'w' ).write( json.dumps( shed_tools_dict ) ) + def print_install_and_test_results( install_stage_type, install_and_test_statistics_dict, error_message ): "Print statistics for the current test run." if error_message: print "Error returned from install_and_test_repositories:" print error_message - elif isinstance ( install_and_test_statistics_dict, dict ): + elif isinstance( install_and_test_statistics_dict, dict ): all_tests_passed = install_and_test_statistics_dict.get( 'all_tests_passed', None ) at_least_one_test_failed = install_and_test_statistics_dict.get( 'at_least_one_test_failed', None ) repositories_with_installation_error = \ @@ -1049,6 +1055,7 @@ def print_install_and_test_results( install_stage_type, install_and_test_statist display_repositories_by_owner( at_least_one_test_failed ) print "####################################################################################" + def remove_protocol_from_tool_shed_url( base_url ): """Eliminate the protocol from the received base_url and return the possibly altered url.""" # The tool_shed value stored in the tool_shed_repository record does not include the protocol, but does @@ -1057,7 +1064,7 @@ def remove_protocol_from_tool_shed_url( base_url ): if base_url.find( '://' ) > -1: try: protocol, base = base_url.split( '://' ) - except ValueError, e: + except ValueError: # The received base_url must be an invalid url. log.debug( "Returning unchanged invalid base_url from remove_protocol_from_tool_shed_url: %s" % str( base_url ) ) return base_url @@ -1066,8 +1073,9 @@ def remove_protocol_from_tool_shed_url( base_url ): log.debug( "Returning base_url from remove_protocol_from_tool_shed_url: %s" % str( base_url ) ) return base_url + def run_tests( test_config ): - ## TODO: replace whole method with... + # TODO: replace whole method with... # from base import nose_util # result = nose_util.run( test_config, plugins=[ new ReportResults() ] ) # return result, test_config.plugins._plugins @@ -1086,6 +1094,7 @@ def run_tests( test_config ): result = test_runner.run( tests ) return result, test_config.plugins._plugins + def save_test_results_for_changeset_revision( url, tool_test_results_dicts, tool_test_results_dict, repository_dict, params, can_update_tool_shed ): """ @@ -1145,5 +1154,5 @@ def save_test_results_for_changeset_revision( url, tool_test_results_dicts, tool ( changeset_revision, name, owner, str( response_from_update ) ) print '\n=============================================================\n' except Exception, e: - log.exception( 'Error updating tool_test_results for repository_metadata id %s:\n%s' % \ + log.exception( 'Error updating tool_test_results for repository_metadata id %s:\n%s' % ( str( metadata_revision_id ), str( e ) ) ) diff --git a/test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py b/test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py index d4d61dee283..7aaff36f3b5 100644 --- a/test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py +++ b/test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py @@ -1,5 +1,6 @@ import logging import new + import install_and_test_tool_shed_repositories.base.test_db_util as test_db_util from install_and_test_tool_shed_repositories.base.twilltestcase import InstallTestRepository @@ -16,11 +17,12 @@ class InstallTestRepositories( InstallTestRepository ): self.login( email=admin_email, username=admin_username ) admin_user = test_db_util.get_user( admin_email ) assert admin_user is not None, 'Problem retrieving user with email %s from the database' % admin_email - admin_user_private_role = test_db_util.get_private_role( admin_user ) - # Install the repository through the web interface using twill. The install_repository() method may + test_db_util.get_private_role( admin_user ) + # Install the repository through the web interface using twill. The install_repository() method may # actually install more than this singe repository because repository dependencies can be installed. self.install_repository( repository_dict ) + def generate_install_method( repository_dict=None ): """Generate abstract test cases for the defined list of repositories.""" diff --git a/test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py b/test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py index 55f76141133..c62e5bb71a8 100644 --- a/test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py @@ -4,8 +4,18 @@ This script cannot be run directly, because it needs to have test/functional/tes order to run functional tests on repository tools after installation. The install_and_test_tool_shed_repositories.sh will execute this script with the appropriate parameters. """ +import httplib +import logging import os +import random +import re +import shutil +import socket import sys +import tempfile +import threading +import time + # Assume we are run from the galaxy root directory, add lib to the python path cwd = os.getcwd() sys.path.append( cwd ) @@ -23,32 +33,17 @@ eggs.require( 'mercurial' ) # This should not be required, but it is under certain conditions thanks to this bug: # http://code.google.com/p/python-nose/issues/detail?id=284 eggs.require( "pysqlite" ) - -import httplib -import install_and_test_tool_shed_repositories.base.test_db_util as test_db_util -import install_and_test_tool_shed_repositories.functional.test_install_repositories as test_install_repositories -import logging import nose -import random -import re -import shutil -import socket -import tempfile -import time -import threading +from paste import httpserver import install_and_test_tool_shed_repositories.base.util as install_and_test_base_util - from base.tool_shed_util import parse_tool_panel_config - +from functional import database_contexts +from functional_tests import generate_config_file from galaxy.app import UniverseApplication from galaxy.util.json import loads from galaxy.util import asbool from galaxy.web import buildapp -from functional_tests import generate_config_file -from paste import httpserver - -from functional import database_contexts log = logging.getLogger( 'install_and_test_repositories_with_tools' ) @@ -75,6 +70,7 @@ else: can_update_tool_shed = True test_framework = install_and_test_base_util.REPOSITORIES_WITH_TOOLS +test_toolbox = None def get_failed_test_dicts( test_result, from_tool_test=True ): @@ -407,7 +403,7 @@ def main(): else: tempdir = tempfile.mkdtemp( dir=galaxy_test_tmp_dir ) galaxy_db_path = os.path.join( tempdir, 'database' ) - # Checks if galaxy_db_path exists, if not create it. + # Checks if galaxy_db_path exists, if not create it. if not os.path.exists(galaxy_db_path): os.makedirs(galaxy_db_path) # Configure the paths Galaxy needs to install and test tools. diff --git a/test/install_and_test_tool_shed_repositories/tool_dependency_definitions/functional_tests.py b/test/install_and_test_tool_shed_repositories/tool_dependency_definitions/functional_tests.py index ead3909b512..4bcc80361d7 100644 --- a/test/install_and_test_tool_shed_repositories/tool_dependency_definitions/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/tool_dependency_definitions/functional_tests.py @@ -4,8 +4,17 @@ This script cannot be run directly, because it needs to have test/functional/tes order to run functional tests on repository tools after installation. The install_and_test_tool_shed_repositories.sh will execute this script with the appropriate parameters. """ +import httplib +import logging import os +import random +import shutil +import socket import sys +import tempfile +import threading +import time + # Assume we are run from the galaxy root directory, add lib to the python path cwd = os.getcwd() sys.path.append( cwd ) @@ -17,34 +26,19 @@ new_path.extend( sys.path ) sys.path = new_path from galaxy import eggs -eggs.require( "nose" ) eggs.require( "Paste" ) eggs.require( 'mercurial' ) # This should not be required, but it is under certain conditions thanks to this bug: # http://code.google.com/p/python-nose/issues/detail?id=284 eggs.require( "pysqlite" ) - -import httplib -import install_and_test_tool_shed_repositories.base.test_db_util as test_db_util -import install_and_test_tool_shed_repositories.functional.test_install_repositories as test_install_repositories -import logging -import nose -import random -import shutil -import socket -import tempfile -import time -import threading +from paste import httpserver import install_and_test_tool_shed_repositories.base.util as install_and_test_base_util - +from functional import database_contexts +from functional_tests import generate_config_file from galaxy.app import UniverseApplication from galaxy.util import asbool from galaxy.web import buildapp -from functional_tests import generate_config_file -from paste import httpserver - -from functional import database_contexts log = logging.getLogger( 'install_and_test_tool_dependency_definitions' )