Fixes for tool shed functional tests.

test/tool_shed/base/twilltestcase.py

@@ -1,4 +1,5 @@
 from base.twilltestcase import *
+import ConfigParser
 
 class ShedTwillTestCase( TwillTestCase ):
     def setUp( self ):
@@ -6,6 +7,7 @@ class ShedTwillTestCase( TwillTestCase ):
         self.security = security.SecurityHelper( id_secret='changethisinproductiontoo' )
         self.history_id = None
         self.hgweb_config_manager = None
+        self.hgweb_config_dir = os.environ.get( 'TEST_HG_WEB_CONFIG_DIR' )
         self.host = os.environ.get( 'TOOL_SHED_TEST_HOST' )
         self.port = os.environ.get( 'TOOL_SHED_TEST_PORT' )
         self.url = "http://%s:%s" % ( self.host, self.port )
@@ -108,8 +110,8 @@ class ShedTwillTestCase( TwillTestCase ):
         self.check_for_strings( strings_displayed, strings_not_displayed )
     def delete_files_from_repository( self, repository, filenames=[], strings_displayed=[ 'were deleted from the repository' ], strings_not_displayed=[] ):
         files_to_delete = []
-        basepath = self.get_repository_base_path_from_browse_page( repository )
-        repository_files = self.get_repository_file_list( repository, basepath )
+        basepath = self.get_repo_path( repository )
+        repository_files = self.get_repository_file_list( base_path=basepath, current_path=None )
         # Verify that the files to delete actually exist in the repository.
         for filename in repository_files:
             if filename in filenames:
@@ -133,14 +135,15 @@ class ShedTwillTestCase( TwillTestCase ):
         self.check_for_strings( strings_displayed, strings_not_displayed )
     def display_repository_file_contents( self, repository, filename, filepath=None, strings_displayed=[], strings_not_displayed=[] ):
         '''Find a file in the repository and display the contents.'''
-        basepath = self.get_repository_base_path_from_browse_page( repository )
+        basepath = self.get_repo_path( repository )
         repository_file_list = []
-        relative_path = filename
-        if filepath is not None:
-            relative_path = os.path.join( filepath, filename )
-        repository_file_list = self.get_repository_file_list( repository, basepath )
-        assert relative_path in repository_file_list, 'File %s not found in the repository under %s.' % ( filename, filepath )
-        url = '/repository/get_file_contents?file_path=%s' % os.path.join( basepath, relative_path ).replace( '/', '%2f' )
+        if filepath:
+            relative_path = os.path.join( basepath, filepath )
+        else:
+            relative_path = basepath
+        repository_file_list = self.get_repository_file_list( base_path=relative_path, current_path=None )
+        assert filename in repository_file_list, 'File %s not found in the repository under %s.' % ( filename, relative_path )
+        url = '/repository/get_file_contents?file_path=%s' % os.path.join( relative_path, filename )
         self.visit_url( url )
         self.check_for_strings( strings_displayed, strings_not_displayed )
     def edit_repository_categories( self, repository, categories_to_add=[], categories_to_remove=[], restore_original=True ):
@@ -196,27 +199,27 @@ class ShedTwillTestCase( TwillTestCase ):
     def get_latest_repository_metadata_for_repository( self, repository ):
         # TODO: This will not work as expected. Fix it.
         return repository.metadata_revisions[ 0 ]
-    def get_repository_base_path_from_browse_page( self, repository ):
-        '''Get the base path from the javascript dynatree parameters. This must be updated if the browse feature is altered.'''
-        self.visit_url( '/repository/browse_repository?id=%s' % self.security.encode_id( repository.id ) )
-        html = self.last_page()
-        found_file = None
-        basepath = None
-        search = re.search( 'data: { folder_path: "([^"]+)" },', html )
-        if search is not None:
-            basepath = search.group( 1 )
-        return basepath
-    def get_repository_file_list( self, repository, base_path, current_path=None ):
-        '''
-        Recursively load repository folder contents and append them to a list. Similar to os.walk, 
-        but via /repository/open_folder.
-        '''
+    def get_repo_path( self, repository ):
+        # An entry in the hgweb.config file looks something like: repos/test/mira_assembler = database/community_files/000/repo_123
+        lhs = "repos/%s/%s" % ( repository.user.username, repository.name )
+        hgweb_config = "%s/hgweb.config" %  self.hgweb_config_dir
+        if not os.path.exists( hgweb_config ):
+            raise Exception( "Required file hgweb.config does not exist in directory %s" % self.hgweb_config_dir )
+        config = ConfigParser.ConfigParser()
+        config.read( hgweb_config )
+        for option in config.options( "paths" ):
+            if option == lhs:
+                return config.get( "paths", option )
+        raise Exception( "Entry for repository %s missing in %s/hgweb.config file." % ( lhs, self.hgweb_config_dir ) )
+    def get_repository_file_list( self, base_path, current_path=None ):
+        '''Recursively load repository folder contents and append them to a list. Similar to os.walk but via /repository/open_folder.'''
         if current_path is None:
-            full_path = base_path
+            request_param_path = base_path
         else:
-            full_path = os.path.join( base_path, current_path )
+            request_param_path = os.path.join( base_path, current_path )
+        #request_param_path = request_param_path.replace( '/', '%2f' )
         # Get the current folder's contents.
-        url = '/repository/open_folder?folder_path=%s' % full_path.replace( '/', '%2f' )
+        url = '/repository/open_folder?folder_path=%s' % request_param_path
         self.visit_url( url )
         file_list = from_json_string( self.last_page() )
         returned_file_list = []
@@ -228,10 +231,10 @@ class ShedTwillTestCase( TwillTestCase ):
                 # This is a folder. Get the contents of the folder and append it to the list, 
                 # prefixed with the path relative to the repository root, if any.
                 if current_path is None:
-                    returned_file_list.extend( self.get_repository_file_list( repository, base_path, file_dict[ 'title' ] ) )
+                    returned_file_list.extend( self.get_repository_file_list( base_path=base_path, current_path=file_dict[ 'title' ] ) )
                 else:
                     sub_path = os.path.join( current_path, file_dict[ 'title' ] )
-                    returned_file_list.extend( self.get_repository_file_list( repository, base_path, sub_path ) )
+                    returned_file_list.extend( self.get_repository_file_list( base_path=base_path, current_path=sub_path ) )
             else:
                 # This is a regular file, prefix the filename with the current path and append it to the list.
                 if current_path is not None:

test/tool_shed/functional/test_0000_basic_repository_features.py

@@ -15,7 +15,7 @@ repository_name = 'filtering'
 repository_description = "Galaxy's filtering tool"
 repository_long_description = "Long description of Galaxy's filtering tool"
 
-class TestCreateRepository( ShedTwillTestCase ):
+class TestBasicRepositoryFeatures( ShedTwillTestCase ):
  
     def test_0000_initiate_users( self ):
         """Create necessary user accounts and login as an admin user."""
@@ -89,9 +89,13 @@ class TestCreateRepository( ShedTwillTestCase ):
                                         strings_displayed=[ 'has been marked as not deprecated', 'Mark as deprecated' ],
                                         set_deprecated=False )
     def test_0045_display_repository_tip_file( self ):
-        '''Display the contents of a file in the repository tip revision'''
+        '''Display the contents of filtering.xml in the repository tip revision'''
         repository = get_repository_by_name_and_owner( repository_name, admin_username )
-        self.display_repository_file_contents( repository, filename='filtering.xml', strings_displayed=[ '1.1.0' ] )
+        self.display_repository_file_contents( repository=repository,
+                                               filename='filtering.xml',
+                                               filepath=None,
+                                               strings_displayed=[ '1.1.0' ],
+                                               strings_not_displayed=[] )
     def test_0050_upload_filtering_txt_file( self ):
         '''Upload filtering.txt file associated with tool version 1.1.0.'''
         repository = get_repository_by_name_and_owner( repository_name, admin_username )
@@ -105,7 +109,11 @@ class TestCreateRepository( ShedTwillTestCase ):
         '''Upload filtering test data.'''
         repository = get_repository_by_name_and_owner( repository_name, admin_username )
         self.upload_file( repository, 'filtering_test_data.tar', commit_message="Uploaded filtering test data", remove_repo_files_not_in_tar='No' )
-        self.display_repository_file_contents( repository, filename='1.bed', filepath='test-data' )
+        self.display_repository_file_contents( repository=repository,
+                                               filename='1.bed',
+                                               filepath='test-data',
+                                               strings_displayed=[],
+                                               strings_not_displayed=[] )
         self.check_repository_metadata( repository, tip_only=True )
     def test_0060_upload_filtering_2_2_0( self ):
         '''Upload filtering version 2.2.0'''

test/tool_shed/functional/test_0010_repository_with_tool_dependencies.py

@@ -0,0 +1,19 @@
+from tool_shed.base.twilltestcase import *
+from tool_shed.base.test_db_util import *
+
+admin_user = None
+admin_user_private_role = None
+admin_email = 'test@bx.psu.edu'
+admin_username = 'admin-user'
+
+regular_user = None
+regular_user_private_role = None
+regular_email = 'test-1@bx.psu.edu'
+regular_username = 'user1'
+
+repository_name = 'freebayes'
+repository_description = "Galaxy's freebayes tool"
+repository_long_description = "Long description of Galaxy's freebayes tool"
+
+class TestRepositoryWithToolDependencies( ShedTwillTestCase ):
+    pass

test/tool_shed/functional_tests.py

@@ -6,6 +6,10 @@ import os, sys, shutil, tempfile, re
 cwd = os.getcwd()
 tool_shed_home_directory = os.path.join( cwd, 'test', 'tool_shed' )
 default_tool_shed_test_file_dir = os.path.join( tool_shed_home_directory, 'test_data' )
+# Here's the directory where everything happens.  Temporary directories are created within this directory to contain
+# the hgweb.config file, the database, new repositories, etc.  Since the tool shed browses repository contents via HTTP,
+# the full path to the temporary directroy wher eht repositories are located cannot contain invalid url characters.
+tool_shed_test_tmp_dir = os.path.join( tool_shed_home_directory, 'tmp' )
 new_path = [ os.path.join( cwd, "lib" ) ]
 new_path.extend( sys.path[1:] )
 sys.path = new_path
@@ -31,7 +35,6 @@ from paste import httpserver
 import galaxy.webapps.community.app
 from galaxy.webapps.community.app import UniverseApplication
 from galaxy.webapps.community import buildapp
-#from galaxy.webapps.community.util.hgweb_config import *
 
 import nose.core
 import nose.config
@@ -81,6 +84,8 @@ def main():
     use_distributed_object_store = os.environ.get( 'TOOL_SHED_USE_DISTRIBUTED_OBJECT_STORE', False )
     
     if start_server:
+        if not os.path.isdir( tool_shed_test_tmp_dir ):
+            os.mkdir( tool_shed_test_tmp_dir )
         psu_production = False
         tool_shed_test_proxy_port = None
         if 'TOOL_SHED_TEST_PSU_PRODUCTION' in os.environ:
@@ -102,7 +107,8 @@ def main():
             if not nginx_upload_store:
                 raise Exception( 'Set TOOL_SHED_TEST_NGINX_UPLOAD_STORE to the path where the nginx upload module places uploaded files' )
             file_path = tempfile.mkdtemp( dir=base_file_path )
-            new_file_path = tempfile.mkdtemp( dir=base_new_file_path )
+            new_repos_path = tempfile.mkdtemp( dir=base_new_file_path )
+            hgweb_config_file_path = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir )
             kwargs = dict( database_engine_option_pool_size = '10',
                            database_engine_option_max_overflow = '20',
                            database_engine_option_strategy = 'threadlocal',
@@ -113,16 +119,17 @@ def main():
             if 'TOOL_SHED_TEST_DBPATH' in os.environ:
                 db_path = os.environ[ 'TOOL_SHED_TEST_DBPATH' ]
             else: 
-                tempdir = tempfile.mkdtemp()
+                tempdir = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir )
                 db_path = os.path.join( tempdir, 'database' )
             file_path = os.path.join( db_path, 'files' )
-            new_file_path = os.path.join( db_path, 'tmp' )
+            hgweb_config_file_path = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir )
+            new_repos_path = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir )
             if 'TOOL_SHED_TEST_DBURI' in os.environ:
                 database_connection = os.environ[ 'TOOL_SHED_TEST_DBURI' ]
             else:
                 database_connection = 'sqlite:///' + os.path.join( db_path, 'universe.sqlite' )
             kwargs = {}
-        for dir in file_path, new_file_path:
+        for dir in tool_shed_test_tmp_dir:
             try:
                 os.makedirs( dir )
             except OSError:
@@ -130,6 +137,11 @@ def main():
 
     print "Database connection:", database_connection
 
+    hgweb_config_dir = hgweb_config_file_path
+    os.environ[ 'TEST_HG_WEB_CONFIG_DIR' ] = hgweb_config_dir
+
+    print "Directory location for hgweb.config:", hgweb_config_dir
+
     # ---- Build Application -------------------------------------------------- 
     app = None 
     if start_server:
@@ -150,7 +162,7 @@ def main():
                                    database_connection = database_connection,
                                    database_engine_option_pool_size = '10',
                                    file_path = file_path,
-                                   new_file_path = new_file_path,
+                                   new_file_path = new_repos_path,
                                    tool_path=tool_path,
                                    datatype_converters_config_file = 'datatype_converters_conf.xml.sample',
                                    tool_parse_help = False,
@@ -163,9 +175,8 @@ def main():
                                    admin_users = 'test@bx.psu.edu',
                                    global_conf = global_conf,
                                    running_functional_tests = True,
-                                   hgweb_config_dir = new_file_path,
+                                   hgweb_config_dir = hgweb_config_dir,
                                    **kwargs )
-#        app.hgweb_config_manager = HgWebConfigManager()
 
         log.info( "Embedded Universe application started" )
 
@@ -254,20 +265,14 @@ def main():
         app.shutdown()
         app = None
         log.info( "Embedded Universe application stopped" )
-    try:
-        if os.path.exists( tempdir ) and 'TOOL_SHED_TEST_NO_CLEANUP' not in os.environ:
-            log.info( "Cleaning up temporary files in %s" % tempdir )
-            shutil.rmtree( tempdir )
-    except:
-        pass
-    if psu_production and 'TOOL_SHED_TEST_NO_CLEANUP' not in os.environ:
-        for dir in ( file_path, new_file_path ):
-            try:
+    if 'TOOL_SHED_TEST_NO_CLEANUP' not in os.environ:
+        try:
+            for dir in [ tool_shed_test_tmp_dir ]:
                 if os.path.exists( dir ):
-                    log.info( 'Cleaning up temporary files in %s' % dir )
+                    log.info( "Cleaning up temporary files in %s" % dir )
                     shutil.rmtree( dir )
-            except:
-                pass
+        except:
+            pass
     if success:
         return 0
     else: