diff --git a/lib/galaxy/visualization/genomes.py b/lib/galaxy/visualization/genomes.py index 6fd0282e36a..44e36f0ad2c 100644 --- a/lib/galaxy/visualization/genomes.py +++ b/lib/galaxy/visualization/genomes.py @@ -1,9 +1,11 @@ -import os, re, sys, glob +import os, re, sys, glob, logging from bx.seq.twobit import TwoBitFile from galaxy.util.json import from_json_string from galaxy import model from galaxy.util.bunch import Bunch +log = logging.getLogger( __name__ ) + # FIXME: copied from tracks.py # Message strings returned to browser messages = Bunch( @@ -166,13 +168,18 @@ class Genomes( object ): self.genomes[ key ] = Genome( key, len_file=f ) # Add genome data (twobit files) to genomes. - for line in open( os.path.join( app.config.tool_data_path, "twobit.loc" ) ): - if line.startswith("#"): continue - val = line.split() - if len( val ) == 2: - key, path = val - if key in self.genomes: - self.genomes[ key ].twobit_file = path + # FIXME: If a galaxy instance does not have ~/tool-data/twobit.loc file, the following error is thrown: + # IOError: [Errno 2] No such file or directory: '~/tool-data/twobit.loc' + try: + for line in open( os.path.join( app.config.tool_data_path, "twobit.loc" ) ): + if line.startswith("#"): continue + val = line.split() + if len( val ) == 2: + key, path = val + if key in self.genomes: + self.genomes[ key ].twobit_file = path + except IOError, e: + log.exception( str( e ) ) def get_build( self, dbkey ): """ Returns build for the given key. """ diff --git a/test/tool_shed/base/twilltestcase.py b/test/tool_shed/base/twilltestcase.py index 8ff2aa35aad..5e30524c7de 100644 --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -1,6 +1,7 @@ import galaxy.webapps.community.util.hgweb_config import galaxy.model as galaxy_model -import common, string, os, re, test_db_util +import common, string, os, re, test_db_util, simplejson +import galaxy.util as util from base.twilltestcase import tc, from_json_string, TwillTestCase, security, urllib from galaxy.tool_shed.encoding_util import tool_shed_encode @@ -23,8 +24,10 @@ class ShedTwillTestCase( TwillTestCase ): self.galaxy_host = os.environ.get( 'GALAXY_TEST_HOST' ) self.galaxy_port = os.environ.get( 'GALAXY_TEST_PORT' ) self.galaxy_url = "http://%s:%s" % ( self.galaxy_host, self.galaxy_port ) + self.shed_tool_data_table_conf = os.environ.get( 'TOOL_SHED_TEST_TOOL_DATA_TABLE_CONF' ) self.file_dir = os.environ.get( 'TOOL_SHED_TEST_FILE_DIR', None ) self.tool_shed_test_file = None + self.tool_data_path = os.environ.get( 'GALAXY_TEST_TOOL_DATA_PATH' ) self.shed_tools_dict = {} self.home() def browse_category( self, category, strings_displayed=[], strings_not_displayed=[] ): @@ -439,9 +442,14 @@ class ShedTwillTestCase( TwillTestCase ): repo = hg.repository( ui.ui(), self.get_repo_path( repository ) ) tip_ctx = repo.changectx( repo.changelog.tip() ) return tip_ctx.rev() < 0 + def reset_installed_repository_metadata( self, repository ): + url = '/admin_toolshed/reset_repository_metadata?id=%s' % self.security.encode_id( repository.id ) + self.visit_galaxy_url( url ) + self.check_for_strings( [ 'Metadata has been reset' ] ) def reset_repository_metadata( self, repository ): url = '/repository/reset_all_metadata?id=%s' % self.security.encode_id( repository.id ) self.visit_url( url ) + self.check_for_strings( [ 'All repository metadata has been reset.' ] ) def revoke_write_access( self, repository, username ): url = '/repository/manage_repository?user_access_button=Remove&id=%s&remove_auth=%s' % \ ( self.security.encode_id( repository.id ), username ) @@ -466,6 +474,10 @@ class ShedTwillTestCase( TwillTestCase ): def tip_has_metadata( self, repository ): tip = self.get_repository_tip( repository ) return test_db_util.get_repository_metadata_by_repository_id_changeset_revision( repository.id, tip ) + def update_installed_repository( self, installed_repository, strings_displayed=[], strings_not_displayed=[] ): + url = '/admin_toolshed/check_for_updates?id=%s' % self.security.encode_id( installed_repository.id ) + self.visit_galaxy_url( url ) + self.check_for_strings( strings_displayed, strings_not_displayed ) def upload_file( self, repository, filename, @@ -482,6 +494,27 @@ class ShedTwillTestCase( TwillTestCase ): tc.formfile( "1", "file_data", self.get_filename( filename, filepath ) ) tc.submit( "upload_button" ) self.check_for_strings( strings_displayed, strings_not_displayed ) + def verify_installed_repository_metadata_unchanged( self, name, owner ): + installed_repository = test_db_util.get_installed_repository_by_name_owner( name, owner ) + differs = False + metadata = installed_repository.metadata + self.reset_installed_repository_metadata( installed_repository ) + new_metadata = installed_repository.metadata + # This test assumes that the different metadata components will always appear in the same order. If this is ever not + # the case, this test must be updated. + for metadata_key in [ 'datatypes', 'tools', 'tool_dependencies', 'repository_dependencies', 'workflows' ]: + if ( metadata_key in metadata and metadata_key not in new_metadata ) or \ + ( metadata_key not in metadata and metadata_key in new_metadata ): + differs = True + break + elif metadata_key not in metadata and metadata_key not in new_metadata: + continue + else: + if metadata[ metadata_key ] != new_metadata[ metadata_key ]: + differs = True + break + if differs: + raise AssertionError( 'Metadata for installed repository %s differs after metadata reset.' % name ) def verify_installed_repository_on_browse_page( self, installed_repository, strings_displayed=[], strings_not_displayed=[] ): url = '/admin_toolshed/browse_repositories' self.visit_galaxy_url( url ) @@ -491,6 +524,17 @@ class ShedTwillTestCase( TwillTestCase ): installed_repository.tool_shed, installed_repository.installed_changeset_revision ] ) self.check_for_strings( strings_displayed, strings_not_displayed ) + def verify_installed_repository_data_table_entries( self, data_tables=[] ): + data_table = util.parse_xml( self.shed_tool_data_table_conf ) + found = False + for table_elem in data_table.findall( 'table' ): + for data_table in data_tables: + if 'name' in table_elem.attrib and table_elem.attrib[ 'name' ] == data_table: + file_elem = table_elem.find( 'file' ) + assert os.path.exists( file_elem.attrib[ 'path' ] ), 'Tool data table file %s not found.' % file_elem.path + found = True + break + assert found, 'No entry for %s in %s.' % ( data_table, self.shed_tool_data_table_conf ) def verify_tool_metadata_for_installed_repository( self, installed_repository, strings_displayed=[], strings_not_displayed=[] ): repository_id = self.security.encode_id( installed_repository.id ) for tool in installed_repository.metadata[ 'tools' ]: @@ -499,6 +543,11 @@ class ShedTwillTestCase( TwillTestCase ): url = '/admin_toolshed/view_tool_metadata?repository_id=%s&tool_id=%s' % ( repository_id, urllib.quote_plus( tool[ 'id' ] ) ) self.visit_galaxy_url( url ) self.check_for_strings( strings, strings_not_displayed ) + def verify_unchanged_repository_metadata( self, repository ): + self.check_repository_changelog( repository ) + html = self.last_page() + self.reset_repository_metadata( repository ) + self.check_repository_changelog( repository, strings_displayed=[ html ] ) def visit_galaxy_url( self, url ): url = '%s%s' % ( self.galaxy_url, url ) self.visit_url( url ) diff --git a/test/tool_shed/functional/test_0000_basic_repository_features.py b/test/tool_shed/functional/test_0000_basic_repository_features.py index 7f602eb423b..db2789a8f11 100644 --- a/test/tool_shed/functional/test_0000_basic_repository_features.py +++ b/test/tool_shed/functional/test_0000_basic_repository_features.py @@ -155,3 +155,7 @@ class TestBasicRepositoryFeatures( ShedTwillTestCase ): tip_changeset = self.get_repository_tip( repository ) search_fields = dict( tool_id='Filter1', tool_name='filter', tool_version='2.2.0' ) self.search_for_valid_tools( search_fields=search_fields, strings_displayed=[ tip_changeset ], strings_not_displayed=[] ) + def test_0085_verify_repository_metadata( self ): + '''Verify that resetting the metadata does not change it.''' + repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + self.verify_unchanged_repository_metadata( repository ) diff --git a/test/tool_shed/functional/test_0020_basic_repository_dependencies.py b/test/tool_shed/functional/test_0020_basic_repository_dependencies.py index e93ec07ad53..73fa5ab20a7 100644 --- a/test/tool_shed/functional/test_0020_basic_repository_dependencies.py +++ b/test/tool_shed/functional/test_0020_basic_repository_dependencies.py @@ -68,3 +68,9 @@ class TestBasicRepositoryDependencies( ShedTwillTestCase ): changeset_revision = self.get_repository_tip( datatypes_repository ) strings_displayed = [ datatypes_repository_name, common.test_user_1_name, changeset_revision, 'Repository dependencies' ] self.display_manage_repository_page( repository, strings_displayed=strings_displayed ) + def test_0040_verify_repository_metadata( self ): + '''Verify that resetting the metadata does not change it.''' + emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) + datatypes_repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) + self.verify_unchanged_repository_metadata( emboss_repository ) + self.verify_unchanged_repository_metadata( datatypes_repository ) diff --git a/test/tool_shed/functional/test_0030_repository_dependency_revisions.py b/test/tool_shed/functional/test_0030_repository_dependency_revisions.py index 753595cb5d7..2ea481747c0 100644 --- a/test/tool_shed/functional/test_0030_repository_dependency_revisions.py +++ b/test/tool_shed/functional/test_0030_repository_dependency_revisions.py @@ -131,3 +131,11 @@ class TestRepositoryDependencyRevisions( ShedTwillTestCase ): self.display_manage_repository_page( repository, changeset_revision=changeset_revision, strings_displayed=[ str( metadata ) for metadata in repository_dependency_metadata ] ) + def test_0040_verify_repository_metadata( self ): + '''Verify that resetting the metadata does not change it.''' + emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) + emboss_5_repository = test_db_util.get_repository_by_name_and_owner( emboss_5_repository_name, common.test_user_1_name ) + emboss_6_repository = test_db_util.get_repository_by_name_and_owner( emboss_6_repository_name, common.test_user_1_name ) + datatypes_repository = test_db_util.get_repository_by_name_and_owner( datatypes_repository_name, common.test_user_1_name ) + for repository in [ emboss_repository, emboss_5_repository, emboss_6_repository, datatypes_repository ]: + self.verify_unchanged_repository_metadata( repository ) diff --git a/test/tool_shed/functional/test_0040_repository_circular_dependencies.py b/test/tool_shed/functional/test_0040_repository_circular_dependencies.py index 255cafc7585..222670ae754 100644 --- a/test/tool_shed/functional/test_0040_repository_circular_dependencies.py +++ b/test/tool_shed/functional/test_0040_repository_circular_dependencies.py @@ -97,3 +97,9 @@ class TestRepositoryCircularDependencies( ShedTwillTestCase ): # In this case, the displayed dependency will specify the tip revision, but this will not always be the case. self.check_repository_dependency( filtering_repository, freebayes_repository, self.get_repository_tip( freebayes_repository ) ) self.check_repository_dependency( freebayes_repository, filtering_repository, self.get_repository_tip( filtering_repository ) ) + def test_0035_verify_repository_metadata( self ): + '''Verify that resetting the metadata does not change it.''' + freebayes_repository = test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) + filtering_repository = test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) + for repository in [ freebayes_repository, filtering_repository ]: + self.verify_unchanged_repository_metadata( repository ) diff --git a/test/tool_shed/functional/test_0050_circular_n_levels.py b/test/tool_shed/functional/test_0050_circular_n_levels.py index be7b7de4f5e..d682f5bf06c 100644 --- a/test/tool_shed/functional/test_0050_circular_n_levels.py +++ b/test/tool_shed/functional/test_0050_circular_n_levels.py @@ -147,3 +147,11 @@ class TestRepositoryCircularDependenciesToNLevels( ShedTwillTestCase ): for changeset_revision in self.get_repository_metadata_revisions( emboss_repository ): self.check_repository_dependency( freebayes_repository, emboss_repository, changeset_revision ) self.display_manage_repository_page( freebayes_repository, strings_displayed=[ 'Freebayes depends on the filtering repository.' ] ) + def test_0035_verify_repository_metadata( self ): + '''Verify that resetting the metadata does not change it.''' + emboss_datatypes_repository = test_db_util.get_repository_by_name_and_owner( emboss_datatypes_repository_name, common.test_user_1_name ) + emboss_repository = test_db_util.get_repository_by_name_and_owner( emboss_repository_name, common.test_user_1_name ) + filtering_repository = test_db_util.get_repository_by_name_and_owner( filtering_repository_name, common.test_user_1_name ) + freebayes_repository = test_db_util.get_repository_by_name_and_owner( freebayes_repository_name, common.test_user_1_name ) + for repository in [ emboss_datatypes_repository, emboss_repository, freebayes_repository, filtering_repository ]: + self.verify_unchanged_repository_metadata( repository ) diff --git a/test/tool_shed/functional/test_0060_workflows.py b/test/tool_shed/functional/test_0060_workflows.py index 39f0797146a..cd8ef163812 100644 --- a/test/tool_shed/functional/test_0060_workflows.py +++ b/test/tool_shed/functional/test_0060_workflows.py @@ -54,3 +54,7 @@ class TestToolShedWorkflowFeatures( ShedTwillTestCase ): commit_message="Uploaded filtering 2.2.0", remove_repo_files_not_in_tar='No' ) self.load_workflow_image( repository, workflow_name, strings_not_displayed=[ '#EBBCB2' ] ) + def test_0025_verify_repository_metadata( self ): + '''Verify that resetting the metadata does not change it.''' + repository = test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name ) + self.verify_unchanged_repository_metadata( repository ) diff --git a/test/tool_shed/functional/test_1000_install_basic_repository.py b/test/tool_shed/functional/test_1000_install_basic_repository.py index 1d6627aef5d..62d5797de41 100644 --- a/test/tool_shed/functional/test_1000_install_basic_repository.py +++ b/test/tool_shed/functional/test_1000_install_basic_repository.py @@ -28,4 +28,7 @@ class BasicToolShedFeatures( ShedTwillTestCase ): self.verify_installed_repository_on_browse_page( installed_repository ) self.display_installed_repository_manage_page( installed_repository, strings_displayed=[ 'Installed tool shed repository', 'Tools', 'Filter1' ] ) - self.verify_tool_metadata_for_installed_repository( installed_repository ) \ No newline at end of file + self.verify_tool_metadata_for_installed_repository( installed_repository ) + def test_0025_verify_installed_repository_metadata( self ): + '''Verify that resetting the metadata on an installed repository does not change the metadata.''' + self.verify_installed_repository_metadata_unchanged( 'filtering_0000', common.test_user_1_name ) diff --git a/test/tool_shed/functional/test_1010_install_repository_with_tool_dependencies.py b/test/tool_shed/functional/test_1010_install_repository_with_tool_dependencies.py index 8b2e1a206db..e56800ff34e 100644 --- a/test/tool_shed/functional/test_1010_install_repository_with_tool_dependencies.py +++ b/test/tool_shed/functional/test_1010_install_repository_with_tool_dependencies.py @@ -25,3 +25,10 @@ class ToolWithToolDependencies( ShedTwillTestCase ): strings_displayed=[ 'Installed tool shed repository', 'Tools', 'FreeBayes' ] ) self.check_installed_repository_tool_dependencies( installed_repository, dependencies_installed=False ) self.verify_tool_metadata_for_installed_repository( installed_repository ) + def test_0020_verify_installed_repository_metadata( self ): + '''Verify that resetting the metadata on an installed repository does not change the metadata.''' + self.verify_installed_repository_metadata_unchanged( 'freebayes_0010', common.test_user_1_name ) + def test_0025_verify_sample_files( self ): + '''Verify that the installed repository populated shed_tool_data_table.xml and the sample files.''' + self.verify_installed_repository_data_table_entries( data_tables=[ 'sam_fa_indexes' ] ) + diff --git a/test/tool_shed/functional/test_1020_install_repository_with_repository_dependencies.py b/test/tool_shed/functional/test_1020_install_repository_with_repository_dependencies.py index 5f6c470c4a7..75a94ef698c 100644 --- a/test/tool_shed/functional/test_1020_install_repository_with_repository_dependencies.py +++ b/test/tool_shed/functional/test_1020_install_repository_with_repository_dependencies.py @@ -25,3 +25,6 @@ class ToolWithRepositoryDependencies( ShedTwillTestCase ): strings_displayed=[ 'Installed tool shed repository', 'Tools', 'antigenic' ] ) self.check_installed_repository_tool_dependencies( installed_repository, dependencies_installed=False ) self.verify_tool_metadata_for_installed_repository( installed_repository ) + def test_0020_verify_installed_repository_metadata( self ): + '''Verify that resetting the metadata on an installed repository does not change the metadata.''' + self.verify_installed_repository_metadata_unchanged( 'emboss_0020', common.test_user_1_name ) diff --git a/test/tool_shed/functional/test_1030_install_repository_with_dependency_revisions.py b/test/tool_shed/functional/test_1030_install_repository_with_dependency_revisions.py index c84e915fb4b..4f139979de7 100644 --- a/test/tool_shed/functional/test_1030_install_repository_with_dependency_revisions.py +++ b/test/tool_shed/functional/test_1030_install_repository_with_dependency_revisions.py @@ -27,3 +27,7 @@ class RepositoryWithDependencyRevisions( ShedTwillTestCase ): strings_displayed=[ 'Installed tool shed repository', 'Tools', 'antigenic' ] ) self.check_installed_repository_tool_dependencies( installed_repository, dependencies_installed=False ) self.verify_tool_metadata_for_installed_repository( installed_repository ) + self.update_installed_repository( installed_repository, strings_displayed=[ "there are no updates available" ] ) + def test_0025_verify_installed_repository_metadata( self ): + '''Verify that resetting the metadata on an installed repository does not change the metadata.''' + self.verify_installed_repository_metadata_unchanged( 'emboss_0030', common.test_user_1_name ) diff --git a/test/tool_shed/functional_tests.py b/test/tool_shed/functional_tests.py index e2c9d21f86d..4857d279864 100644 --- a/test/tool_shed/functional_tests.py +++ b/test/tool_shed/functional_tests.py @@ -68,6 +68,14 @@ shed_tool_conf_xml_template = ''' ''' +tool_conf_xml = ''' + +
+ +
+
+''' + tool_data_table_conf_xml_template = ''' @@ -113,8 +121,14 @@ def main(): shed_db_path = os.path.join( tempdir, 'database' ) shed_tool_data_table_conf_file = os.environ.get( 'TOOL_SHED_TEST_TOOL_DATA_TABLE_CONF', os.path.join( tool_shed_test_tmp_dir, 'shed_tool_data_table_conf.xml' ) ) galaxy_tool_data_table_conf_file = os.environ.get( 'GALAXY_TEST_TOOL_DATA_TABLE_CONF', os.path.join( tool_shed_test_tmp_dir, 'tool_data_table_conf.xml' ) ) - galaxy_shed_tool_conf_file = os.environ.get( 'GALAXY_TEST_TOOL_CONF', os.path.join( tool_shed_test_tmp_dir, 'test_tool_conf.xml' ) ) + galaxy_tool_conf_file = os.environ.get( 'GALAXY_TEST_TOOL_CONF', os.path.join( tool_shed_test_tmp_dir, 'test_tool_conf.xml' ) ) + galaxy_shed_tool_conf_file = os.environ.get( 'GALAXY_TEST_SHED_TOOL_CONF', os.path.join( tool_shed_test_tmp_dir, 'test_shed_tool_conf.xml' ) ) galaxy_tool_sheds_conf_file = os.environ.get( 'GALAXY_TEST_SHED_TOOLS_CONF', os.path.join( tool_shed_test_tmp_dir, 'test_sheds_conf.xml' ) ) + if 'GALAXY_TEST_TOOL_DATA_PATH' in os.environ: + tool_data_path = os.environ.get( 'GALAXY_TEST_TOOL_DATA_PATH' ) + else: + tool_data_path = tempfile.mkdtemp( dir=tool_shed_test_tmp_dir ) + os.environ[ 'GALAXY_TEST_TOOL_DATA_PATH' ] = tool_data_path if 'GALAXY_TEST_DBPATH' in os.environ: galaxy_db_path = os.environ[ 'GALAXY_TEST_DBPATH' ] else: @@ -153,7 +167,7 @@ def main(): file( galaxy_tool_data_table_conf_file, 'w' ).write( tool_data_table_conf_xml_template ) # Generate the shed_tool_data_table_conf.xml file. file( shed_tool_data_table_conf_file, 'w' ).write( tool_data_table_conf_xml_template ) - + os.environ[ 'TOOL_SHED_TEST_TOOL_DATA_TABLE_CONF' ] = shed_tool_data_table_conf_file # ---- Build Tool Shed Application -------------------------------------------------- toolshedapp = None global_conf = { '__file__' : 'community_wsgi.ini.sample' } @@ -231,6 +245,8 @@ def main(): # ---- Optionally start up a Galaxy instance ------------------------------------------------------ if 'TOOL_SHED_TEST_OMIT_GALAXY' not in os.environ: + # Generate the tool_conf.xml file. + file( galaxy_tool_conf_file, 'w' ).write( tool_conf_xml ) # Generate the shed_tool_conf.xml file. tool_sheds_conf_template_parser = string.Template( tool_sheds_conf_xml_template ) tool_sheds_conf_xml = tool_sheds_conf_template_parser.safe_substitute( shed_url=tool_shed_test_host, shed_port=tool_shed_test_port ) @@ -251,10 +267,11 @@ def main(): database_engine_option_pool_size = '10', file_path = galaxy_file_path, tool_path = tool_path, + tool_data_path = tool_data_path, tool_dependency_dir=galaxy_tool_dependency_dir, shed_tool_path=galaxy_shed_tool_path, update_integrated_tool_panel = False, - tool_config_file = galaxy_shed_tool_conf_file, + tool_config_file = [ galaxy_tool_conf_file, galaxy_shed_tool_conf_file ], tool_sheds_config_file = galaxy_tool_sheds_conf_file, datatype_converters_config_file = "datatype_converters_conf.xml.sample", tool_parse_help = False,