diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 8b21664decb..32076d0ee4e 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -18,6 +18,7 @@ class UniverseApplication( object ): """Encapsulates the state of a Universe application""" def __init__( self, **kwargs ): print >> sys.stderr, "python path is: " + ", ".join( sys.path ) + self.new_installation = False # Read config file and check for errors self.config = config.Configuration( **kwargs ) self.config.check() @@ -27,9 +28,13 @@ class UniverseApplication( object ): db_url = self.config.database_connection else: db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database - # Initialize database / check for appropriate schema version + # Initialize database / check for appropriate schema version. # If this + # is a new installation, we'll restrict the tool migration messaging. from galaxy.model.migrate.check import create_or_verify_database - create_or_verify_database( db_url, kwargs.get( 'global_conf', {} ).get( '__file__', None ), self.config.database_engine_options ) + create_or_verify_database( db_url, kwargs.get( 'global_conf', {} ).get( '__file__', None ), self.config.database_engine_options, app=self ) + # Alert the Galaxy admin to tools that have been moved from the distribution to the tool shed. + from galaxy.tool_shed.migrate.check import verify_tools + verify_tools( self, db_url, kwargs.get( 'global_conf', {} ).get( '__file__', None ), self.config.database_engine_options ) # Object store manager self.object_store = build_object_store_from_config(self.config) # Setup the database engine and ORM @@ -65,15 +70,13 @@ class UniverseApplication( object ): self.tag_handler = GalaxyTagHandler() # Tool data tables self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( self.config.tool_data_table_config_path ) - # Initialize the tools - self.toolbox = tools.ToolBox( self.config.tool_configs, self.config.tool_path, self ) + # Initialize the tools, making sure the list of tool configs includes the reserved migrated_tools_conf.xml file. + tool_configs = self.config.tool_configs + if self.config.migrated_tools_config not in tool_configs: + tool_configs.append( self.config.migrated_tools_config ) + self.toolbox = tools.ToolBox( tool_configs, self.config.tool_path, self ) # Search support for tools self.toolbox_search = galaxy.tools.search.ToolBoxSearch( self.toolbox ) - # If enabled, check for tools missing from the distribution because they - # have been moved to the tool shed and install all such discovered tools. - if self.config.get_bool( 'enable_tool_shed_install', False ): - from tool_shed import install_manager - self.install_manager = install_manager.InstallManager( self, self.config.tool_shed_install_config, self.config.install_tool_config ) # If enabled, poll respective tool sheds to see if updates are available for any installed tool shed repositories. if self.config.get_bool( 'enable_tool_shed_check', False ): from tool_shed import update_manager diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 4473e561fbf..4f36d41f9a2 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -52,9 +52,9 @@ class Configuration( object ): self.tool_data_path = resolve_path( kwargs.get( "tool_data_path", "tool-data" ), os.getcwd() ) self.len_file_path = kwargs.get( "len_file_path", resolve_path(os.path.join(self.tool_data_path, 'shared','ucsc','chrom'), self.root) ) self.test_conf = resolve_path( kwargs.get( "test_conf", "" ), self.root ) - self.enable_tool_shed_install = string_as_bool( kwargs.get( 'enable_tool_shed_install', False ) ) - self.tool_shed_install_config = resolve_path( kwargs.get( "tool_shed_install_config_file", "tool_shed_install.xml" ), self.root ) - self.install_tool_config = resolve_path( kwargs.get( "install_tool_config_file", "shed_tool_conf.xml" ), self.root ) + # The value of migrated_tools_config is the file reserved for containing only those tools that have been eliminated from the distribution + # and moved to the tool shed. + self.migrated_tools_config = resolve_path( "migrated_tools_conf.xml", self.root ) if 'tool_config_file' in kwargs: tcf = kwargs[ 'tool_config_file' ] elif 'tool_config_files' in kwargs: @@ -210,7 +210,10 @@ class Configuration( object ): def check( self ): paths_to_check = [ self.root, self.tool_path, self.tool_data_path, self.template_path ] # Look for any tool shed configs and retrieve the tool_path attribute from the tag. - for config_filename in self.tool_configs: + tool_configs = self.tool_configs + if self.migrated_tools_config not in tool_configs: + tool_configs.append( self.migrated_tools_config ) + for config_filename in tool_configs: tree = parse_xml( config_filename ) root = tree.getroot() tool_path = root.get( 'tool_path' ) @@ -241,10 +244,12 @@ class Configuration( object ): os.makedirs( path ) except Exception, e: raise ConfigurationError( "Unable to create missing directory: %s\n%s" % ( path, e ) ) - # Check that required files exist - for path in self.tool_configs: - if not os.path.isfile(path): + tool_configs = self.tool_configs + if self.migrated_tools_config not in tool_configs: + tool_configs.append( self.migrated_tools_config ) + for path in tool_configs: + if not os.path.isfile( path ): raise ConfigurationError("File not found: %s" % path ) if not os.path.isfile( self.datatypes_config ): raise ConfigurationError("File not found: %s" % self.datatypes_config ) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 0b16d1cc96c..1bbe01aea94 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2682,10 +2682,10 @@ class ToolShedRepository( object ): self.dist_to_shed = dist_to_shed @property def includes_tools( self ): - return 'tools' in self.metadata + return self.metadata and 'tools' in self.metadata @property def includes_workflows( self ): - return 'workflows' in self.metadata + return self.metadata and 'workflows' in self.metadata class ToolVersion( object ): def __init__( self, id=None, create_time=None, tool_id=None, tool_shed_repository=None ): @@ -2746,6 +2746,12 @@ class ToolVersionAssociation( object ): self.tool_id = tool_id self.parent_id = parent_id +class MigrateTools( object ): + def __init__( self, repository_id=None, repository_path=None, version=None ): + self.repository_id = repository_id + self.repository_path = repository_path + self.version = version + ## ---- Utility methods ------------------------------------------------------- def directory_hash_id( id ): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 6d41ab4044d..6661d0cc4c5 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -394,6 +394,11 @@ ToolVersionAssociation.table = Table( "tool_version_association", metadata, Column( "tool_id", Integer, ForeignKey( "tool_version.id" ), index=True, nullable=False ), Column( "parent_id", Integer, ForeignKey( "tool_version.id" ), index=True, nullable=False ) ) +MigrateTools.table = Table( "migrate_tools", metadata, + Column( "repository_id", TrimmedString( 255 ) ), + Column( "repository_path", TEXT ), + Column( "version", Integer ) ) + Job.table = Table( "job", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), diff --git a/lib/galaxy/model/migrate/check.py b/lib/galaxy/model/migrate/check.py index e699e6ddf9d..482ddf03bd0 100644 --- a/lib/galaxy/model/migrate/check.py +++ b/lib/galaxy/model/migrate/check.py @@ -1,7 +1,6 @@ import sys, os.path, logging from galaxy import eggs - import pkg_resources pkg_resources.require( "sqlalchemy-migrate" ) @@ -20,7 +19,7 @@ dialect_to_egg = { "mysql" : "MySQL_python" } -def create_or_verify_database( url, galaxy_config_file, engine_options={} ): +def create_or_verify_database( url, galaxy_config_file, engine_options={}, app=None ): """ Check that the database is use-able, possibly creating it if empty (this is the only time we automatically create tables, otherwise we force the @@ -30,9 +29,7 @@ def create_or_verify_database( url, galaxy_config_file, engine_options={} ): 2) Database older than migration support --> fail and require manual update 3) Database at state where migrate support introduced --> add version control information but make no changes (might still require manual update) 4) Database versioned but out of date --> fail with informative message, user must run "sh manage_db.sh upgrade" - """ - dialect = ( url.split( ':', 1 ) )[0] try: egg = dialect_to_egg[dialect] @@ -45,7 +42,6 @@ def create_or_verify_database( url, galaxy_config_file, engine_options={} ): except KeyError: # Let this go, it could possibly work with db's we don't support log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) - # Create engine and metadata engine = create_engine( url, **engine_options ) meta = MetaData( bind=engine ) @@ -53,8 +49,10 @@ def create_or_verify_database( url, galaxy_config_file, engine_options={} ): try: dataset_table = Table( "dataset", meta, autoload=True ) except NoSuchTableError: - # No 'dataset' table means a completely uninitialized database, which - # is fine, init the database in a versioned state + # No 'dataset' table means a completely uninitialized database. If we have an app, we'll + # set it's new_installation setting to True so the tool migration process will be skipped. + if app: + app.new_installation = True log.info( "No database, initializing" ) # Database might or might not be versioned try: diff --git a/lib/galaxy/model/migrate/versions/0092_add_migrate_tools_table.py b/lib/galaxy/model/migrate/versions/0092_add_migrate_tools_table.py new file mode 100644 index 00000000000..9f99806b2ca --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0092_add_migrate_tools_table.py @@ -0,0 +1,49 @@ +""" +Migration script to create the migrate_tools table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import datetime +now = datetime.datetime.utcnow +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +MigrateTools_table = Table( "migrate_tools", metadata, + Column( "repository_id", TrimmedString( 255 ) ), + Column( "repository_path", TEXT ), + Column( "version", Integer ) ) + +def upgrade(): + print __doc__ + + metadata.reflect() + # Create the table. + try: + MigrateTools_table.create() + cmd = "INSERT INTO migrate_tools VALUES ('GalaxyTools', 'lib/galaxy/tool_shed/migrate', %d)" % 1 + db_session.execute( cmd ) + except Exception, e: + log.debug( "Creating migrate_tools table failed: %s" % str( e ) ) + +def downgrade(): + metadata.reflect() + try: + MigrateTools_table.drop() + except Exception, e: + log.debug( "Dropping migrate_tools table failed: %s" % str( e ) ) diff --git a/lib/galaxy/tool_shed/__init__.py b/lib/galaxy/tool_shed/__init__.py index 32304807b3b..0bee4a97b27 100644 --- a/lib/galaxy/tool_shed/__init__.py +++ b/lib/galaxy/tool_shed/__init__.py @@ -2,7 +2,7 @@ Classes encapsulating the management of repositories installed from Galaxy tool sheds. """ import os, logging -from galaxy.util.shed_util import create_repository_dict_for_proprietary_datatypes, load_datatype_items +from galaxy.util.shed_util import * from galaxy.model.orm import * log = logging.getLogger(__name__) diff --git a/lib/galaxy/tool_shed/install_manager.py b/lib/galaxy/tool_shed/install_manager.py index 7857480ec39..f475ffc73da 100644 --- a/lib/galaxy/tool_shed/install_manager.py +++ b/lib/galaxy/tool_shed/install_manager.py @@ -4,77 +4,197 @@ at some point included in the Galaxy distribution, but are now hosted in the mai shed. Tools included in tool_shed_install.xml that have already been installed will not be re-installed. """ -import urllib2 +import logging, urllib2 from galaxy.tools import ToolSection from galaxy.util.json import from_json_string, to_json_string from galaxy.util.shed_util import * - log = logging.getLogger( __name__ ) class InstallManager( object ): - def __init__( self, app, tool_shed_install_config, install_tool_config ): + def __init__( self, app, latest_migration_script_number, tool_shed_install_config, migrated_tools_config ): """ - Check tool settings in tool_shed_install_config and install all tools that are - not already installed. The tool panel configuration file is the received - shed_tool_config, which defaults to shed_tool_conf.xml. + Check tool settings in tool_shed_install_config and install all tools that are not already installed. The tool panel + configuration file is the received migrated_tools_config, which is the reserved file named migrated_tools_conf.xml. """ self.app = app - self.sa_session = self.app.model.context.current - self.install_tool_config = install_tool_config - # Parse shed_tool_config to get the install location (tool_path). - tree = util.parse_xml( install_tool_config ) + self.toolbox = self.app.toolbox + self.migrated_tools_config = migrated_tools_config + # Get the local non-shed related tool panel config (the default name is tool_conf.xml). If the user has more than 1 + # non-shed tool panel config it could cause problems. + self.proprietary_tool_conf = self.non_shed_tool_panel_config + self.proprietary_tool_panel_elems = self.get_proprietary_tool_panel_elems( latest_migration_script_number ) + # Set the location where the repositories will be installed by retrieving the tool_path setting from migrated_tools_config. + tree = util.parse_xml( migrated_tools_config ) root = tree.getroot() self.tool_path = root.get( 'tool_path' ) - # Keep an in-memory list of xml elements to enable persistence of the changing tool config. - config_elems = [] + print "Repositories will be installed into configured tool_path location ", str( self.tool_path ) # Parse tool_shed_install_config to check each of the tools. - log.debug( "Parsing tool shed install configuration %s" % tool_shed_install_config ) self.tool_shed_install_config = tool_shed_install_config tree = util.parse_xml( tool_shed_install_config ) root = tree.getroot() self.tool_shed = clean_tool_shed_url( root.get( 'name' ) ) - log.debug( "Repositories will be installed from tool shed '%s' into configured tool_path location '%s'" % ( str( self.tool_shed ), str( self.tool_path ) ) ) self.repository_owner = 'devteam' + for repository_elem in root: + self.install_repository( repository_elem ) + def get_guid( self, repository_clone_url, relative_install_dir, tool_config ): + found = False + for root, dirs, files in os.walk( relative_install_dir ): + if root.find( '.hg' ) < 0 and root.find( 'hgrc' ) < 0: + if '.hg' in dirs: + dirs.remove( '.hg' ) + for name in files: + if name == tool_config: + found = True + break + if found: + break + full_path = os.path.abspath( os.path.join( root, name ) ) + tool = self.toolbox.load_tool( full_path ) + return generate_tool_guid( repository_clone_url, tool ) + def get_proprietary_tool_panel_elems( self, latest_tool_migration_script_number ): + # Parse self.proprietary_tool_conf (the default is tool_conf.xml) and generate a list of Elements that are either ToolSection elements + # or Tool elements. These will be used to generate new entries in the migrated_tools_conf.xml file for the installed tools. + tools_xml_file_path = os.path.abspath( os.path.join( 'scripts', 'migrate_tools', '%04d_tools.xml' % latest_tool_migration_script_number ) ) + # Parse the XML and load the file attributes for later checking against self.proprietary_tool_conf. + migrated_tool_configs = [] + tree = util.parse_xml( tools_xml_file_path ) + root = tree.getroot() for elem in root: - config_elems.append( elem ) if elem.tag == 'repository': - self.install_repository( elem ) + for tool_elem in elem: + migrated_tool_configs.append( tool_elem.get( 'file' ) ) + # Parse self.proprietary_tool_conf and generate the list of tool panel Elements that contain them. + tool_panel_elems = [] + tree = util.parse_xml( self.proprietary_tool_conf ) + root = tree.getroot() + for elem in root: + if elem.tag == 'tool': + # Tools outside of sections. + file_path = elem.get( 'file', None ) + if file_path: + path, name = os.path.split( file_path ) + if name in migrated_tool_configs: + if elem not in tool_panel_elems: + tool_panel_elems.append( elem ) elif elem.tag == 'section': - self.install_section( elem ) - shed_tool_conf_dict = dict( config_filename=install_tool_config, - tool_path=self.tool_path, - config_elems=config_elems ) - self.app.toolbox.shed_tool_confs.append( shed_tool_conf_dict ) - def install_repository( self, elem, section_name='', section_id='' ): - # Install a single repository into the tool config. If outside of any sections, the entry looks something like: - # - # - # - name = elem.get( 'name' ) - description = elem.get( 'description' ) - changeset_revision = elem.get( 'changeset_revision' ) + # Tools contained in a section. + for section_elem in elem: + if section_elem.tag == 'tool': + file_path = section_elem.get( 'file', None ) + if file_path: + path, name = os.path.split( file_path ) + if name in migrated_tool_configs: + # Append the section, not the tool. + if elem not in tool_panel_elems: + tool_panel_elems.append( elem ) + return tool_panel_elems + def get_containing_tool_section( self, tool_config ): + """ + If tool_config is defined somewhere in self.proprietary_tool_panel_elems, return True and the ToolSection in which the tool is + displayed or None if it is displayed outside of any sections. + """ + for proprietary_tool_panel_elem in self.proprietary_tool_panel_elems: + if proprietary_tool_panel_elem.tag == 'tool': + # The proprietary_tool_panel_elem looks something like . + proprietary_tool_config = proprietary_tool_panel_elem.get( 'file' ) + proprietary_path, proprietary_name = os.path.split( proprietary_tool_config ) + if tool_config == proprietary_name: + # The tool is loaded outside of any sections. + return True, None + if proprietary_tool_panel_elem.tag == 'section': + # The proprietary_tool_panel_elem looks something like
. + for section_elem in proprietary_tool_panel_elem: + if section_elem.tag == 'tool': + # The section_elem looks something like . + proprietary_tool_config = section_elem.get( 'file' ) + proprietary_path, proprietary_name = os.path.split( proprietary_tool_config ) + if tool_config == proprietary_name: + # The tool is loaded inside of the section_elem. + return True, ToolSection( proprietary_tool_panel_elem ) + return False, None + def handle_repository_contents( self, current_working_dir, repository_clone_url, relative_install_dir, repository_elem, repository_name, description, + changeset_revision, tmp_name ): + # Generate the metadata for the installed tool shed repository, among other things. It is critical that the installed repository is + # updated to the desired changeset_revision before metadata is set because the process for setting metadata uses the repository files on disk. + tool_panel_dict = {} + for tool_elem in repository_elem: + # The tool_elem looks something like this: + tool_config = tool_elem.get( 'file' ) + # See if tool_config is defined somewhere in self.proprietary_tool_panel_elems. + is_loaded, tool_section = self.get_containing_tool_section( tool_config ) + if is_loaded: + guid = self.get_guid( repository_clone_url, relative_install_dir, tool_config ) + tool_panel_dict_for_tool_config = generate_tool_panel_dict_for_tool_config( guid, tool_config, tool_section=tool_section ) + # { : { tool_config : , id: , version : , name : }} + for k, v in tool_panel_dict_for_tool_config.items(): + tool_panel_dict[ k ] = v + metadata_dict = generate_metadata( self.toolbox, relative_install_dir, repository_clone_url, tool_panel_dict=tool_panel_dict ) + # Add a new record to the tool_shed_repository table if one doesn't already exist. If one exists but is marked + # deleted, undelete it. It is critical that this happens before the call to add_to_tool_panel() below because + # tools will not be properly loaded if the repository is marked deleted. + print "Adding new row (or updating an existing row) for repository '%s' in the tool_shed_repository table." % repository_name + tool_shed_repository = create_or_update_tool_shed_repository( self.app, + repository_name, + description, + changeset_revision, + repository_clone_url, + metadata_dict, + dist_to_shed=True ) + if 'tools' in metadata_dict: + repository_tools_tups = get_repository_tools_tups( self.app, metadata_dict ) + if repository_tools_tups: + sample_files = metadata_dict.get( 'sample_files', [] ) + # Handle missing data table entries for tool parameters that are dynamically generated select lists. + repository_tools_tups = handle_missing_data_table_entry( self.app, self.tool_path, sample_files, repository_tools_tups ) + # Handle missing index files for tool parameters that are dynamically generated select lists. + repository_tools_tups = handle_missing_index_file( self.app, self.tool_path, sample_files, repository_tools_tups ) + # Handle tools that use fabric scripts to install dependencies. + handle_tool_dependencies( current_working_dir, relative_install_dir, repository_tools_tups ) + add_to_tool_panel( self.app, + repository_name, + repository_clone_url, + changeset_revision, + repository_tools_tups, + self.repository_owner, + self.migrated_tools_config, + tool_panel_dict, + new_install=True ) + # Remove the temporary file + try: + os.unlink( tmp_name ) + except: + pass + if 'datatypes_config' in metadata_dict: + datatypes_config = os.path.abspath( metadata_dict[ 'datatypes_config' ] ) + # Load proprietary data types required by tools. The value of override is not important here since the Galaxy server will be started + # after this installation completes. + converter_path, display_path = alter_config_and_load_prorietary_datatypes( self.app, datatypes_config, relative_install_dir, override=False ) + if converter_path or display_path: + # Create a dictionary of tool shed repository related information. + repository_dict = create_repository_dict_for_proprietary_datatypes( tool_shed=self.tool_shed, + name=repository_name, + owner=self.repository_owner, + installed_changeset_revision=changeset_revision, + tool_dicts=metadata_dict.get( 'tools', [] ), + converter_path=converter_path, + display_path=display_path ) + if converter_path: + # Load proprietary datatype converters + self.app.datatypes_registry.load_datatype_converters( self.toolbox, installed_repository_dict=repository_dict ) + if display_path: + # Load proprietary datatype display applications + self.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) + return tool_shed_repository, metadata_dict + def install_repository( self, repository_elem ): + # Install a single repository, loading contained tools into the tool config. + name = repository_elem.get( 'name' ) + description = repository_elem.get( 'description' ) + changeset_revision = repository_elem.get( 'changeset_revision' ) # Install path is of the form: //repos/// clone_dir = os.path.join( self.tool_path, self.tool_shed, 'repos', self.repository_owner, name, changeset_revision ) if self.__isinstalled( clone_dir ): - log.debug( "Skipping automatic install of repository '%s' because it has already been installed in location '%s'" % ( name, clone_dir ) ) + print "Skipping automatic install of repository '", name, "' because it has already been installed in location ", clone_dir else: - if section_name and section_id: - section_key = 'section_%s' % str( section_id ) - if section_key in self.app.toolbox.tool_panel: - # Appending a tool to an existing section in self.app.toolbox.tool_panel - log.debug( "Appending to tool panel section: %s" % section_name ) - tool_section = self.app.toolbox.tool_panel[ section_key ] - else: - # Appending a new section to self.app.toolbox.tool_panel - log.debug( "Loading new tool panel section: %s" % section_name ) - new_section_elem = Element( 'section' ) - new_section_elem.attrib[ 'name' ] = section_name - new_section_elem.attrib[ 'id' ] = section_id - new_section_elem.attrib[ 'version' ] = '' - tool_section = ToolSection( new_section_elem ) - self.app.toolbox.tool_panel[ section_key ] = tool_section - else: - tool_section = None current_working_dir = os.getcwd() tool_shed_url = self.__get_url_from_tool_shed( self.tool_shed ) repository_clone_url = os.path.join( tool_shed_url, 'repos', self.repository_owner, name ) @@ -83,25 +203,18 @@ class InstallManager( object ): if returncode == 0: returncode, tmp_name = update_repository( current_working_dir, relative_install_dir, changeset_revision ) if returncode == 0: - tool_shed_repository, metadata_dict = load_repository_contents( app=self.app, - repository_name=name, - description=description, - owner=self.repository_owner, - changeset_revision=changeset_revision, - tool_path=self.tool_path, - repository_clone_url=repository_clone_url, - relative_install_dir=relative_install_dir, - current_working_dir=current_working_dir, - tmp_name=tmp_name, - tool_shed=self.tool_shed, - tool_section=tool_section, - shed_tool_conf=self.install_tool_config, - new_install=True, - dist_to_shed=True ) + tool_shed_repository, metadata_dict = self.handle_repository_contents( current_working_dir, + repository_clone_url, + relative_install_dir, + repository_elem, + name, + description, + changeset_revision, + tmp_name ) if 'tools' in metadata_dict: # Get the tool_versions from the tool shed for each tool in the installed change set. url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ - ( tool_shed_url, name, self.repository_owner, changeset_revision ) + ( tool_shed_url, tool_shed_repository.name, self.repository_owner, changeset_revision ) response = urllib2.urlopen( url ) text = response.read() response.close() @@ -121,13 +234,13 @@ class InstallManager( object ): if not tool_version_using_old_id: tool_version_using_old_id = self.app.model.ToolVersion( tool_id=old_tool_id, tool_shed_repository=tool_shed_repository ) - self.sa_session.add( tool_version_using_old_id ) - self.sa_session.flush() + self.app.sa_session.add( tool_version_using_old_id ) + self.app.sa_session.flush() if not tool_version_using_guid: tool_version_using_guid = self.app.model.ToolVersion( tool_id=tool_id, tool_shed_repository=tool_shed_repository ) - self.sa_session.add( tool_version_using_guid ) - self.sa_session.flush() + self.app.sa_session.add( tool_version_using_guid ) + self.app.sa_session.flush() # Associate the two versions as parent / child. tool_version_association = get_tool_version_association( self.app, tool_version_using_old_id, @@ -135,28 +248,29 @@ class InstallManager( object ): if not tool_version_association: tool_version_association = self.app.model.ToolVersionAssociation( tool_id=tool_version_using_guid.id, parent_id=tool_version_using_old_id.id ) - self.sa_session.add( tool_version_association ) - self.sa_session.flush() + self.app.sa_session.add( tool_version_association ) + self.app.sa_session.flush() else: tmp_stderr = open( tmp_name, 'rb' ) - log.debug( "Error updating repository '%s': %s" % ( name, tmp_stderr.read() ) ) + print "Error updating repository ', name, "': ', str( tmp_stderr.read() ) tmp_stderr.close() else: tmp_stderr = open( tmp_name, 'rb' ) - log.debug( "Error cloning repository '%s': %s" % ( name, tmp_stderr.read() ) ) + print "Error cloning repository '", name, "': ", str( tmp_stderr.read() ) tmp_stderr.close() - def install_section( self, elem ): - # Install 1 or more repositories into a section in the tool config. An entry looks something like: - #
- # - # - # ... - # - #
- section_name = elem.get( 'name' ) - section_id = elem.get( 'id' ) - for repository_elem in elem: - self.install_repository( repository_elem, section_name=section_name, section_id=section_id ) + @property + def non_shed_tool_panel_config( self ): + # Get the non-shed related tool panel config value from the Galaxy config - the default is tool_conf.xml. + for config_filename in self.app.config.tool_configs: + # Any config file that includes a tool_path attribute in the root tag set like the following is shed-related. + # + tree = util.parse_xml( config_filename ) + root = tree.getroot() + tool_path = root.get( 'tool_path', None ) + if tool_path is None: + # There will be a problem here if the user has defined 2 non-shed related configs. + return config_filename + return None def __get_url_from_tool_shed( self, tool_shed ): # The value of tool_shed is something like: toolshed.g2.bx.psu.edu # We need the URL to this tool shed, which is something like: diff --git a/lib/galaxy/tool_shed/migrate/__init__.py b/lib/galaxy/tool_shed/migrate/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/galaxy/tool_shed/migrate/check.py b/lib/galaxy/tool_shed/migrate/check.py new file mode 100644 index 00000000000..fbb573e90a0 --- /dev/null +++ b/lib/galaxy/tool_shed/migrate/check.py @@ -0,0 +1,119 @@ +import sys, os, logging, subprocess +from galaxy import eggs +import pkg_resources +pkg_resources.require( "sqlalchemy-migrate" ) + +from migrate.versioning import repository, schema +from sqlalchemy import * +from common import * + +log = logging.getLogger( __name__ ) + +# Path relative to galaxy +migrate_repository_directory = os.path.dirname( __file__ ).replace( os.getcwd() + os.path.sep, '', 1 ) +migrate_repository = repository.Repository( migrate_repository_directory ) +dialect_to_egg = { + "sqlite" : "pysqlite>=2", + "postgres" : "psycopg2", + "mysql" : "MySQL_python" +} + +def verify_tools( app, url, galaxy_config_file, engine_options={} ): + # Check the value in the migrate_tools.version database table column to verify that the number is in + # sync with the number of version scripts in ~/lib/galaxy/tools/migrate/versions. + dialect = ( url.split( ':', 1 ) )[0] + try: + egg = dialect_to_egg[ dialect ] + try: + pkg_resources.require( egg ) + log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) + except: + # If the module is in the path elsewhere (i.e. non-egg), it'll still load. + log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) + except KeyError: + # Let this go, it could possibly work with db's we don't support + log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) + # Create engine and metadata + engine = create_engine( url, **engine_options ) + meta = MetaData( bind=engine ) + # The migrate_tools table was created in database version script 0092_add_migrate_tools_table.py. + version_table = Table( "migrate_tools", meta, autoload=True ) + # Verify that the code and the database are in sync. + db_schema = schema.ControlledSchema( engine, migrate_repository ) + latest_tool_migration_script_number = migrate_repository.versions.latest + if latest_tool_migration_script_number != db_schema.version: + if app.new_installation: + # New installations will not be missing tools, so we don't need to worry about them. + missing_tool_configs = [] + else: + tool_panel_config = get_non_shed_tool_panel_config( app ) + if tool_panel_config is None: + missing_tool_configs = [] + else: + missing_tool_configs = check_for_missing_tools( tool_panel_config, latest_tool_migration_script_number ) + config_arg = '' + if os.path.abspath( os.path.join( os.getcwd(), 'universe_wsgi.ini' ) ) != galaxy_config_file: + config_arg = ' -c %s' % galaxy_config_file.replace( os.path.abspath( os.getcwd() ), '.' ) + # Automatically update the value of the migrate_tools.version database table column. + cmd = 'sh manage_tools.sh%s upgrade' % config_arg + proc = subprocess.Popen( args=cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) + return_code = proc.wait() + output = proc.stdout.read( 32768 ) + if return_code != 0: + raise Exception( "Error attempting to update the value of migrate_tools.version: %s" % output ) + elif missing_tool_configs: + msg = "\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + msg += "\n\nThe list of files at the end of this message refer to tools that are configured to load into the tool panel for\n" + msg += "this Galaxy instance, but have been removed from the Galaxy distribution. These tools can be automatically installed\n" + msg += "from the Galaxy tool shed at http://toolshed.g2.bx.psu.edu.\n\n" + msg += "To skip this process, attempt to start your Galaxy server again (e.g., sh run.sh or whatever you use). If you do this,\n" + msg += "be aware that these tools will no longer be available in your Galaxy tool panel, and entries for each of them should\n" + msg += "be removed from your file named %s.\n\n" % tool_panel_config + msg += "CRITICAL NOTE IF YOU PLAN TO INSTALL\n" + msg += "The location in which the tool repositories will be installed is the value of the 'tool_path' attribute in the \n" + msg += 'tag of the file named ./migrated_tool_conf.xml (i.e., ). The default location\n' + msg += "setting is '../shed_tools', which may be problematic for some cluster environments, so make sure to change it before\n" + msg += "you execute the installation process if appropriate. The configured location must be outside of the Galaxy installation\n" + msg += "directory or it must be in a sub-directory protected by a properly configured .hgignore file if the directory is within\n" + msg += "the Galaxy installation directory hierarchy. This is because tool shed repositories will be installed using mercurial's\n" + msg += "clone feature, which creates .hg directories and associated mercurial repository files. Not having .hgignore properly\n" + msg += "configured could result in undesired behavior when modifying or updating your local Galaxy instance or the tool shed\n" + msg += "repositories if they are in directories that pose conflicts. See mercurial's .hgignore documentation at the following\n" + msg += "URL for details.\n\nhttp://mercurial.selenic.com/wiki/.hgignore\n\n" + msg += output + msg += "After the installation process finishes, you can start your Galaxy server. As part of this installation process,\n" + msg += "entries for each of the following tool config files will be added to the file named ./migrated_tool_conf.xml, so these\n" + msg += "tools will continue to be loaded into your tool panel. Because of this, existing entries for these files should be\n" + msg += "removed from your file named %s, but only after the installation process finishes.\n\n" % tool_panel_config + for i, missing_tool_config in enumerate( missing_tool_configs ): + msg += "%s\n" % missing_tool_config + # Should we do the following? + #if i > 10: + # msg += "\n...and %d more tools...\n" % ( len( missing_tool_configs ) - ( i + 1 ) ) + # break + msg += "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n" + raise Exception( msg ) + else: + log.info( "At migrate_tools version %d" % db_schema.version ) + +def migrate_to_current_version( engine, schema ): + # Changes to get to current version. + changeset = schema.changeset( None ) + for ver, change in changeset: + nextver = ver + changeset.step + log.info( 'Installing tools from version %s -> %s... ' % ( ver, nextver ) ) + old_stdout = sys.stdout + class FakeStdout( object ): + def __init__( self ): + self.buffer = [] + def write( self, s ): + self.buffer.append( s ) + def flush( self ): + pass + sys.stdout = FakeStdout() + try: + schema.runchange( ver, change, changeset.step ) + finally: + for message in "".join( sys.stdout.buffer ).split( "\n" ): + log.info( message ) + sys.stdout = old_stdout diff --git a/lib/galaxy/tool_shed/migrate/common.py b/lib/galaxy/tool_shed/migrate/common.py new file mode 100644 index 00000000000..da4b21212eb --- /dev/null +++ b/lib/galaxy/tool_shed/migrate/common.py @@ -0,0 +1,112 @@ +import sys, os, ConfigParser +import galaxy.config +import galaxy.datatypes.registry +from galaxy import util, tools +import galaxy.model.mapping +import galaxy.tools.search +from galaxy.objectstore import build_object_store_from_config +import galaxy.tool_shed.tool_shed_registry +from galaxy.tool_shed import install_manager +from galaxy.tool_shed.migrate.common import * + +def check_for_missing_tools( tool_panel_config, latest_tool_migration_script_number ): + # Get the 000x_tools.xml file associated with the current migrate_tools version number. + tools_xml_file_path = os.path.abspath( os.path.join( 'scripts', 'migrate_tools', '%04d_tools.xml' % latest_tool_migration_script_number ) ) + # Parse the XML and load the file attributes for later checking against the proprietary tool_panel_config. + migrated_tool_configs = [] + tree = util.parse_xml( tools_xml_file_path ) + root = tree.getroot() + for elem in root: + if elem.tag == 'repository': + for tool_elem in elem.findall( 'tool' ): + migrated_tool_configs.append( tool_elem.get( 'file' ) ) + # Parse the proprietary tool_panel_config (the default is tool_conf.xml) and generate the list of missing tool config file names. + missing_tool_configs = [] + tree = util.parse_xml( tool_panel_config ) + root = tree.getroot() + for elem in root: + if elem.tag == 'tool': + missing_tool_configs = check_tool_tag_set( elem, migrated_tool_configs, missing_tool_configs ) + elif elem.tag == 'section': + for section_elem in elem: + if section_elem.tag == 'tool': + missing_tool_configs = check_tool_tag_set( section_elem, migrated_tool_configs, missing_tool_configs ) + return missing_tool_configs +def check_tool_tag_set( elem, migrated_tool_configs, missing_tool_configs ): + file_path = elem.get( 'file', None ) + if file_path: + path, name = os.path.split( file_path ) + if name in migrated_tool_configs: + missing_tool_configs.append( name ) + return missing_tool_configs +def get_non_shed_tool_panel_config( app ): + # Get the non-shed related tool panel config value from the Galaxy config - the default is tool_conf.xml. + for config_filename in app.config.tool_configs: + # Any config file that includes a tool_path attribute in the root tag set like the following is shed-related. + # + tree = util.parse_xml( config_filename ) + root = tree.getroot() + tool_path = root.get( 'tool_path', None ) + if tool_path is None: + # There will be a problem here if the user has defined 2 non-shed related configs. + return config_filename + return None +class MigrateToolsApplication( object ): + """Encapsulates the state of a basic Galaxy Universe application in order to initiate the Install Manager""" + def __init__( self, tools_migration_config ): + galaxy_config_file = 'universe_wsgi.ini' + if '-c' in sys.argv: + pos = sys.argv.index( '-c' ) + sys.argv.pop( pos ) + galaxy_config_file = sys.argv.pop( pos ) + if not os.path.exists( galaxy_config_file ): + print "Galaxy config file does not exist (hint: use '-c config.ini' for non-standard locations): %s" % galaxy_config_file + sys.exit( 1 ) + config_parser = ConfigParser.ConfigParser( { 'here':os.getcwd() } ) + config_parser.read( galaxy_config_file ) + galaxy_config_dict = {} + for key, value in config_parser.items( "app:main" ): + galaxy_config_dict[ key ] = value + self.config = galaxy.config.Configuration( **galaxy_config_dict ) + if self.config.database_connection is None: + self.config.database_connection = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database + self.object_store = build_object_store_from_config( self.config ) + # Setup the database engine and ORM + self.model = galaxy.model.mapping.init( self.config.file_path, + self.config.database_connection, + engine_options={}, + create_tables=False, + object_store=self.object_store ) + # Create an empty datatypes registry. + self.datatypes_registry = galaxy.datatypes.registry.Registry() + # Load the data types in the Galaxy distribution, which are defined in self.config.datatypes_config. + self.datatypes_registry.load_datatypes( self.config.root, self.config.datatypes_config ) + # Initialize the tools, making sure the list of tool configs includes the reserved migrated_tools_conf.xml file. + tool_configs = self.config.tool_configs + if self.config.migrated_tools_config not in tool_configs: + tool_configs.append( self.config.migrated_tools_config ) + self.toolbox = tools.ToolBox( tool_configs, self.config.tool_path, self ) + # Search support for tools + self.toolbox_search = galaxy.tools.search.ToolBoxSearch( self.toolbox ) + # Set up the tool sheds registry. + if os.path.isfile( self.config.tool_sheds_config ): + self.tool_shed_registry = galaxy.tool_shed.tool_shed_registry.Registry( self.config.root, self.config.tool_sheds_config ) + else: + self.tool_shed_registry = None + # Get the latest tool migration script number to send to the Install manager. + latest_migration_script_number = int( tools_migration_config.split( '_' )[ 0 ] ) + # The value of migrated_tools_config is migrated_tools_conf.xml, and is reserved for containing only those tools that have been + # eliminated from the distribution and moved to the tool shed. A side-effect of instantiating the InstallManager is the automatic + # installation of all appropriate tool shed repositories. + self.install_manager = install_manager.InstallManager( app=self, + latest_migration_script_number=latest_migration_script_number, + tool_shed_install_config=os.path.join( self.config.root, + 'scripts', + 'migrate_tools', + tools_migration_config ), + migrated_tools_config=self.config.migrated_tools_config ) + @property + def sa_session( self ): + return self.model.context.current + def shutdown( self ): + self.object_store.shutdown() diff --git a/lib/galaxy/tool_shed/migrate/migrate.cfg b/lib/galaxy/tool_shed/migrate/migrate.cfg new file mode 100644 index 00000000000..3240536e20b --- /dev/null +++ b/lib/galaxy/tool_shed/migrate/migrate.cfg @@ -0,0 +1,20 @@ +[db_settings] +# Used to identify which repository this database is versioned under. +# You can use the name of your project. +repository_id=GalaxyTools + +# The name of the database table used to track the schema version. +# This name shouldn't already be used by your project. +# If this is changed once a database is under version control, you'll need to +# change the table name in each database too. +version_table=migrate_tools + +# When committing a change script, Migrate will attempt to generate the +# sql for all supported databases; normally, if one of them fails - probably +# because you don't have that database installed - it is ignored and the +# commit continues, perhaps ending successfully. +# Databases in this list MUST compile successfully during a commit, or the +# entire commit will fail. List the databases your application will actually +# be using to ensure your updates to that database work properly. +# This must be a list; example: ['postgres','sqlite'] +required_dbs=[] \ No newline at end of file diff --git a/lib/galaxy/tool_shed/migrate/versions/0001_tools.py b/lib/galaxy/tool_shed/migrate/versions/0001_tools.py new file mode 100644 index 00000000000..10ff908ba6e --- /dev/null +++ b/lib/galaxy/tool_shed/migrate/versions/0001_tools.py @@ -0,0 +1,9 @@ +""" +Initialize the version column of the migrate_tools database table to 1. No tool migrations are handled in this version. +""" +import sys + +def upgrade(): + print __doc__ +def downgrade(): + pass diff --git a/lib/galaxy/tool_shed/update_manager.py b/lib/galaxy/tool_shed/update_manager.py index f1d75471d4a..eeba4f2de12 100644 --- a/lib/galaxy/tool_shed/update_manager.py +++ b/lib/galaxy/tool_shed/update_manager.py @@ -9,15 +9,9 @@ log = logging.getLogger( __name__ ) class UpdateManager( object ): def __init__( self, app ): - """ - Check tool settings in tool_shed_install_config and install all tools that are - not already installed. The tool panel configuration file is the received - shed_tool_config, which defaults to shed_tool_conf.xml. - """ self.app = app self.sa_session = self.app.model.context.current - # Ideally only one Galaxy server process - # should be able to check for repository updates. + # Ideally only one Galaxy server process should be able to check for repository updates. self.running = True self.sleeper = Sleeper() self.restarter = threading.Thread( target=self.__restarter ) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index f330e3f2947..77b2626a950 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -30,6 +30,7 @@ from galaxy.datatypes import sniff from cgi import FieldStorage from galaxy.util.hash_util import * from galaxy.util import listify +from galaxy.util.shed_util import * from galaxy.visualization.tracks.visual_analytics import TracksterConfig @@ -178,38 +179,31 @@ class ToolBox( object ): metadata = tool_shed_repository.metadata update_needed = False if 'tool_panel_section' in metadata: + if panel_entry_per_tool( metadata[ 'tool_panel_section' ] ): + # { : { tool_config : , id: , version : , name : }} + tool_panel_dict = metadata[ 'tool_panel_section' ] + else: + # { id: , version : , name : } + tool_section_dict = metadata[ 'tool_panel_section' ] + tool_section = generate_tool_section_element_from_dict( tool_section_dict ) + tool_panel_dict = generate_tool_panel_dict_for_repository_tools( metadata, tool_section=tool_section ) if section: - # If the tool_panel_section dictionary is included in the metadata, update it if necessary. - tool_panel_section = metadata[ 'tool_panel_section' ] - if tool_panel_section [ 'id' ] != section.id or \ - tool_panel_section [ 'version' ] != section.version or \ - tool_panel_section [ 'name' ] != section.name: - tool_panel_section [ 'id' ] = section.id - tool_panel_section [ 'version' ] = section.version - tool_panel_section [ 'name' ] = section.name - update_needed = True + # This means all tools are loaded into the same tool panel section or are all outside of any sections. + for guid, tool_section_dict in tool_panel_dict.items(): + if tool_section_dict [ 'id' ] != section.id or \ + tool_section_dict [ 'version' ] != section.version or \ + tool_section_dict [ 'name' ] != section.name: + tool_section_dict [ 'id' ] = section.id + tool_section_dict [ 'version' ] = section.version + tool_section_dict [ 'name' ] = section.name + tool_panel_dict[ guid ] = tool_section_dict + update_needed = True else: # The tool_panel_section was introduced late, so set it's value if its missing in the metadata. - if section: - if section.id is None: - section_id = '' - else: - section_id = section.id - if section.version is None: - section_version = '' - else: - section_version = section.version - if section.name is None: - section_name = '' - else: - section_name = section.name - tool_panel_section = dict( id=section_id, version=section_version, name=section_name ) - update_needed = True - else: - tool_panel_section = dict( id='', version='', name='' ) - update_needed = True + tool_panel_dict = generate_tool_panel_dict_for_repository_tools( tool_shed_repository.metadata, tool_section=section ) + update_needed = True if update_needed: - metadata[ 'tool_panel_section' ] = tool_panel_section + metadata[ 'tool_panel_section' ] = tool_panel_dict tool_shed_repository.metadata = metadata self.sa_session.add( tool_shed_repository ) self.sa_session.flush() diff --git a/lib/galaxy/util/shed_util.py b/lib/galaxy/util/shed_util.py index 8c75bc8f68e..520a63db09e 100644 --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -12,6 +12,111 @@ from elementtree.ElementTree import Element, SubElement log = logging.getLogger( __name__ ) +def add_to_tool_panel( app, repository_name, repository_clone_url, changeset_revision, repository_tools_tups, owner, shed_tool_conf, tool_panel_dict, new_install=True ): + """A tool shed repository is being installed or updated so handle tool panel alterations accordingly.""" + # We need to change the in-memory version and the file system version of the shed_tool_conf file. + index, shed_tool_conf_dict = get_shed_tool_conf_dict( app, shed_tool_conf ) + tool_path = shed_tool_conf_dict[ 'tool_path' ] + config_elems = shed_tool_conf_dict[ 'config_elems' ] + # Generate the list of ElementTree Element objects for each section or list of tools. + elem_list = generate_tool_panel_elem_list( repository_name, repository_clone_url, changeset_revision, tool_panel_dict, repository_tools_tups, owner=owner ) + # Load the tools into the tool panel outside of any sections. + for config_elem in elem_list: + if config_elem.tag == 'section': + app.toolbox.load_section_tag_set( config_elem, app.toolbox.tool_panel, tool_path ) + elif config_elem.tag == 'tool': + guid = config_elem.get( 'guid' ) + app.toolbox.load_tool_tag_set( config_elem, app.toolbox.tool_panel, tool_path=tool_path, guid=guid ) + if new_install: + # Add the elements to the in-memory list of config_elems. + for elem_entry in elem_list: + config_elems.append( elem_entry ) + # Replace the old list of config_elems with the new list. + shed_tool_conf_dict[ 'config_elems' ] = config_elems + app.toolbox.shed_tool_confs[ index ] = shed_tool_conf_dict + # Append the new entry (either section or list of tools) to the shed_tool_config file. + config_elems_to_xml_file( app, shed_tool_conf_dict ) + if app.toolbox_search.enabled: + # If search support for tools is enabled, index the new installed tools. + app.toolbox_search = ToolBoxSearch( app.toolbox ) +def alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir, deactivate=False, override=True ): + """ + Parse a proprietary datatypes config (a datatypes_conf.xml file included in an installed tool shed repository) and + add information to appropriate element attributes that will enable proprietary datatype class modules, datatypes converters + and display applications to be discovered and properly imported by the datatypes registry. The value of override will + be False when a tool shed repository is being installed. Since installation is occurring after the datatypes registry + has been initialized, the registry's contents cannot be overridden by conflicting data types. + """ + tree = util.parse_xml( datatypes_config ) + datatypes_config_root = tree.getroot() + # Path to datatype converters + converter_path = None + # Path to datatype display applications + display_path = None + relative_path_to_datatype_file_name = None + datatype_files = datatypes_config_root.find( 'datatype_files' ) + datatype_class_modules = [] + if datatype_files: + # The tag set contains any number of tags. + # + # + # + # + # We'll add attributes to the datatype tag sets so that the modules can be properly imported by the datatypes registry. + for elem in datatype_files.findall( 'datatype_file' ): + datatype_file_name = elem.get( 'name', None ) + if datatype_file_name: + # Find the file in the installed repository. + for root, dirs, files in os.walk( relative_install_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name == datatype_file_name: + datatype_class_modules.append( os.path.join( root, name ) ) + break + break + if datatype_class_modules: + registration = datatypes_config_root.find( 'registration' ) + converter_path, display_path = get_converter_and_display_paths( registration, relative_install_dir ) + if converter_path: + registration.attrib[ 'proprietary_converter_path' ] = converter_path + if display_path: + registration.attrib[ 'proprietary_display_path' ] = display_path + for relative_path_to_datatype_file_name in datatype_class_modules: + relative_head, relative_tail = os.path.split( relative_path_to_datatype_file_name ) + for elem in registration.findall( 'datatype' ): + # Handle 'type' attribute which should be something like one of the following: + # type="gmap:GmapDB" + # type="galaxy.datatypes.gmap:GmapDB" + dtype = elem.get( 'type', None ) + if dtype: + fields = dtype.split( ':' ) + proprietary_datatype_module = fields[ 0 ] + if proprietary_datatype_module.find( '.' ) >= 0: + # Handle the case where datatype_module is "galaxy.datatypes.gmap". + proprietary_datatype_module = proprietary_datatype_module.split( '.' )[ -1 ] + # The value of proprietary_path must be an absolute path due to job_working_directory. + elem.attrib[ 'proprietary_path' ] = os.path.abspath( relative_head ) + elem.attrib[ 'proprietary_datatype_module' ] = proprietary_datatype_module + + sniffers = datatypes_config_root.find( 'sniffers' ) + fd, proprietary_datatypes_config = tempfile.mkstemp() + os.write( fd, '\n' ) + os.write( fd, '\n' ) + os.write( fd, '%s' % util.xml_to_string( registration ) ) + os.write( fd, '%s' % util.xml_to_string( sniffers ) ) + os.write( fd, '\n' ) + os.close( fd ) + os.chmod( proprietary_datatypes_config, 0644 ) + else: + proprietary_datatypes_config = datatypes_config + # Load proprietary datatypes + app.datatypes_registry.load_datatypes( root_dir=app.config.root, config=proprietary_datatypes_config, deactivate=deactivate, override=override ) + if datatype_files: + try: + os.unlink( proprietary_datatypes_config ) + except: + pass + return converter_path, display_path def config_elems_to_xml_file( app, shed_tool_conf_dict ): # Persist the current in-memory list of config_elems in the received shed_tool_conf_dict # to a file named by the value of config_filename in the received shed_tool_conf_dict. @@ -80,10 +185,8 @@ def create_repository_dict_for_proprietary_datatypes( tool_shed, name, owner, in converter_path=converter_path, display_path=display_path ) def create_or_update_tool_shed_repository( app, name, description, changeset_revision, repository_clone_url, metadata_dict, owner='', dist_to_shed=False ): - # This method is used by the InstallManager, which does not have access to trans. The - # received value for dist_to_shed will be True if the InstallManager is installing a repository - # that contains tools or datatypes that used to be in the Galaxy distribution, but have been - # moved to the main Galaxy tool shed. + # The received value for dist_to_shed will be True if the InstallManager is installing a repository that contains tools or datatypes that used + # to be in the Galaxy distribution, but have been moved to the main Galaxy tool shed. sa_session = app.model.context.current tmp_url = clean_repository_clone_url( repository_clone_url ) tool_shed = tmp_url.split( 'repos' )[ 0 ].rstrip( '/' ) @@ -111,13 +214,12 @@ def create_or_update_tool_shed_repository( app, name, description, changeset_rev sa_session.add( tool_shed_repository ) sa_session.flush() return tool_shed_repository +def generate_clone_url( trans, repository ): + """Generate the URL for cloning a repository.""" + tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) + return '%s/repos/%s/%s' % ( tool_shed_url, repository.owner, repository.name ) def generate_datatypes_metadata( datatypes_config, metadata_dict ): - """ - Update the received metadata_dict with changes that have been applied - to the received datatypes_config. This method is used by the InstallManager, - which does not have access to trans. - """ - # Parse datatypes_config. + """Update the received metadata_dict with changes that have been applied to the received datatypes_config.""" tree = ElementTree.parse( datatypes_config ) root = tree.getroot() ElementInclude.include( root ) @@ -156,23 +258,33 @@ def generate_datatypes_metadata( datatypes_config, metadata_dict ): if datatypes: metadata_dict[ 'datatypes' ] = datatypes return metadata_dict -def generate_metadata( toolbox, relative_install_dir, repository_clone_url, tool_section_dict=None ): +def generate_metadata( toolbox, relative_install_dir, repository_clone_url, tool_section_dict=None, tool_panel_dict=None ): """ - Browse the repository files on disk to generate metadata. Since we are using disk files, it - is imperative that the repository is updated to the desired change set revision before metadata - is generated. This method is used by the InstallManager, which does not have access to trans. + Browse the repository files on disk to generate metadata. Since we are using disk files, it is imperative that the + repository is updated to the desired change set revision before metadata is generated. """ metadata_dict = {} sample_files = [] datatypes_config = None - # Keep track of the section in the tool panel in which this repository's tools will be contained. + new_tool_panel_dict = {} + # Keep track of the section in the tool panel in which this repository's tools will be contained by using the information in either the + # tool_section_dict or the tool_panel_dict (at least 1 of these 2 dictionaries should be None). The tool_section_dict is passed when the + # Admin is manually installing a repository into a single selected section. It looks something like this. + # { id: , version : , name : } + # The tool_panel_dict is fully populated with all tools in the repository that should be loaded into the tool panel. It is received when + # this method is called by the InstallManager or when metadata is being reset on an existing repository. This dictionary looks something + # like this. + # { : { tool_config : , id: , version : , name : }} + # The tool_panel_dict enables each tool in the repository to be contained inside or outside a specified ToolSection in the tool panel. The + # new_tool_panel_dict with be populated with a subset of the items in the received tool_panel_dict. This will allow for the case where + # repository metadata is being updated where the previous change set revision included more tools than the current change set revision. if tool_section_dict: + # The received tool_panel_dict must be None, so we'll populate it here. Currently tools must all be installed in a single + # section or outside any sections. for k, v in tool_section_dict.items(): if v is None: + # Coerce None values into empty strings because ElementTree.tostring() throws exceptions on None values. tool_section_dict[ k ] = '' - metadata_dict[ 'tool_panel_section' ] = tool_section_dict - else: - metadata_dict[ 'tool_panel_section' ] = dict( id='', version='', name='' ) # Find datatypes_conf.xml if it exists. for root, dirs, files in os.walk( relative_install_dir ): if root.find( '.hg' ) < 0: @@ -207,7 +319,28 @@ def generate_metadata( toolbox, relative_install_dir, repository_clone_url, tool tool = None if tool is not None: tool_config = os.path.join( root, name ) + guid = generate_tool_guid( repository_clone_url, tool ) metadata_dict = generate_tool_metadata( tool_config, tool, repository_clone_url, metadata_dict ) + # Populate the tool_section_dict if necessary. + if tool_panel_dict and guid in tool_panel_dict: + # We're updating metadata on a previously installed repository. + old_tool_panel_dict = tool_panel_dict[ guid ] + if 'tool_config' not in old_tool_panel_dict or old_tool_panel_dict[ 'tool_config' ] in [ None, '' ]: + old_tool_panel_dict[ 'tool_config' ] = name + new_tool_panel_dict[ guid ] = old_tool_panel_dict + else: + # The admin is manually installing a new repository. + new_tool_section_dict = {} + if tool_section_dict: + for k, v in tool_section_dict.items(): + new_tool_section_dict[ k ] = v + else: + new_tool_section_dict[ 'id' ] = '' + new_tool_section_dict[ 'name' ] = '' + new_tool_section_dict[ 'version' ] = '' + if 'tool_config' not in new_tool_section_dict or new_tool_section_dict[ 'tool_config' ] in [ None, '' ]: + new_tool_section_dict[ 'tool_config' ] = name + new_tool_panel_dict[ guid ] = new_tool_section_dict # Find all exported workflows elif name.endswith( '.ga' ): relative_path = os.path.join( root, name ) @@ -217,6 +350,7 @@ def generate_metadata( toolbox, relative_install_dir, repository_clone_url, tool exported_workflow_dict = from_json_string( workflow_text ) if 'a_galaxy_workflow' in exported_workflow_dict and exported_workflow_dict[ 'a_galaxy_workflow' ] == 'true': metadata_dict = generate_workflow_metadata( relative_path, exported_workflow_dict, metadata_dict ) + metadata_dict[ 'tool_panel_section' ] = new_tool_panel_dict return metadata_dict def generate_tool_guid( repository_clone_url, tool ): """ @@ -227,11 +361,7 @@ def generate_tool_guid( repository_clone_url, tool ): tmp_url = clean_repository_clone_url( repository_clone_url ) return '%s/%s/%s' % ( tmp_url, tool.id, tool.version ) def generate_tool_metadata( tool_config, tool, repository_clone_url, metadata_dict ): - """ - Update the received metadata_dict with changes that have been - applied to the received tool. This method is used by the InstallManager, - which does not have access to trans. - """ + """Update the received metadata_dict with changes that have been applied to the received tool.""" # Generate the guid guid = generate_tool_guid( repository_clone_url, tool ) # Handle tool.requirements. @@ -276,31 +406,38 @@ def generate_tool_metadata( tool_config, tool, repository_clone_url, metadata_di else: metadata_dict[ 'tools' ] = [ tool_dict ] return metadata_dict -def generate_tool_panel_elem_list( repository_name, repository_clone_url, changeset_revision, repository_tools_tups, tool_section=None, owner='' ): - """Generate a list of ElementTree Element objects for each section or list of tools.""" +def generate_tool_panel_elem_list( repository_name, repository_clone_url, changeset_revision, tool_panel_dict, repository_tools_tups, owner='' ): + """Generate a list of ElementTree Element objects for each section or tool.""" + # Each entry in the tool_panel_dict looks like this. + # { : { tool_config : , id: , version : , name : }} elem_list = [] tool_elem = None tmp_url = clean_repository_clone_url( repository_clone_url ) if not owner: owner = get_repository_owner( tmp_url ) - if tool_section: - root_elem = Element( 'section' ) - if tool_section.name is None: - root_elem.attrib[ 'name' ] = '' - else: - root_elem.attrib[ 'name' ] = tool_section.name - if tool_section.id is None: - root_elem.attrib[ 'id' ] = '' - else: - root_elem.attrib[ 'id' ] = tool_section.id - if tool_section.version is None: - root_elem.attrib[ 'version' ] = '' - else: - root_elem.attrib[ 'version' ] = tool_section.version - for repository_tool_tup in repository_tools_tups: - tool_file_path, guid, tool = repository_tool_tup - if tool_section: - tool_elem = SubElement( root_elem, 'tool' ) + for guid, tool_section_dict in tool_panel_dict.items(): + tool_section = None + inside_section = False + section_in_elem_list = False + if tool_section_dict[ 'id' ]: + inside_section = True + # Create a new section element only if we haven't already created it. + for index, elem in enumerate( elem_list ): + if elem.tag == 'section': + section_id = elem.get( 'id', None ) + if section_id == tool_section_dict[ 'id' ]: + section_in_elem_list = True + tool_section = elem + break + if tool_section is None: + tool_section = generate_tool_section_element_from_dict( tool_section_dict ) + # Find the tuple containing the current guid from the list of repository_tools_tups. + for repository_tool_tup in repository_tools_tups: + tool_file_path, tup_guid, tool = repository_tool_tup + if tup_guid == guid: + break + if inside_section: + tool_elem = SubElement( tool_section, 'tool' ) else: tool_elem = Element( 'tool' ) tool_elem.attrib[ 'file' ] = tool_file_path @@ -317,24 +454,133 @@ def generate_tool_panel_elem_list( repository_name, repository_clone_url, change id_elem.text = tool.id version_elem = SubElement( tool_elem, 'version' ) version_elem.text = tool.version - if not tool_section: + if inside_section: + if section_in_elem_list: + elem_list[ index ] = tool_section + else: + elem_list.append( tool_section ) + else: elem_list.append( tool_elem ) - if tool_section: - elem_list.append( root_elem ) return elem_list +def generate_tool_panel_dict_for_repository_tools( repository_metadata, tool_section=None, tool_section_dict=None ): + """ + Create a dictionary of the following type for every tool in the repository where the tools are all + contained in the same tool section or no tool section. If tool_section is None, tools will be displayed + outside of any sections in the tool panel. + { : { tool_config : , id: , version : , name : }} + """ + tool_panel_dict = {} + tool_dicts = repository_metadata[ 'tools' ] + for tool_dict in tool_dicts: + guid = tool_dict[ 'guid' ] + tool_config = tool_dict[ 'tool_config' ] + new_tool_section_dict = {} + if tool_section_dict: + for k, v in tool_section_dict.items(): + new_tool_section_dict[ k ] = v + file_path, file_name = os.path.split( tool_config ) + new_tool_section_dict[ 'tool_config' ] = file_name + else: + new_tool_section_dict = generate_tool_section_dict( tool_config=tool_config, tool_section=tool_section ) + tool_panel_dict[ guid ] = tool_section_dict + return tool_panel_dict +def generate_tool_panel_dict_for_tool_config( guid, tool_config, tool_section=None ): + """ + Create a dictionary of the following type for a single tool config file name. The intent is to call this method for every tool config + in a repository and append each of these as entries to a tool panel dictionary for the repository. This allows for each tool to be + loaded into a different section in the tool panel. + { : { tool_config : , id: , version : , name : }} + """ + tool_panel_dict = {} + file_path, file_name = os.path.split( tool_config ) + tool_section_dict = generate_tool_section_dict( tool_config=file_name, tool_section=tool_section ) + tool_panel_dict[ guid ] = tool_section_dict + return tool_panel_dict +def generate_tool_section_dict( tool_config=None, tool_section=None ): + if tool_config is None: + tool_config = '' + if tool_section: + section_id = tool_section.id or '' + section_version = tool_section.version or '' + section_name = tool_section.name or '' + tool_section_dict = dict( tool_config=tool_config, id=section_id, version=section_version, name=section_name ) + else: + tool_section_dict = dict( tool_config=tool_config, id='', version='', name='' ) + return tool_section_dict +def generate_tool_section_element_from_dict( tool_section_dict ): + # The value of tool_section_dict looks like the following. + # { id: , version : , name : } + if tool_section_dict[ 'id' ]: + # Create a new tool section. + tool_section = Element( 'section' ) + tool_section.attrib[ 'id' ] = tool_section_dict[ 'id' ] + tool_section.attrib[ 'name' ] = tool_section_dict[ 'name' ] + tool_section.attrib[ 'version' ] = tool_section_dict[ 'version' ] + else: + tool_section = None + return tool_section def generate_workflow_metadata( relative_path, exported_workflow_dict, metadata_dict ): - """ - Update the received metadata_dict with changes that have been applied - to the received exported_workflow_dict. Store everything in the database. - This method is used by the InstallManager, which does not have access to trans. - """ + """Update the received metadata_dict with changes that have been applied to the received exported_workflow_dict.""" if 'workflows' in metadata_dict: metadata_dict[ 'workflows' ].append( ( relative_path, exported_workflow_dict ) ) else: metadata_dict[ 'workflows' ] = [ ( relative_path, exported_workflow_dict ) ] return metadata_dict +def get_converter_and_display_paths( registration_elem, relative_install_dir ): + """Find the relative path to data type converters and display applications included in installed tool shed repositories.""" + converter_path = None + display_path = None + for elem in registration_elem.findall( 'datatype' ): + if not converter_path: + # If any of the tag sets contain tags, set the converter_path + # if it is not already set. This requires developers to place all converters in the + # same subdirectory within the repository hierarchy. + for converter in elem.findall( 'converter' ): + converter_config = converter.get( 'file', None ) + if converter_config: + relative_head, relative_tail = os.path.split( converter_config ) + for root, dirs, files in os.walk( relative_install_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name == relative_tail: + # The value of converter_path must be absolute due to job_working_directory. + converter_path = os.path.abspath( root ) + break + if converter_path: + break + if not display_path: + # If any of the tag sets contain tags, set the display_path + # if it is not already set. This requires developers to place all display acpplications + # in the same subdirectory within the repository hierarchy. + for display_app in elem.findall( 'display' ): + display_config = display_app.get( 'file', None ) + if display_config: + relative_head, relative_tail = os.path.split( display_config ) + for root, dirs, files in os.walk( relative_install_dir ): + if root.find( '.hg' ) < 0: + for name in files: + if name == relative_tail: + # The value of display_path must be absolute due to job_working_directory. + display_path = os.path.abspath( root ) + break + if display_path: + break + if converter_path and display_path: + break + return converter_path, display_path +def get_shed_tool_conf_dict( app, shed_tool_conf ): + """ + Return the in-memory version of the shed_tool_conf file, which is stored in the config_elems entry + in the shed_tool_conf_dict associated with the file. + """ + for index, shed_tool_conf_dict in enumerate( app.toolbox.shed_tool_confs ): + if shed_tool_conf == shed_tool_conf_dict[ 'config_filename' ]: + return index, shed_tool_conf_dict + else: + file_path, file_name = os.path.split( shed_tool_conf_dict[ 'config_filename' ] ) + if shed_tool_conf == file_name: + return index, shed_tool_conf_dict def get_repository_by_shed_name_owner_changeset_revision( app, tool_shed, name, owner, changeset_revision ): - # This method is used by the InstallManager, which does not have access to trans. sa_session = app.model.context.current if tool_shed.find( '//' ) > 0: tool_shed = tool_shed.split( '//' )[1] @@ -361,7 +607,6 @@ def get_repository_tools_tups( app, metadata_dict ): repository_tools_tups.append( ( relative_path, guid, tool ) ) return repository_tools_tups def get_tool_version( app, tool_id ): - # This method is used by the InstallManager, which does not have access to trans. sa_session = app.model.context.current return sa_session.query( app.model.ToolVersion ) \ .filter( app.model.ToolVersion.table.c.tool_id == tool_id ) \ @@ -375,7 +620,6 @@ def get_tool_version_association( app, parent_tool_version, tool_version ): .first() def get_url_from_repository_tool_shed( app, repository ): """ - This method is used by the UpdateManager, which does not have access to trans. The stored value of repository.tool_shed is something like: toolshed.g2.bx.psu.edu We need the URL to this tool shed, which is something like: http://toolshed.g2.bx.psu.edu/ """ @@ -384,14 +628,12 @@ def get_url_from_repository_tool_shed( app, repository ): if shed_url.endswith( '/' ): shed_url = shed_url.rstrip( '/' ) return shed_url - # The tool shed from which the repository was originally - # installed must no longer be configured in tool_sheds_conf.xml. + # The tool shed from which the repository was originally installed must no longer be configured in tool_sheds_conf.xml. return None def handle_missing_data_table_entry( app, tool_path, sample_files, repository_tools_tups ): """ - Inspect each tool to see if any have input parameters that are dynamically - generated select lists that require entries in the tool_data_table_conf.xml file. - This method is used by the InstallManager, which does not have access to trans. + Inspect each tool to see if any have input parameters that are dynamically generated select lists that require entries in the + tool_data_table_conf.xml file. """ missing_data_table_entry = False for index, repository_tools_tup in enumerate( repository_tools_tups ): @@ -400,8 +642,7 @@ def handle_missing_data_table_entry( app, tool_path, sample_files, repository_to missing_data_table_entry = True break if missing_data_table_entry: - # The repository must contain a tool_data_table_conf.xml.sample file that includes - # all required entries for all tools in the repository. + # The repository must contain a tool_data_table_conf.xml.sample file that includes all required entries for all tools in the repository. for sample_file in sample_files: head, tail = os.path.split( sample_file ) if tail == 'tool_data_table_conf.xml.sample': @@ -415,11 +656,7 @@ def handle_missing_data_table_entry( app, tool_path, sample_files, repository_to repository_tools_tups[ index ] = ( tup_path, repository_tool ) return repository_tools_tups def handle_missing_index_file( app, tool_path, sample_files, repository_tools_tups ): - """ - Inspect each tool to see if it has any input parameters that - are dynamically generated select lists that depend on a .loc file. - This method is used by the InstallManager, which does not have access to trans. - """ + """Inspect each tool to see if it has any input parameters that are dynamically generated select lists that depend on a .loc file.""" missing_files_handled = [] for index, repository_tools_tup in enumerate( repository_tools_tups ): tup_path, guid, repository_tool = repository_tools_tup @@ -445,7 +682,6 @@ def handle_tool_dependencies( current_working_dir, repo_files_dir, repository_to """ Inspect each tool to see if it includes a "requirement" that refers to a fabric script. For those that do, execute the fabric script to install tool dependencies. - This method is used by the InstallManager, which does not have access to trans. """ for index, repository_tools_tup in enumerate( repository_tools_tups ): tup_path, guid, repository_tool = repository_tools_tup @@ -479,10 +715,8 @@ def handle_tool_dependencies( current_working_dir, repo_files_dir, repository_to log.debug( 'Problem installing dependencies for tool "%s"\n%s' % ( repository_tool.name, error ) ) def handle_tool_versions( app, tool_version_dicts, tool_shed_repository ): """ - This method is used by the InstallManager, which does not have access to trans. Using the list - of tool_version_dicts retrieved from the tool shed (one per chngeset revison up to the currently - installed changeset revision), create the parent / child pairs of tool versions. Each dictionary - contains { tool id : parent tool id } pairs. + Using the list of tool_version_dicts retrieved from the tool shed (one per changeset revison up to the currently installed changeset revision), + create the parent / child pairs of tool versions. Each dictionary contains { tool id : parent tool id } pairs. """ sa_session = app.model.context.current for tool_version_dict in tool_version_dicts: @@ -527,201 +761,64 @@ def load_datatype_items( app, repository, relative_install_dir, deactivate=False if display_path: # Load or deactivate proprietary datatype display applications app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict, deactivate=deactivate ) -def alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir, deactivate=False, override=True ): +def load_repository_contents( trans, repository_name, description, owner, changeset_revision, tool_path, repository_clone_url, relative_install_dir, + current_working_dir, tmp_name, tool_panel_dict=None, tool_shed=None, tool_section=None, shed_tool_conf=None, + new_install=True ): """ - Parse a proprietary datatypes config (a datatypes_conf.xml file included in an installed tool shed repository) and - add information to appropriate elements that will enable proprietary datatype class modules, datatypes converters - and display application to be discovered and properly imported by the datatypes registry. The value of override will - be False when a tool shed repository is being installed. Since installation is occurring after the datatypes registry - has been initialized, its contents cannot be overridden by conflicting data types. This method is used by the InstallManager, - which does not have access to trans. + Generate the metadata for the installed tool shed repository, among other things. It is critical that the installed repository + is updated to the desired changeset_revision before metadata is set because the process for setting metadata uses the repository + files on disk. If this method is called when a new repository is being installed, the value of tool_panel_dict will be None and + the value of tool_section (a ToolSection or None) will be used. This method is also called when updates have been pulled to a + previously installed repository, in which case the value of tool_panel_dict will be used and the value of new_install will be False. """ - tree = util.parse_xml( datatypes_config ) - datatypes_config_root = tree.getroot() - # Path to datatype converters - converter_path = None - # Path to datatype display applications - display_path = None - relative_path_to_datatype_file_name = None - datatype_files = datatypes_config_root.find( 'datatype_files' ) - datatype_class_modules = [] - if datatype_files: - # The tag set contains any number of tags. - # - # - # - # - # We'll add attributes to the datatype tag sets so that the modules can be properly imported by the datatypes registry. - for elem in datatype_files.findall( 'datatype_file' ): - datatype_file_name = elem.get( 'name', None ) - if datatype_file_name: - # Find the file in the installed repository. - for root, dirs, files in os.walk( relative_install_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name == datatype_file_name: - datatype_class_modules.append( os.path.join( root, name ) ) - break - break - if datatype_class_modules: - registration = datatypes_config_root.find( 'registration' ) - converter_path, display_path = get_converter_and_display_paths( registration, relative_install_dir ) - if converter_path: - registration.attrib[ 'proprietary_converter_path' ] = converter_path - if display_path: - registration.attrib[ 'proprietary_display_path' ] = display_path - for relative_path_to_datatype_file_name in datatype_class_modules: - relative_head, relative_tail = os.path.split( relative_path_to_datatype_file_name ) - for elem in registration.findall( 'datatype' ): - # Handle 'type' attribute which should be something like one of the following: - # type="gmap:GmapDB" - # type="galaxy.datatypes.gmap:GmapDB" - dtype = elem.get( 'type', None ) - if dtype: - fields = dtype.split( ':' ) - proprietary_datatype_module = fields[ 0 ] - if proprietary_datatype_module.find( '.' ) >= 0: - # Handle the case where datatype_module is "galaxy.datatypes.gmap". - proprietary_datatype_module = proprietary_datatype_module.split( '.' )[ -1 ] - # The value of proprietary_path must be an absolute path due to job_working_directory. - elem.attrib[ 'proprietary_path' ] = os.path.abspath( relative_head ) - elem.attrib[ 'proprietary_datatype_module' ] = proprietary_datatype_module - - sniffers = datatypes_config_root.find( 'sniffers' ) - fd, proprietary_datatypes_config = tempfile.mkstemp() - os.write( fd, '\n' ) - os.write( fd, '\n' ) - os.write( fd, '%s' % util.xml_to_string( registration ) ) - os.write( fd, '%s' % util.xml_to_string( sniffers ) ) - os.write( fd, '\n' ) - os.close( fd ) - os.chmod( proprietary_datatypes_config, 0644 ) + if tool_panel_dict: + # We're resetting metadata on a previously installed repository. For backward compatibility we have to handle 2 types of dictionaries. + # In the past, all repository tools had to be installed into a single ToolSection (or outside of any sections) in the tool panel. + if panel_entry_per_tool( tool_panel_dict ): + # { : { tool_config : , id: , version : , name : }} + metadata_dict = generate_metadata( trans.app.toolbox, relative_install_dir, repository_clone_url, tool_panel_dict=tool_panel_dict ) + else: + # { id: , version : , name : } + metadata_dict = generate_metadata( trans.app.toolbox, relative_install_dir, repository_clone_url, tool_section_dict=tool_panel_dict ) else: - proprietary_datatypes_config = datatypes_config - # Load proprietary datatypes - app.datatypes_registry.load_datatypes( root_dir=app.config.root, config=proprietary_datatypes_config, deactivate=deactivate, override=override ) - try: - os.unlink( proprietary_datatypes_config ) - except: - pass - return converter_path, display_path -def get_converter_and_display_paths( registration_elem, relative_install_dir ): - """ - Find the relative path to data type converters and display - applications included in installed tool shed repositories. - """ - converter_path = None - display_path = None - for elem in registration_elem.findall( 'datatype' ): - if not converter_path: - # If any of the tag sets contain tags, set the converter_path - # if it is not already set. This requires developers to place all converters in the - # same subdirectory within the repository hierarchy. - for converter in elem.findall( 'converter' ): - converter_config = converter.get( 'file', None ) - if converter_config: - relative_head, relative_tail = os.path.split( converter_config ) - for root, dirs, files in os.walk( relative_install_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name == relative_tail: - # The value of converter_path must be absolute due to job_working_directory. - converter_path = os.path.abspath( root ) - break - if converter_path: - break - if not display_path: - # If any of the tag sets contain tags, set the display_path - # if it is not already set. This requires developers to place all display acpplications - # in the same subdirectory within the repository hierarchy. - for display_app in elem.findall( 'display' ): - display_config = display_app.get( 'file', None ) - if display_config: - relative_head, relative_tail = os.path.split( display_config ) - for root, dirs, files in os.walk( relative_install_dir ): - if root.find( '.hg' ) < 0: - for name in files: - if name == relative_tail: - # The value of display_path must be absolute due to job_working_directory. - display_path = os.path.abspath( root ) - break - if display_path: - break - if converter_path and display_path: - break - return converter_path, display_path -def load_repository_contents( app, repository_name, description, owner, changeset_revision, tool_path, repository_clone_url, relative_install_dir, - current_working_dir, tmp_name, tool_shed=None, tool_section=None, shed_tool_conf=None, new_install=True, dist_to_shed=False ): - """ - This method is used by the InstallManager (which does not have access to trans), to generate the metadata - for the installed tool shed repository, among other things. It is imperative that the installed repository - is updated to the desired changeset_revision before metadata is set because the process for setting metadata - uses the repository files on disk. This method is called when new tools have been installed (in which case - values should be received for tool_section and shed_tool_conf, and new_install should be left at it's default - value) and when updates have been pulled to previously installed repositories (in which case the default value - None is set for tool_section and shed_tool_conf, and the value of new_install is passed as False). When a new - install is being done by the InstallManager (and not a user manually installing a repository from the Admin - perspective), the value of dist_to_shed will be set to True, enabling determinatin of which installed repositories - resulted from the InstallManager installing a repository that contains tools that used to be in the Galaxy - distribution but are now in the main Galaxy tool shed. - """ - if tool_section: - if tool_section.id is None: - section_id = '' - else: - section_id = tool_section.id - if tool_section.version is None: - section_version = '' - else: - section_version = tool_section.version - if tool_section.name is None: - section_name = '' - else: - section_name = tool_section.name - else: - section_id = '' - section_version = '' - section_name = '' - tool_section_dict = dict( id=section_id, version=section_version, name=section_name ) - metadata_dict = generate_metadata( app.toolbox, relative_install_dir, repository_clone_url, tool_section_dict=tool_section_dict ) - # Add a new record to the tool_shed_repository table if one doesn't already exist. If one exists but is marked - # deleted, undelete it. It is imperative that this happens before the call to alter_tool_panel() below because - # tools will not be properly loaded if the repository is marked deleted. + # We're installing a new repository or reinstalling an uninstalled repository where all tools are contained in the same tool panel section + # or outside of any sections in the tool panel. We cannot pass a specific tool_config since we do not yet have one. + tool_section_dict = generate_tool_section_dict( tool_config=None, tool_section=tool_section ) + metadata_dict = generate_metadata( trans.app.toolbox, relative_install_dir, repository_clone_url, tool_section_dict=tool_section_dict ) + # Add a new record to the tool_shed_repository table if one doesn't already exist. If one exists but is marked deleted, undelete it. This + # must happen before the call to add_to_tool_panel() below because tools will not be properly loaded if the repository is marked deleted. log.debug( "Adding new row (or updating an existing row) for repository '%s' in the tool_shed_repository table." % repository_name ) - tool_shed_repository = create_or_update_tool_shed_repository( app, + tool_shed_repository = create_or_update_tool_shed_repository( trans.app, repository_name, description, changeset_revision, repository_clone_url, metadata_dict, - dist_to_shed=dist_to_shed ) + dist_to_shed=False ) if 'tools' in metadata_dict: - repository_tools_tups = get_repository_tools_tups( app, metadata_dict ) + repository_tools_tups = get_repository_tools_tups( trans.app, metadata_dict ) if repository_tools_tups: sample_files = metadata_dict.get( 'sample_files', [] ) # Handle missing data table entries for tool parameters that are dynamically generated select lists. - repository_tools_tups = handle_missing_data_table_entry( app, tool_path, sample_files, repository_tools_tups ) + repository_tools_tups = handle_missing_data_table_entry( trans.app, tool_path, sample_files, repository_tools_tups ) # Handle missing index files for tool parameters that are dynamically generated select lists. - repository_tools_tups = handle_missing_index_file( app, tool_path, sample_files, repository_tools_tups ) + repository_tools_tups = handle_missing_index_file( trans.app, tool_path, sample_files, repository_tools_tups ) # Handle tools that use fabric scripts to install dependencies. handle_tool_dependencies( current_working_dir, relative_install_dir, repository_tools_tups ) if new_install: - alter_tool_panel( app=app, - repository_name=repository_name, - repository_clone_url=repository_clone_url, - changeset_revision=changeset_revision, - repository_tools_tups=repository_tools_tups, - tool_section=tool_section, - shed_tool_conf=shed_tool_conf, - tool_path=tool_path, - owner=owner, - new_install=new_install, - deactivate=False, - uninstall=False ) - elif app.toolbox_search.enabled: - # If search support for tools is enabled, index the new installed tools. In the - # condition above, this happens in the alter_tool_panel() method. - app.toolbox_search = ToolBoxSearch( app.toolbox ) + add_to_tool_panel( app=trans.app, + repository_name=repository_name, + repository_clone_url=repository_clone_url, + changeset_revision=changeset_revision, + repository_tools_tups=repository_tools_tups, + owner=owner, + shed_tool_conf=shed_tool_conf, + tool_panel_dict=tool_panel_dict, + new_install=new_install ) + elif trans.app.toolbox_search.enabled: + # If search support for tools is enabled, index the new installed tools. In the condition above, this happens in the + # add_to_tool_panel() method. + trans.app.toolbox_search = ToolBoxSearch( trans.app.toolbox ) # Remove the temporary file try: os.unlink( tmp_name ) @@ -731,7 +828,7 @@ def load_repository_contents( app, repository_name, description, owner, changese datatypes_config = os.path.abspath( metadata_dict[ 'datatypes_config' ] ) # Load data types required by tools. override = not new_install - converter_path, display_path = alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir, override=override ) + converter_path, display_path = alter_config_and_load_prorietary_datatypes( trans.app, datatypes_config, relative_install_dir, override=override ) if converter_path or display_path: # Create a dictionary of tool shed repository related information. repository_dict = create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed, @@ -743,118 +840,22 @@ def load_repository_contents( app, repository_name, description, owner, changese display_path=display_path ) if converter_path: # Load proprietary datatype converters - app.datatypes_registry.load_datatype_converters( app.toolbox, installed_repository_dict=repository_dict ) + trans.app.datatypes_registry.load_datatype_converters( trans.app.toolbox, installed_repository_dict=repository_dict ) if display_path: # Load proprietary datatype display applications - app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) + trans.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) return tool_shed_repository, metadata_dict -def alter_tool_panel( app, repository_name, repository_clone_url, changeset_revision, repository_tools_tups, tool_section, - shed_tool_conf, tool_path, owner, new_install=True, deactivate=False, uninstall=False ): - """ - A tool shed repository is being installed / updated / deactivated / uninstalled, - so handle tool panel alterations accordingly. - """ - # We need to change the in-memory version of the shed_tool_conf file, which is stored in the config_elems entry - # in the shed_tool_conf_dict associated with the file. - for index, shed_tool_conf_dict in enumerate( app.toolbox.shed_tool_confs ): - config_filename = shed_tool_conf_dict[ 'config_filename' ] - if config_filename == shed_tool_conf: - config_elems = shed_tool_conf_dict[ 'config_elems' ] - break - else: - head, tail = os.path.split( config_filename ) - if tail == shed_tool_conf: - config_elems = shed_tool_conf_dict[ 'config_elems' ] - break - # Geneate the list of ElementTree Element objects for each section or list of tools. - elem_list = generate_tool_panel_elem_list( repository_name, - repository_clone_url, - changeset_revision, - repository_tools_tups, - tool_section=tool_section, - owner=owner ) - if deactivate: - # Remove appropriate entries from the shed_tool_conf file on disk. We first create an list of - # guids for all tools that will be removed from the tool config. - tool_elements_removed = 0 - guids_to_remove = [ repository_tool_tup[1] for repository_tool_tup in repository_tools_tups ] - if tool_section: - # Remove all appropriate tool sub-elements from the section element. - for section_elem in elem_list: - section_elem_id = section_elem.get( 'id' ) - section_elem_name = section_elem.get( 'name' ) - section_elem_version = section_elem.get( 'version' ) - for config_elem in config_elems: - config_elems_to_remove = [] - if config_elem.tag == 'section' and \ - config_elem.get( 'id' ) == section_elem_id and \ - config_elem.get( 'name' ) == section_elem_name and \ - config_elem.get( 'version' ) == section_elem_version: - # We've located the section element in the in-memory list of config_elems, so we can remove - # all of the appropriate tools sub-elements from the section. - tool_elems_to_remove = [] - for tool_elem in config_elem: - tool_elem_guid = tool_elem.get( 'guid' ) - if tool_elem_guid in guids_to_remove: - tool_elems_to_remove.append( tool_elem ) - for tool_elem in tool_elems_to_remove: - # Remove all of the appropriate tool sub-elements from the section element. - tool_elem_guid = tool_elem.get( 'guid' ) - config_elem.remove( tool_elem ) - log.debug( "Removed tool with guid '%s'." % str( tool_elem_guid ) ) - tool_elements_removed += 1 - if len( config_elem ) < 1: - # Keep a list of all empty section elements so they can be removed. - config_elems_to_remove.append( config_elem ) - if tool_elements_removed == len( guids_to_remove ): - break - for config_elem in config_elems_to_remove: - # The section element includes no tool sub-elements, so remove it. - config_elems.remove( config_elem ) - log.debug( "Removed section with id '%s'." % str( section_elem_id ) ) - if tool_elements_removed == len( guids_to_remove ): - break - else: - # Remove all appropriate tool elements from the root (tools are outside of any sections). - tool_elems_to_remove = [] - for tool_elem in elem_list: - tool_elem_guid = tool_elem.get( 'guid' ) - for config_elem in config_elems: - if config_elem.tag == 'tool' and config_elem.get( 'guid' ) == tool_elem_guid: - tool_elems_to_remove.append( tool_elem ) - for config_elem in config_elems: - for tool_elem in tool_elems_to_remove: - try: - # Remove the tool element from the in-memory list of config_elems. - config_elem.remove( tool_elem ) - log.debug( "Removed tool with guid '%s'." % str( tool_elem_guid ) ) - except: - # The tool_elem is not a sub-element of the current config_elem. - pass - else: - # Generate a new entry for the tool config. - if tool_section: - for section_elem in elem_list: - # Load the section into the tool panel. - app.toolbox.load_section_tag_set( section_elem, app.toolbox.tool_panel, tool_path ) - else: - # Load the tools into the tool panel outside of any sections. - for tool_elem in elem_list: - guid = tool_elem.get( 'guid' ) - app.toolbox.load_tool_tag_set( tool_elem, app.toolbox.tool_panel, tool_path=tool_path, guid=guid ) - if new_install: - # Append the new entry (either section or list of tools) to the shed_tool_config file, - # and add the xml element to the in-memory list of config_elems. - for elem_entry in elem_list: - config_elems.append( elem_entry ) - shed_tool_conf_dict[ 'config_elems' ] = config_elems - app.toolbox.shed_tool_confs[ index ] = shed_tool_conf_dict - if uninstall or not deactivate: - # Persist the altered in-memory version of the tool config. - config_elems_to_xml_file( app, shed_tool_conf_dict ) - if app.toolbox_search.enabled: - # If search support for tools is enabled, index the new installed tools. - app.toolbox_search = ToolBoxSearch( app.toolbox ) +def panel_entry_per_tool( tool_section_dict ): + # Return True if tool_section_dict looks like this. + # { : { tool_config : , id: , version : , name : }} + # But not like this. + # { id: , version : , name : } + if len( tool_section_dict ) != 3: + return True + for k, v in tool_section_dict: + if k not in [ 'id', 'version', 'name' ]: + return True + return False def pull_repository( current_working_dir, repo_files_dir, name ): # Pull the latest possible contents to the repository. log.debug( "Pulling latest updates to the repository named '%s'" % name ) @@ -867,6 +868,60 @@ def pull_repository( current_working_dir, repo_files_dir, name ): os.chdir( current_working_dir ) tmp_stderr.close() return returncode, tmp_name +def remove_from_tool_panel( app, shed_tool_conf, tool_panel_dict, uninstall=False ): + # A tool shed repository is being deactivated or uninstalled so handle tool panel alterations accordingly. We need to change the + # in-memory version and the file system version of the shed_tool_conf file. The value of tool_panel_dict is a dictionary of entries + # that look like this. + # { : { tool_config : , id: , version : , name : }} + index, shed_tool_conf_dict = get_shed_tool_conf_dict( app, shed_tool_conf ) + config_elems = shed_tool_conf_dict[ 'config_elems' ] + tool_elements_removed = 0 + config_elems_to_remove = [] + # Create a list of guids for all tools that will be removed from the tool panel config. + guids_to_remove = [ k for k in tool_panel_dict.keys() ] + for config_elem in config_elems: + config_elems_to_remove = [] + if config_elem.tag == 'section': + tool_elems_to_remove = [] + for tool_elem in config_elem: + if tool_elem.get( 'guid' ) in guids_to_remove: + tool_elems_to_remove.append( tool_elem ) + for tool_elem in tool_elems_to_remove: + # Remove all of the appropriate tool sub-elements from the section element. + config_elem.remove( tool_elem ) + log.debug( "Removed tool with guid '%s'." % str( tool_elem.get( 'guid' ) ) ) + tool_elements_removed += 1 + if len( config_elem ) < 1: + # Keep a list of all empty section elements so they can be removed. + config_elems_to_remove.append( config_elem ) + if tool_elements_removed == len( guids_to_remove ): + break + elif config_elem.tag == 'tool': + if config_elem.get( 'guid' ) in guids_to_remove: + config_elems_to_remove.append( config_elem ) + log.debug( "Removed tool with guid '%s'." % str( config_elem.get( 'guid' ) ) ) + tool_elements_removed += 1 + if tool_elements_removed == len( guids_to_remove ): + break + for config_elem in config_elems_to_remove: + # Remove the element from the in-menory tool panel. + if config_elem.tag == 'section': + key = 'section_%s' % str( config_elem.get( "id" ) ) + del app.toolbox.tool_panel[ key ] + elif config_elem.tag == 'tool': + key = 'tool_%s' % str( config_elem.get( 'guid' ) ) + del app.toolbox.tool_panel[ key ] + # Remove the element from the in-memory list of elements. + config_elems.remove( config_elem ) + # Update the config_elems of the in-memory shed_tool_conf_dict. + shed_tool_conf_dict[ 'config_elems' ] = config_elems + app.toolbox.shed_tool_confs[ index ] = shed_tool_conf_dict + if uninstall: + # Persist the altered in-memory version of the tool config. + config_elems_to_xml_file( app, shed_tool_conf_dict ) + if app.toolbox_search.enabled: + # If search support for tools is enabled, index the new installed tools. + app.toolbox_search = ToolBoxSearch( app.toolbox ) def update_repository( current_working_dir, repo_files_dir, changeset_revision ): # Update the cloned repository to changeset_revision. It is imperative that the # installed repository is updated to the desired changeset_revision before metadata diff --git a/lib/galaxy/web/controllers/admin_toolshed.py b/lib/galaxy/web/controllers/admin_toolshed.py index 8e609587c15..430ed948168 100644 --- a/lib/galaxy/web/controllers/admin_toolshed.py +++ b/lib/galaxy/web/controllers/admin_toolshed.py @@ -74,6 +74,95 @@ class AdminToolshed( AdminGalaxy ): repository_list_grid = RepositoryListGrid() + @web.expose + @web.require_admin + def activate_repository( self, trans, **kwd ): + """Activate a repository that was deactivated but not uninstalled.""" + repository = get_repository( trans, kwd[ 'id' ] ) + shed_tool_conf, tool_path, relative_install_dir = self.__get_tool_panel_config_tool_path_install_dir( trans, repository ) + repository.deleted = False + trans.sa_session.add( repository ) + trans.sa_session.flush() + if repository.includes_tools: + # Reload tools into the appropriate tool panel section. + metadata = repository.metadata + if 'tool_panel_section' in metadata: + if panel_entry_per_tool( metadata[ 'tool_panel_section' ] ): + # { : { tool_config : , id: , version : , name : }} + tool_panel_dict = metadata[ 'tool_panel_section' ] + # TODO: Fix this to handle the case where the tools are distributed across in more than 1 section. The + # following assumes everything was loaded into 1 section (or no section) in the tool panel. + tool_section_dict = tool_panel_dict[ tool_panel_dict.keys()[ 0 ] ] + else: + # The value of tool_panel_section is the old dictionary type like this, so update to the new dictionary type like that above. + # { id: , version : , name : } + tool_section_dict = metadata[ 'tool_panel_section' ] + original_section_id = tool_section_dict[ 'id' ] + else: + # Tools were loaded outside of any panel sections. + original_section_id = '' + repository_tools_tups = get_repository_tools_tups( trans.app, metadata ) + guids_to_activate = [ repository_tool_tup[1] for repository_tool_tup in repository_tools_tups ] + # Make sure we have a tool_version for each guid. + for guid_to_activate in guids_to_activate: + if not get_tool_version( trans.app, guid_to_activate ): + # We're somehow missing a tool_version, so create a new one. + tool_version = trans.model.ToolVersion( tool_id=guid_to_activate, tool_shed_repository=repository ) + trans.sa_session.add( tool_version ) + trans.sa_session.flush() + if original_section_id in [ '' ]: + # If the repository includes tools, reload them into the tool panel outside of any sections. + self.__add_tools_to_tool_panel( trans, repository, repository_tools_tups, tool_section=None, section_key=None ) + else: + original_section_name = tool_section_dict[ 'name' ] + original_section_version = tool_section_dict[ 'version' ] + section_key = 'section_%s' % str( original_section_id ) + if section_key in trans.app.toolbox.tool_panel: + # Load the repository tools into a section that still exists in the tool panel. + tool_section = trans.app.toolbox.tool_panel[ section_key ] + self.__add_tools_to_tool_panel( trans, repository, repository_tools_tups, tool_section=tool_section, section_key=section_key ) + else: + # Load the repository tools into a section that no longer exists in the tool panel. The section must + # still exist in the tool config since the repository was only deactivated and not uninstalled. + sections_to_load = [] + tool_elems_found = 0 + # Only inspect tool configs that contain installed tool shed repositories. + for shed_tool_conf_dict in trans.app.toolbox.shed_tool_confs: + config_filename = shed_tool_conf_dict[ 'config_filename' ] + log.info( "Parsing the tool configuration %s" % config_filename ) + tree = util.parse_xml( config_filename ) + root = tree.getroot() + tool_path = root.get( 'tool_path' ) + if tool_path is not None: + # Tool configs that contain tools installed from tool shed repositories must have a tool_path attribute. + for elem in root: + if elem.tag == 'section' and \ + elem.get( 'id' ) == original_section_id and \ + elem.get( 'name' ) == original_section_name and \ + elem.get( 'version' ) == original_section_version: + # We've found the section, but we have to make sure it contains the + # correct tool tag set. This is necessary because the shed tool configs + # can include multiple sections of the same id, name and version, each + # containing one or more tool tag sets. + for tool_elem in elem: + if tool_elem.get( 'guid' ) in guids_to_activate: + tool_elems_found += 1 + if elem not in sections_to_load: + sections_to_load.append( elem ) + if tool_elems_found == len( guids_to_activate ): + break + if tool_elems_found == len( guids_to_activate ): + break + if tool_elems_found == len( guids_to_activate ): + break + for elem in sections_to_load: + trans.app.toolbox.load_section_tag_set( elem, trans.app.toolbox.tool_panel, tool_path ) + message = 'The %s repository has been activated.' % repository.name + status = 'done' + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='browse_repositories', + message=message, + status=status ) ) @web.expose @web.require_admin def browse_repository( self, trans, **kwd ): @@ -96,10 +185,14 @@ class AdminToolshed( AdminGalaxy ): return self.check_for_updates( trans, **kwd ) if operation == "activate or reinstall": repository = get_repository( trans, kwd[ 'id' ] ) - if repository.uninstalled and repository.includes_tools: - # Only allow selecting a different section in the tool panel if the repository was uninstalled. - return self.reselect_tool_panel_section( trans, **kwd ) - return self.activate_or_reinstall_repository( trans, **kwd ) + if repository.uninstalled: + if repository.includes_tools: + # Only allow selecting a different section in the tool panel if the repository was uninstalled. + return self.reselect_tool_panel_section( trans, **kwd ) + else: + return self.reinstall_repository( trans, **kwd ) + else: + return self.activate_repository( trans, **kwd ) if operation == "deactivate or uninstall": return self.deactivate_or_uninstall_repository( trans, **kwd ) if 'message' not in kwd or not kwd[ 'message' ]: @@ -107,6 +200,13 @@ class AdminToolshed( AdminGalaxy ): return self.repository_list_grid( trans, **kwd ) @web.expose @web.require_admin + def browse_tool_shed( self, trans, **kwd ): + tool_shed_url = kwd[ 'tool_shed_url' ] + galaxy_url = url_for( '/', qualified=True ) + url = '%s/repository/browse_valid_repositories?galaxy_url=%s&webapp=galaxy' % ( tool_shed_url, galaxy_url ) + return trans.response.send_redirect( url ) + @web.expose + @web.require_admin def browse_tool_sheds( self, trans, **kwd ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) @@ -117,6 +217,88 @@ class AdminToolshed( AdminGalaxy ): status='error' ) @web.expose @web.require_admin + def check_for_updates( self, trans, **kwd ): + # Send a request to the relevant tool shed to see if there are any updates. + repository = get_repository( trans, kwd[ 'id' ] ) + tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) + url = '%s/repository/check_for_updates?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ + ( tool_shed_url, url_for( '/', qualified=True ), repository.name, repository.owner, repository.changeset_revision ) + return trans.response.send_redirect( url ) + @web.expose + @web.require_admin + def deactivate_or_uninstall_repository( self, trans, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + remove_from_disk = params.get( 'remove_from_disk', '' ) + remove_from_disk_checked = CheckboxField.is_checked( remove_from_disk ) + repository = get_repository( trans, kwd[ 'id' ] ) + shed_tool_conf, tool_path, relative_install_dir = self.__get_tool_panel_config_tool_path_install_dir( trans, repository ) + if params.get( 'deactivate_or_uninstall_repository_button', False ): + metadata = repository.metadata + if repository.includes_tools: + repository_tools_tups = get_repository_tools_tups( trans.app, metadata ) + # Generate the list of tool panel keys derived from the tools included in the repository. + if repository_tools_tups: + repository_tool_panel_keys = [ 'tool_%s' % repository_tools_tup[ 1 ] for repository_tools_tup in repository_tools_tups ] + else: + repository_tool_panel_keys = [] + if 'tool_panel_section' in metadata: + if panel_entry_per_tool( metadata[ 'tool_panel_section' ] ): + # The tool_panel_section dictionary contains entries that look like this. + # { : { tool_config : , id: , version : , name : }} + tool_panel_dict = metadata[ 'tool_panel_section' ] + else: + # The tool_panel_section dictionary looks like the following. This is the old definition of the tool_panel_section, + # so update it to the current dictionary like that above. All of the repository tools will be installed in the same + # section or outside of any sections. + # { id: , version : , name : } + tool_section_dict = metadata[ 'tool_panel_section' ] + tool_panel_dict = generate_tool_panel_dict_for_repository_tools( metadata, tool_section_dict=tool_section_dict ) + repository.metadata[ 'tool_panel_section' ] = tool_panel_dict + else: + # The tool_panel_section was introduced late, so set it's value if its missing in the metadata. + tool_panel_dict = generate_tool_panel_dict_for_repository_tools( metadata ) + repository.metadata[ 'tool_panel_section' ] = tool_panel_dict + repository_clone_url = generate_clone_url( trans, repository ) + # The repository is either being deactivated or uninstalled, so handle tool panel alterations accordingly. + # If the repository is being uninstalled, the appropriate tools or tool sections will be removed from the + # appropriate shed-related tool config file on disk. + remove_from_tool_panel( app=trans.app, + shed_tool_conf=shed_tool_conf, + tool_panel_dict=tool_panel_dict, + uninstall=remove_from_disk_checked ) + if repository.includes_datatypes: + # Deactivate proprietary datatypes. + load_datatype_items( trans.app, repository, relative_install_dir, deactivate=True ) + if remove_from_disk_checked: + # Remove the repository from disk. + try: + shutil.rmtree( relative_install_dir ) + log.debug( "Removed repository installation directory: %s" % str( relative_install_dir ) ) + except Exception, e: + log.debug( "Error removing repository installation directory %s: %s" % ( str( relative_install_dir ), str( e ) ) ) + repository.uninstalled = True + repository.deleted = True + trans.sa_session.add( repository ) + trans.sa_session.flush() + if remove_from_disk_checked: + message = 'The repository named %s has been uninstalled.' % repository.name + else: + message = 'The repository named %s has been deactivated.' % repository.name + status = 'done' + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='browse_repositories', + message=message, + status=status ) ) + remove_from_disk_check_box = CheckboxField( 'remove_from_disk', checked=remove_from_disk_checked ) + return trans.fill_template( '/admin/tool_shed_repository/deactivate_or_uninstall_repository.mako', + repository=repository, + remove_from_disk_check_box=remove_from_disk_check_box, + message=message, + status=status ) + @web.expose + @web.require_admin def find_tools_in_tool_shed( self, trans, **kwd ): tool_shed_url = kwd[ 'tool_shed_url' ] galaxy_url = url_for( '/', qualified=True ) @@ -131,13 +313,6 @@ class AdminToolshed( AdminGalaxy ): return trans.response.send_redirect( url ) @web.expose @web.require_admin - def browse_tool_shed( self, trans, **kwd ): - tool_shed_url = kwd[ 'tool_shed_url' ] - galaxy_url = url_for( '/', qualified=True ) - url = '%s/repository/browse_valid_repositories?galaxy_url=%s&webapp=galaxy' % ( tool_shed_url, galaxy_url ) - return trans.response.send_redirect( url ) - @web.expose - @web.require_admin def install_repository( self, trans, **kwd ): if not trans.app.toolbox.shed_tool_confs: message = 'The tool_config_file setting in universe_wsgi.ini must include at least one shed tool configuration file name with a ' @@ -159,10 +334,9 @@ class AdminToolshed( AdminGalaxy ): if includes_tools: shed_tool_conf = kwd[ 'shed_tool_conf' ] else: - # If installing a repository that includes no tools, get the relative - # tool_path from the file to which the install_tool_config_file config - # setting points. - shed_tool_conf = trans.app.config.install_tool_config + # If installing a repository that includes no tools, get the relative tool_path from the file + # to which the migrated_tools_config setting points. + shed_tool_conf = trans.app.config.migrated_tools_config # Get the tool path by searching the list of shed_tool_confs for the dictionary # that contains the information about shed_tool_conf. for shed_tool_conf_dict in trans.app.toolbox.shed_tool_confs: @@ -208,7 +382,6 @@ class AdminToolshed( AdminGalaxy ): relative_install_dir = os.path.join( clone_dir, name ) if os.path.exists( clone_dir ): # Repository and revision has already been cloned. - # TODO: implement the ability to re-install or revert an existing repository. message += 'Revision %s of repository %s was previously installed.
' % ( changeset_revision, name ) else: returncode, tmp_name = clone_repository( name, clone_dir, current_working_dir, repository_clone_url ) @@ -217,7 +390,7 @@ class AdminToolshed( AdminGalaxy ): if returncode == 0: owner = get_repository_owner( clean_repository_clone_url( repository_clone_url ) ) tool_shed = clean_tool_shed_url( tool_shed_url ) - tool_shed_repository, metadata_dict = load_repository_contents( app=trans.app, + tool_shed_repository, metadata_dict = load_repository_contents( trans, repository_name=name, description=description, owner=owner, @@ -227,11 +400,11 @@ class AdminToolshed( AdminGalaxy ): relative_install_dir=relative_install_dir, current_working_dir=current_working_dir, tmp_name=tmp_name, + tool_panel_dict=None, tool_shed=tool_shed, tool_section=tool_section, shed_tool_conf=shed_tool_conf, - new_install=True, - dist_to_shed=False ) + new_install=True ) if 'tools' in metadata_dict: @@ -299,140 +472,214 @@ class AdminToolshed( AdminGalaxy ): status=status ) @web.expose @web.require_admin - def set_tool_versions( self, trans, **kwd ): - # Get the tool_versions from the tool shed for each tool in the installed change set. + def manage_repository( self, trans, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) repository = get_repository( trans, kwd[ 'id' ] ) - tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) - url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ - ( tool_shed_url, repository.name, repository.owner, repository.changeset_revision ) - response = urllib2.urlopen( url ) - text = response.read() - response.close() - if text: - tool_version_dicts = from_json_string( text ) - handle_tool_versions( trans.app, tool_version_dicts, repository ) - message = "Tool versions have been set for all included tools." - status = 'done' - else: - message = "Version information for the tools included in the %s repository is missing. " % repository.name - message += "Reset all of this reppository's metadata in the tool shed, then set the installed tool versions " - message ++ "from the installed repository's Repository Actions menu. " - status = 'error' - shed_tool_conf, tool_path, relative_install_dir = self.__get_tool_path_and_relative_install_dir( trans, repository ) + description = util.restore_text( params.get( 'description', repository.description ) ) + shed_tool_conf, tool_path, relative_install_dir = self.__get_tool_panel_config_tool_path_install_dir( trans, repository ) repo_files_dir = os.path.abspath( os.path.join( relative_install_dir, repository.name ) ) + if params.get( 'edit_repository_button', False ): + if description != repository.description: + repository.description = description + trans.sa_session.add( repository ) + trans.sa_session.flush() + message = "The repository information has been updated." + elif params.get( 'set_metadata_button', False ): + repository_clone_url = generate_clone_url( trans, repository ) + # In case metadata was previously generated for this repository, we'll check to see if it has information needed for the tool_section_dict. + metadata = repository.metadata + if 'tool_panel_section' in metadata: + # For backward compatibility we have to handle 2 types of dictionaries. In the past, all repository tools had to be installed into + # a single ToolSection (or outside of any sections) in the tool panel. In this case. the dictionary looks like this. + # { id: , version : , name : } + # Later, each tool from a repository could be installed into the tool panel inside or outside a specified ToolSection in the tool panel. + if panel_entry_per_tool( metadata[ 'tool_panel_section' ] ): + # { : { tool_config : , id: , version : , name : }} + tool_panel_dict = metadata[ 'tool_panel_section' ] + metadata_dict = generate_metadata( trans.app.toolbox, relative_install_dir, repository_clone_url, tool_panel_dict=tool_panel_dict ) + else: + # { id: , version : , name : } + tool_section_dict = metadata[ 'tool_panel_section' ] + metadata_dict = generate_metadata( trans.app.toolbox, relative_install_dir, repository_clone_url, tool_section_dict=tool_section_dict ) + else: + # Not sure if we'll ever reach here, but just in case... + tool_section_dict = dict( id='', version='', name='' ) + metadata_dict = generate_metadata( trans.app.toolbox, relative_install_dir, repository_clone_url, tool_section_dict=tool_section_dict ) + if metadata_dict: + repository.metadata = metadata_dict + trans.sa_session.add( repository ) + trans.sa_session.flush() + message = "Repository metadata has been reset." return trans.fill_template( '/admin/tool_shed_repository/manage_repository.mako', repository=repository, - description=repository.description, + description=description, repo_files_dir=repo_files_dir, message=message, status=status ) @web.expose @web.require_admin - def deactivate_or_uninstall_repository( self, trans, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - remove_from_disk = params.get( 'remove_from_disk', '' ) - remove_from_disk_checked = CheckboxField.is_checked( remove_from_disk ) + def reinstall_repository( self, trans, **kwd ): repository = get_repository( trans, kwd[ 'id' ] ) - shed_tool_conf, tool_path, relative_install_dir = self.__get_tool_path_and_relative_install_dir( trans, repository ) - if params.get( 'deactivate_or_uninstall_repository_button', False ): - metadata = repository.metadata - if repository.includes_tools: - # Deactivate repository tools. - repository_tools_tups = get_repository_tools_tups( trans.app, metadata ) - # Generate the list of tool panel keys derived from the tools included in the repository. - repository_tool_panel_keys = [] - if repository_tools_tups: - repository_tool_panel_keys = [ 'tool_%s' % repository_tools_tup[ 1 ] for repository_tools_tup in repository_tools_tups ] - tool_panel_section = metadata[ 'tool_panel_section' ] - section_id = tool_panel_section[ 'id' ] - if section_id in [ '' ]: - # If the repository includes tools, they were loaded into the tool panel outside of any sections. - tool_section = None - self.__remove_tools_from_tool_panel( trans, repository_tool_panel_keys ) - else: - # If the repository includes tools, they were loaded into the tool panel inside a section. - section_key = 'section_%s' % str( section_id ) - if section_key in trans.app.toolbox.tool_panel: - tool_section = trans.app.toolbox.tool_panel[ section_key ] - self.__remove_tools_from_tool_panel( trans, repository_tool_panel_keys, tool_section=tool_section, section_key=section_key ) + no_changes = kwd.get( 'no_changes', '' ) + no_changes_checked = CheckboxField.is_checked( no_changes ) + shed_tool_conf, tool_path, relative_install_dir = self.__get_tool_panel_config_tool_path_install_dir( trans, repository ) + current_working_dir = os.getcwd() + repository_clone_url = generate_clone_url( trans, repository ) + clone_dir = os.path.join( tool_path, self.__generate_tool_path( repository_clone_url, repository.installed_changeset_revision ) ) + relative_install_dir = os.path.join( clone_dir, repository.name ) + returncode, tmp_name = clone_repository( repository.name, clone_dir, current_working_dir, repository_clone_url ) + if returncode == 0: + returncode, tmp_name = update_repository( current_working_dir, relative_install_dir, repository.installed_changeset_revision ) + if returncode == 0: + # Get the location in the tool panel in which the tool was originally loaded. + metadata = repository.metadata + if 'tool_panel_section' in metadata: + if panel_entry_per_tool( metadata[ 'tool_panel_section' ] ): + # { : { tool_config : , id: , version : , name : }} + tool_panel_dict = metadata[ 'tool_panel_section' ] + # TODO: Fix this to handle the case where the tools are distributed across in more than 1 ToolSection. The + # following assumes everything was loaded into 1 section (or no section) in the tool panel. + tool_section_dict = tool_panel_dict[ tool_panel_dict.keys()[ 0 ] ] else: - # The tool panel section could not be found, so handle deactivating tools - # as if they were loaded into the tool panel outside of any sections. + # { id: , version : , name : } + tool_section_dict = metadata[ 'tool_panel_section' ] + tool_section = generate_tool_section_element_from_dict( tool_section_dict ) + tool_panel_dict = generate_tool_panel_dict_for_repository_tools( metadata, tool_section=tool_section ) + original_section_id = tool_section_dict[ 'id' ] + original_section_name = tool_section_dict[ 'name' ] + else: + tool_panel_dict = generate_tool_panel_dict_for_repository_tools( metadata ) + original_section_id = '' + original_section_name = '' + if no_changes_checked: + if original_section_id in [ '' ]: tool_section = None - self.__remove_tools_from_tool_panel( trans, repository_tool_panel_keys ) - repository_clone_url = self.__generate_clone_url( trans, repository ) - # The repository is either being deactivated or uninstalled, so handle tool panel alterations accordingly. - # If the repository is being uninstalled, the appropriate tools or tool sections will be removed from the - # appropriate tool config file on disk. - alter_tool_panel( app=trans.app, - repository_name=repository.name, - repository_clone_url=repository_clone_url, - changeset_revision=repository.installed_changeset_revision, - repository_tools_tups=repository_tools_tups, - tool_section=tool_section, - shed_tool_conf=shed_tool_conf, - tool_path=tool_path, - owner=repository.owner, - new_install=False, - deactivate=True, - uninstall=remove_from_disk_checked ) - if repository.includes_datatypes: - # Deactivate proprietary datatypes. - load_datatype_items( trans.app, repository, relative_install_dir, deactivate=True ) - if remove_from_disk_checked: - # Remove the repository from disk. - try: - shutil.rmtree( relative_install_dir ) - log.debug( "Removed repository installation directory: %s" % str( relative_install_dir ) ) - except Exception, e: - log.debug( "Error removing repository installation directory %s: %s" % ( str( relative_install_dir ), str( e ) ) ) - repository.uninstalled = True - repository.deleted = True - trans.sa_session.add( repository ) - trans.sa_session.flush() - if remove_from_disk_checked: - message = 'The repository named %s has been uninstalled.' % repository.name - else: - message = 'The repository named %s has been deactivated.' % repository.name - status = 'done' - return trans.response.send_redirect( web.url_for( controller='admin_toolshed', - action='browse_repositories', - message=message, - status=status ) ) - remove_from_disk_check_box = CheckboxField( 'remove_from_disk', checked=remove_from_disk_checked ) - return trans.fill_template( '/admin/tool_shed_repository/deactivate_or_uninstall_repository.mako', - repository=repository, - remove_from_disk_check_box=remove_from_disk_check_box, - message=message, - status=status ) - def __remove_tools_from_tool_panel( self, trans, repository_tool_panel_keys, tool_section=None, section_key=None ): - # Delete tools loaded into the tool panel by altering the in-memory tool_panel dictionary appropriately. If the - # tool_section is not received, the tools were loaded into the tool_panel dictionary outside of any sections. - # Otherwise all tools were loaded into the received tool_section. - if tool_section: - # The tool_section.elems dictionary looks something like: - # {'tool_gvk.bx.psu.edu:9009/repos/test/filter/Filter1/1.0.1': } - for item_id, item in tool_section.elems.items(): - if item_id in repository_tool_panel_keys: - del tool_section.elems[ item_id ] - if not tool_section.elems: - del trans.app.toolbox.tool_panel[ section_key ] - else: - # The tool panel looks something like: - # {'tool_gvk.bx.psu.edu:9009/repos/test/blast2go/blast2go/0.0.2': } - for item_id, item in trans.app.toolbox.tool_panel.items(): - if item_id in repository_tool_panel_keys: - del trans.app.toolbox.tool_panel[ item_id ] + else: + section_key = 'section_%s' % str( original_section_id ) + if section_key in trans.app.toolbox.tool_panel: + tool_section = trans.app.toolbox.tool_panel[ section_key ] + else: + # The section in which the tool was originally loaded used to be in the tool panel, but no longer is. + elem = Element( 'section' ) + elem.attrib[ 'name' ] = original_section_name + elem.attrib[ 'id' ] = original_section_id + elem.attrib[ 'version' ] = '' + tool_section = tools.ToolSection( elem ) + trans.app.toolbox.tool_panel[ section_key ] = tool_section + else: + # The user elected to change the tool panel section to contain the tools. + new_tool_panel_section = kwd.get( 'new_tool_panel_section', '' ) + tool_panel_section = kwd.get( 'tool_panel_section', '' ) + if new_tool_panel_section: + section_id = new_tool_panel_section.lower().replace( ' ', '_' ) + # Update each tool_section dictionary in tool_panel_dict with the new section attributes. + for guid, tool_section_dict in tool_panel_dict.items(): + tool_section_dict[ 'id' ] = section_id + tool_section_dict[ 'name' ] = new_tool_panel_section + tool_section_dict[ 'version' ] = '' + tool_panel_dict[ guid ] = tool_section_dict + new_section_key = 'section_%s' % str( section_id ) + if new_section_key in trans.app.toolbox.tool_panel: + # Appending a tool to an existing section in trans.app.toolbox.tool_panel + log.debug( "Appending to tool panel section: %s" % new_tool_panel_section ) + tool_section = trans.app.toolbox.tool_panel[ new_section_key ] + else: + # Appending a new section to trans.app.toolbox.tool_panel + log.debug( "Loading new tool panel section: %s" % new_tool_panel_section ) + elem = Element( 'section' ) + elem.attrib[ 'name' ] = new_tool_panel_section + elem.attrib[ 'id' ] = section_id + elem.attrib[ 'version' ] = '' + tool_section = tools.ToolSection( elem ) + trans.app.toolbox.tool_panel[ new_section_key ] = tool_section + elif tool_panel_section: + section_key = 'section_%s' % tool_panel_section + tool_section = trans.app.toolbox.tool_panel[ section_key ] + else: + # Update each tool_section dictionary in tool_panel_dict in case the tools used to be contained in a panel section + # but are now being moved outside of any panel sections. + for guid, tool_section_dict in tool_panel_dict.items(): + tool_section_dict[ 'id' ] = '' + tool_section_dict[ 'name' ] = '' + tool_section_dict[ 'version' ] = '' + tool_panel_dict[ guid ] = tool_section_dict + tool_section = None + tool_shed_repository, metadata_dict = load_repository_contents( trans, + repository_name=repository.name, + description=repository.description, + owner=repository.owner, + changeset_revision=repository.installed_changeset_revision, + tool_path=tool_path, + repository_clone_url=repository_clone_url, + relative_install_dir=relative_install_dir, + current_working_dir=current_working_dir, + tmp_name=tmp_name, + tool_panel_dict=tool_panel_dict, + tool_shed=repository.tool_shed, + tool_section=tool_section, + shed_tool_conf=shed_tool_conf, + new_install=True ) + repository.uninstalled = False + repository.deleted = False + trans.sa_session.add( repository ) + trans.sa_session.flush() + message = 'The %s repository has been reinstalled.' % repository.name + status = 'done' + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='browse_repositories', + message=message, + status=status ) ) @web.expose @web.require_admin def reselect_tool_panel_section( self, trans, **kwd ): + """ + INSIDE SECTION: + "{"tool_panel_section": {"gvk.bx.psu.edu:9009/repos/test/grouping/Grouping1/1.9.0": {"id": "group", + "name": "Group", + "tool_config": "grouping.xml", + "version": ""}}, + "tools": [{"description": "data by a column and perform aggregate operation on other columns.", + "guid": "gvk.bx.psu.edu:9009/repos/test/grouping/Grouping1/1.9.0", + "id": "Grouping1", + "name": "Group", + "requirements": [{"fabfile": null, "method": null, "name": "rpy", "type": "python-module", "version": null}], + "tests": [{"inputs": [["input1", "1.bed", {"children": [], "value": "1.bed"}], ["groupcol", "1", {"children": [], "value": "1"}], ["ignorecase", "true", {"children": [], "value": "true"}], ["optype", "mean", {"children": [], "value": "mean"}], ["opcol", "2", {"children": [], "value": "2"}], ["opround", "no", {"children": [], "value": "no"}]], "name": "Test-1", "outputs": [["out_file1", "groupby_out1.dat", {"assert_list": null, "compare": "diff", "delta": 10000, "extra_files": [], "lines_diff": 0, "sort": false}]], "required_files": [["1.bed", {"children": [], "value": "1.bed"}]]}, {"inputs": [["input1", "1.tabular", {"children": [], "value": "1.tabular"}], ["groupcol", "1", {"children": [], "value": "1"}], ["ignorecase", "true", {"children": [], "value": "true"}], ["optype", "mean", {"children": [], "value": "mean"}], ["opcol", "2", {"children": [], "value": "2"}], ["opround", "no", {"children": [], "value": "no"}]], "name": "Test-2", "outputs": [["out_file1", "groupby_out2.dat", {"assert_list": null, "compare": "diff", "delta": 10000, "extra_files": [], "lines_diff": 0, "sort": false}]], "required_files": [["1.tabular", {"children": [], "value": "1.tabular"}]]}], + "tool_config": "../shed_tools/gvk.bx.psu.edu/repos/test/grouping/935f00105de8/grouping/grouping.xml", + "version": "1.9.0", + "version_string_cmd": null}]}" + OUTSIDE SECTION: + "{"tool_panel_section": {"gvk.bx.psu.edu:9009/repos/test/filter/Filter1/1.0.1": {"id": "", + "name": "", + "tool_config": "filtering.xml", + "version": ""}}, + "tools": [{"description": "data on any column using simple expressions", + "guid": "gvk.bx.psu.edu:9009/repos/test/filter/Filter1/1.0.1", + "id": "Filter1", + "name": "Filter", + "requirements": [], + "tests": [{"inputs": [["input", "1.bed", {"children": [], "value": "1.bed"}], ["cond", "c1=='chr22'", {"children": [], "value": "c1=='chr22'"}]], "name": "Test-1", "outputs": [["out_file1", "filter1_test1.bed", {"assert_list": null, "compare": "diff", "delta": 10000, "extra_files": [], "lines_diff": 0, "sort": false}]], "required_files": [["1.bed", {"children": [], "value": "1.bed"}]]}, {"inputs": [["input", "7.bed", {"children": [], "value": "7.bed"}], ["cond", "c1=='chr1' and c3-c2>=2000 and c6=='+'", {"children": [], "value": "c1=='chr1' and c3-c2>=2000 and c6=='+'"}]], "name": "Test-2", "outputs": [["out_file1", "filter1_test2.bed", {"assert_list": null, "compare": "diff", "delta": 10000, "extra_files": [], "lines_diff": 0, "sort": false}]], "required_files": [["7.bed", {"children": [], "value": "7.bed"}]]}], + "tool_config": "../shed_tools/gvk.bx.psu.edu/repos/test/filter/b2df18d723c5/filter/filtering.xml", + "version": "1.0.1", + "version_string_cmd": null}]}" + """ repository = get_repository( trans, kwd[ 'id' ] ) # Get the location in the tool panel in which the tool was originally loaded. metadata = repository.metadata - tool_panel_section = metadata[ 'tool_panel_section' ] - original_section_name = tool_panel_section[ 'name' ] + if 'tool_panel_section' in metadata: + tool_panel_dict = metadata[ 'tool_panel_section' ] + if panel_entry_per_tool( tool_panel_dict ): + # { : { tool_config : , id: , version : , name : }} + # TODO: Fix this to handle the case where the tools are distributed across in more than 1 ToolSection. The + # following assumes everything was loaded into 1 section (or no section) in the tool panel. + tool_panel_dict = tool_panel_dict[ tool_panel_dict.keys()[ 0 ] ] + original_section_name = tool_panel_dict[ 'name' ] + else: + # { id: , version : , name : } + original_section_name = tool_panel_dict[ 'name' ] + else: + original_section_name = '' tool_panel_section_select_field = build_tool_panel_section_select_field( trans ) no_changes_check_box = CheckboxField( 'no_changes', checked=True ) if original_section_name: @@ -453,252 +700,35 @@ class AdminToolshed( AdminGalaxy ): status=status ) @web.expose @web.require_admin - def activate_or_reinstall_repository( self, trans, **kwd ): + def set_tool_versions( self, trans, **kwd ): + # Get the tool_versions from the tool shed for each tool in the installed change set. repository = get_repository( trans, kwd[ 'id' ] ) - no_changes = kwd.get( 'no_changes', '' ) - no_changes_checked = CheckboxField.is_checked( no_changes ) - shed_tool_conf, tool_path, relative_install_dir = self.__get_tool_path_and_relative_install_dir( trans, repository ) - uninstalled = repository.uninstalled - if uninstalled: - if repository.dist_to_shed: - href = 'Repository Actions menu. " + status = 'error' + shed_tool_conf, tool_path, relative_install_dir = self.__get_tool_panel_config_tool_path_install_dir( trans, repository ) repo_files_dir = os.path.abspath( os.path.join( relative_install_dir, repository.name ) ) - if params.get( 'edit_repository_button', False ): - if description != repository.description: - repository.description = description - trans.sa_session.add( repository ) - trans.sa_session.flush() - message = "The repository information has been updated." - elif params.get( 'set_metadata_button', False ): - repository_clone_url = self.__generate_clone_url( trans, repository ) - # In case metadata was previously generated for this repository, we'll - # check to see if it has information needed for the tool_section_dict. - metadata = repository.metadata - if 'tool_panel_section' in metadata: - tool_section_dict = metadata[ 'tool_panel_section' ] - metadata_dict = generate_metadata( trans.app.toolbox, relative_install_dir, repository_clone_url, tool_section_dict=tool_section_dict ) - if metadata_dict: - repository.metadata = metadata_dict - trans.sa_session.add( repository ) - trans.sa_session.flush() - message = "Repository metadata has been reset." return trans.fill_template( '/admin/tool_shed_repository/manage_repository.mako', repository=repository, - description=description, + description=repository.description, repo_files_dir=repo_files_dir, message=message, status=status ) @web.expose @web.require_admin - def check_for_updates( self, trans, **kwd ): - # Send a request to the relevant tool shed to see if there are any updates. - repository = get_repository( trans, kwd[ 'id' ] ) - tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) - url = '%s/repository/check_for_updates?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ - ( tool_shed_url, url_for( '/', qualified=True ), repository.name, repository.owner, repository.changeset_revision ) - return trans.response.send_redirect( url ) - @web.expose - @web.require_admin def update_to_changeset_revision( self, trans, **kwd ): """Update a cloned repository to the latest revision possible.""" params = util.Params( kwd ) @@ -710,12 +740,25 @@ class AdminToolshed( AdminGalaxy ): changeset_revision = params.get( 'changeset_revision', None ) latest_changeset_revision = params.get( 'latest_changeset_revision', None ) repository = get_repository_by_shed_name_owner_changeset_revision( trans.app, tool_shed_url, name, owner, changeset_revision ) + # Get the location in the tool panel in which the tool was originally loaded. + metadata = repository.metadata + if 'tool_panel_section' in metadata: + if panel_entry_per_tool( metadata[ 'tool_panel_section' ] ): + # { : { tool_config : , id: , version : , name : }} + tool_panel_dict = metadata[ 'tool_panel_section' ] + else: + # { id: , version : , name : } + tool_section_dict = metadata[ 'tool_panel_section' ] + tool_section = generate_tool_section_element_from_dict( tool_section_dict ) + tool_panel_dict = generate_tool_panel_dict_for_repository_tools( metadata, tool_section=tool_section ) + else: + tool_panel_dict = generate_tool_panel_dict_for_repository_tools( metadata ) if changeset_revision and latest_changeset_revision: if changeset_revision == latest_changeset_revision: message = "The cloned tool shed repository named '%s' is current (there are no updates available)." % name else: current_working_dir = os.getcwd() - shed_tool_conf, tool_path, relative_install_dir = self.__get_tool_path_and_relative_install_dir( trans, repository ) + shed_tool_conf, tool_path, relative_install_dir = self.__get_tool_panel_config_tool_path_install_dir( trans, repository ) if relative_install_dir: repo_files_dir = os.path.join( relative_install_dir, name ) returncode, tmp_name = pull_repository( current_working_dir, repo_files_dir, name ) @@ -725,7 +768,7 @@ class AdminToolshed( AdminGalaxy ): # Update the repository metadata. repository_clone_url = os.path.join( tool_shed_url, 'repos', owner, name ) tool_shed = clean_tool_shed_url( tool_shed_url ) - tool_shed_repository, metadata_dict = load_repository_contents( app=trans.app, + tool_shed_repository, metadata_dict = load_repository_contents( trans, repository_name=name, description=repository.description, owner=owner, @@ -735,11 +778,11 @@ class AdminToolshed( AdminGalaxy ): relative_install_dir=relative_install_dir, current_working_dir=current_working_dir, tmp_name=tmp_name, + tool_panel_dict=tool_panel_dict, tool_shed=tool_shed, tool_section=None, shed_tool_conf=None, - new_install=False, - dist_to_shed=False ) + new_install=False ) # Update the repository changeset_revision in the database. repository.changeset_revision = latest_changeset_revision repository.update_available = False @@ -790,20 +833,38 @@ class AdminToolshed( AdminGalaxy ): metadata=metadata, message=message, status=status ) - def __get_tool_path_and_relative_install_dir( self, trans, repository ): - # Return both the tool_path configured in the relative shed_tool_conf and - # the relative path to the directory where the repository is installed. - tool_shed = clean_tool_shed_url( repository.tool_shed ) - partial_install_dir = '%s/repos/%s/%s/%s' % ( tool_shed, repository.owner, repository.name, repository.installed_changeset_revision ) - # Get the relative tool installation paths from each of the shed tool configs. - relative_install_dir = None - for shed_tool_conf_dict in trans.app.toolbox.shed_tool_confs: - shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ] - tool_path = shed_tool_conf_dict[ 'tool_path' ] - relative_install_dir = os.path.join( tool_path, partial_install_dir ) - if os.path.isdir( relative_install_dir ): - break - return shed_tool_conf, tool_path, relative_install_dir + def __add_tools_to_tool_panel( self, trans, repository, repository_tools_tups, tool_section=None, section_key=None ): + # Load tools. + if tool_section: + elems = tool_section.elems + for repository_tools_tup in repository_tools_tups: + relative_path, guid, tool = repository_tools_tup + tool.tool_shed = repository.tool_shed + tool.repository_name = repository.name + tool.repository_owner = repository.owner + tool.installed_changeset_revision = repository.installed_changeset_revision + tool.guid = guid + # Set the tool's old_id to the id used before the tool shed existed. + tool.old_id = tool.id + # Set the tool's id to the tool shed guid. + tool.id = guid + if tool_section: + if tool.id not in elems: + elems[ 'tool_%s' % tool.id ] = tool + log.debug( "Reactivated tool id: %s, version: %s" % ( tool.id, tool.version ) ) + else: + if tool.id not in trans.app.toolbox.tools_by_id: + # Allow for the same tool to be loaded into multiple places in the tool panel. + trans.app.toolbox.tools_by_id[ tool.id ] = tool + trans.app.toolbox.tool_panel[ 'tool_%s' % tool.id ] = tool + log.debug( "Reactivated tool id: %s, version: %s" % ( tool.id, tool.version ) ) + if tool_section: + trans.app.toolbox.tool_panel[ section_key ] = tool_section + log.debug( "Appended reactivated tool to section: %s" % tool_section.name ) + def __generate_clone_url( self, trans, repository ): + """Generate the URL for cloning a repository.""" + tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) + return '%s/repos/%s/%s' % ( tool_shed_url, repository.owner, repository.name ) def __generate_tool_path( self, repository_clone_url, changeset_revision ): """ Generate a tool path that guarantees repositories with the same name will always be installed @@ -818,10 +879,77 @@ class AdminToolshed( AdminGalaxy ): repo_path = items[ 1 ] tool_shed_url = clean_tool_shed_url( tool_shed_url ) return '%s/repos%s/%s' % ( tool_shed_url, repo_path, changeset_revision ) - def __generate_clone_url( self, trans, repository ): - """Generate the URL for cloning a repository.""" - tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) - return '%s/repos/%s/%s' % ( tool_shed_url, repository.owner, repository.name ) + def __get_tool_panel_config_tool_path_install_dir( self, trans, repository ): + # Return shed-related tool panel config, the tool_path configured in it, and the relative path to the directory where the + # repository is installed. This method assumes all repository tools are defined in a single shed-related tool panel config. + tool_shed = clean_tool_shed_url( repository.tool_shed ) + partial_install_dir = '%s/repos/%s/%s/%s' % ( tool_shed, repository.owner, repository.name, repository.installed_changeset_revision ) + # Get the relative tool installation paths from each of the shed tool configs. + relative_install_dir = None + for shed_tool_conf_dict in trans.app.toolbox.shed_tool_confs: + shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ] + if repository.dist_to_shed: + # The repository is owned by devteam and contains tools migrated from the Galaxy distribution to the tool shed, so + # the reserved tool panel config is migrated_tools_conf.xml, to which trans.app.config.migrated_tools_config refers. + if shed_tool_conf == trans.app.config.migrated_tools_config: + tool_path = shed_tool_conf_dict[ 'tool_path' ] + relative_install_dir = os.path.join( tool_path, partial_install_dir ) + if tool_path and relative_install_dir: + return shed_tool_conf, tool_path, relative_install_dir + elif repository.uninstalled: + # Since the repository is uninstalled we don't know what tool panel config was originally used to + # define the tools in the repository, so we'll just make sure not to use the reserved migrated_tools_conf.xml. + if shed_tool_conf != trans.app.config.migrated_tools_config: + tool_path = shed_tool_conf_dict[ 'tool_path' ] + relative_install_dir = os.path.join( tool_path, partial_install_dir ) + if tool_path and relative_install_dir: + return shed_tool_conf, tool_path, relative_install_dir + else: + if repository.includes_tools: + # We'll check config_elems until we find an element that matches one of the tools in the repository's metadata. + metadata = repository.metadata + tool_dict = metadata[ 'tools' ][ 0 ] + config_elems = shed_tool_conf_dict[ 'config_elems' ] + if config_elems: + tool_path, relative_install_dir = self.__get_tool_path_install_dir( partial_install_dir, + shed_tool_conf_dict, + tool_dict, + config_elems ) + if tool_path and relative_install_dir: + return shed_tool_conf, tool_path, relative_install_dir + else: + # Parse the tool panel config since we have no in-memory config_elems (not sure if this will ever occur). + tool_panel_config = shed_tool_conf_dict[ 'config_filename' ] + tree = util.parse_xml( tool_panel_config ) + root = tree.getroot() + tool_path, relative_install_dir = self.__get_tool_path_install_dir( partial_install_dir, + shed_tool_conf_dict, + tool_dict, + root ) + if tool_path and relative_install_dir: + return shed_tool_conf, tool_path, relative_install_dir + else: + # Nothing will be loaded into the tool panel, so look for the installed repository on disk. + tool_path = shed_tool_conf_dict[ 'tool_path' ] + relative_install_dir = os.path.join( tool_path, partial_install_dir ) + if tool_path and relative_install_dir and os.path.isdir( relative_install_dir ): + return shed_tool_conf, tool_path, relative_install_dir + return None, None, None + def __get_tool_path_install_dir( self, partial_install_dir, shed_tool_conf_dict, tool_dict, config_elems ): + for elem in config_elems: + if elem.tag == 'tool': + if elem.get( 'guid' ) == tool_dict[ 'guid' ]: + tool_path = shed_tool_conf_dict[ 'tool_path' ] + relative_install_dir = os.path.join( tool_path, partial_install_dir ) + return tool_path, relative_install_dir + elif elem.tag == 'section': + for section_elem in elem: + if section_elem.tag == 'tool': + if section_elem.get( 'guid' ) == tool_dict[ 'guid' ]: + tool_path = shed_tool_conf_dict[ 'tool_path' ] + relative_install_dir = os.path.join( tool_path, partial_install_dir ) + return tool_path, relative_install_dir + return None, None ## ---- Utility methods ------------------------------------------------------- @@ -830,11 +958,12 @@ def build_shed_tool_conf_select_field( trans ): options = [] for shed_tool_conf_dict in trans.app.toolbox.shed_tool_confs: shed_tool_conf_filename = shed_tool_conf_dict[ 'config_filename' ] - if shed_tool_conf_filename.startswith( './' ): - option_label = shed_tool_conf_filename.replace( './', '', 1 ) - else: - option_label = shed_tool_conf_filename - options.append( ( option_label, shed_tool_conf_filename ) ) + if shed_tool_conf_filename != trans.app.config.migrated_tools_config: + if shed_tool_conf_filename.startswith( './' ): + option_label = shed_tool_conf_filename.replace( './', '', 1 ) + else: + option_label = shed_tool_conf_filename + options.append( ( option_label, shed_tool_conf_filename ) ) select_field = SelectField( name='shed_tool_conf' ) for option_tup in options: select_field.add_option( option_tup[0], option_tup[1] ) diff --git a/lib/galaxy/webapps/community/model/mapping.py b/lib/galaxy/webapps/community/model/mapping.py index 8b87e5f3d0f..7f476706410 100644 --- a/lib/galaxy/webapps/community/model/mapping.py +++ b/lib/galaxy/webapps/community/model/mapping.py @@ -151,8 +151,7 @@ Tag.table = Table( "tag", metadata, Column( "name", TrimmedString(255) ), UniqueConstraint( "name" ) ) -# With the tables defined we can define the mappers and setup the -# relationships between the model objects. +# With the tables defined we can define the mappers and setup the relationships between the model objects. assign_mapper( context, User, User.table, properties=dict( active_repositories=relation( Repository, primaryjoin=( ( Repository.table.c.user_id == User.table.c.id ) & ( not_( Repository.table.c.deleted ) ) ), order_by=( Repository.table.c.name ) ), galaxy_sessions=relation( GalaxySession, order_by=desc( GalaxySession.table.c.update_time ) ) ) ) diff --git a/migrated_tools_conf.xml.sample b/migrated_tools_conf.xml.sample new file mode 100644 index 00000000000..64252819052 --- /dev/null +++ b/migrated_tools_conf.xml.sample @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/run.sh b/run.sh index 4bfb8b5d177..40e8fd911f3 100755 --- a/run.sh +++ b/run.sh @@ -6,8 +6,10 @@ python ./scripts/check_python.py [ $? -ne 0 ] && exit 1 SAMPLES=" + community_wsgi.ini.sample datatypes_conf.xml.sample external_service_types_conf.xml.sample + migrated_tools_conf.xml.sample reports_wsgi.ini.sample shed_tool_conf.xml.sample tool_conf.xml.sample diff --git a/scripts/manage_tools.py b/scripts/manage_tools.py new file mode 100644 index 00000000000..91a9bd199b2 --- /dev/null +++ b/scripts/manage_tools.py @@ -0,0 +1,58 @@ +import sys, os.path, logging + +new_path = [ os.path.join( os.getcwd(), "lib" ) ] +new_path.extend( sys.path[1:] ) # remove scripts/ from the path +sys.path = new_path + +from galaxy import eggs + +import pkg_resources +pkg_resources.require( "sqlalchemy-migrate" ) + +from migrate.versioning.shell import main +from ConfigParser import SafeConfigParser + +log = logging.getLogger( __name__ ) + +config_file = 'universe_wsgi.ini' +if '-c' in sys.argv: + pos = sys.argv.index( '-c' ) + sys.argv.pop( pos ) + config_file = sys.argv.pop( pos ) +if not os.path.exists( config_file ): + print "Galaxy config file does not exist (hint: use '-c config.ini' for non-standard locations): %s" % config_file + sys.exit( 1 ) +repo = 'lib/galaxy/tool_shed/migrate' + +cp = SafeConfigParser() +cp.read( config_file ) + +if config_file == 'universe_wsgi.ini.sample' and 'GALAXY_TEST_DBURI' in os.environ: + # Running functional tests. + db_url = os.environ[ 'GALAXY_TEST_DBURI' ] +elif cp.has_option( "app:main", "database_connection" ): + db_url = cp.get( "app:main", "database_connection" ) +elif cp.has_option( "app:main", "database_file" ): + db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % cp.get( "app:main", "database_file" ) +else: + db_url = "sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE" + +dialect_to_egg = { + "sqlite" : "pysqlite>=2", + "postgres" : "psycopg2", + "mysql" : "MySQL_python" +} +dialect = ( db_url.split( ':', 1 ) )[0] +try: + egg = dialect_to_egg[dialect] + try: + pkg_resources.require( egg ) + log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) + except: + # If the module is in the path elsewhere (i.e. non-egg), it'll still load. + log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) +except KeyError: + # Let this go, it could possibly work with db's we don't support + log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) + +main( repository=repo, url=db_url ) diff --git a/scripts/migrate_tools/0002_tools.sh b/scripts/migrate_tools/0002_tools.sh new file mode 100644 index 00000000000..926aba9529d --- /dev/null +++ b/scripts/migrate_tools/0002_tools.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +cd `dirname $0`/../.. +python ./scripts/migrate_tools/migrate_tools.py 0002_tools.xml $@ diff --git a/scripts/migrate_tools/0002_tools.xml b/scripts/migrate_tools/0002_tools.xml new file mode 100644 index 00000000000..18e7198f478 --- /dev/null +++ b/scripts/migrate_tools/0002_tools.xml @@ -0,0 +1,113 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/scripts/migrate_tools/migrate_tools.py b/scripts/migrate_tools/migrate_tools.py new file mode 100644 index 00000000000..c8773741f8b --- /dev/null +++ b/scripts/migrate_tools/migrate_tools.py @@ -0,0 +1,26 @@ + +""" +This script will start up its own web application which includes an InstallManager (~/lib/galaxy/tool_shed/install_manager.py). +For each tool discovered missing, the tool shed repository that contains it will be installed on disk and a new entry will be +created for it in the migrated_tools_conf.xml file. These entries will be made so that the tool panel will be displayed the same +as it was before the tools were eliminated from the Galaxy distribution. The InstallManager will properly handle entries in +migrated_tools_conf.xml for tools outside tool panel sections as well as tools inside tool panel sections, depending upon the +layout of the local tool_conf.xml file. Entries will not be created in migrated_tools_conf.xml for tools included in the tool +shed repository but not defined in tool_conf.xml. +""" +import sys, os + +new_path = [ os.path.join( os.getcwd(), "lib" ) ] +# Remove scripts/ from the path. +new_path.extend( sys.path[1:] ) +sys.path = new_path + +from galaxy import eggs +from galaxy.tool_shed.migrate.common import * + +app = MigrateToolsApplication( sys.argv[ 1 ] ) +non_shed_tool_conf = app.install_manager.proprietary_tool_conf +print "\nThe installation process is finished. You should now remove entries for the installed tools from" +print "your file named %s and start your Galaxy server." % non_shed_tool_conf +app.shutdown() +sys.exit( 0 ) diff --git a/templates/admin/tool_shed_repository/reselect_tool_panel_section.mako b/templates/admin/tool_shed_repository/reselect_tool_panel_section.mako index c6cfa97e640..ee39e0496eb 100644 --- a/templates/admin/tool_shed_repository/reselect_tool_panel_section.mako +++ b/templates/admin/tool_shed_repository/reselect_tool_panel_section.mako @@ -8,7 +8,7 @@
Choose the tool panel section to contain the installed tools (optional)
-
+
${no_changes_check_box.get_html()} diff --git a/tool_shed_install.xml.sample b/tool_shed_install.xml.sample deleted file mode 100644 index 01e117f7e4f..00000000000 --- a/tool_shed_install.xml.sample +++ /dev/null @@ -1,116 +0,0 @@ - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- diff --git a/universe_wsgi.ini.sample b/universe_wsgi.ini.sample index f3c5afb6b1b..60625d4d171 100644 --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -136,22 +136,6 @@ paste.app_factory = galaxy.web.buildapp:app_factory # http://wiki.g2.bx.psu.edu/Admin/Config/Tool%20Dependencies #tool_dependency_dir = None -# Enable automatic installation of tools that used to be in the Galaxy -# distribution but are now in the main Galaxy tool shed. The tools -# that will be installed are configured in the config file named -# tool_shed_install.xml, which is located in the Galaxy install directory. -# Tools already installed will not be re-installed even if they are -# referenced in the tool_shed_install.xml file. -#enable_tool_shed_install = False -#tool_shed_install_config_file = tool_shed_install.xml - -# CRITICAL NOTE: the location in which the tools will be installed is the -# location pointed to by the "tool_path" attribute in the following file. -# The default location setting in shed_tool_conf.xml ("../shed_tools") may -# be problematic for some cluster environments, so make sure to change it -# if appropriate or use a different file name for the setting. -#install_tool_config_file = shed_tool_conf.xml - # Enable automatic polling of relative tool sheds to see if any updates # are available for installed repositories. Ideally only one Galaxy # server process should be able to check for repository updates. The