Re-engineer the process for defining valid tool shed repository changeset revisions and add support for automatically installing and building tool dependencies for tools included in tool shed repositories being installed into a local Galaxy instance. Tool dependencies will be uninstalled if the repository is uninstalled.

lib/galaxy/app.py

@@ -3,6 +3,7 @@ import sys, os, atexit
 from galaxy import config, jobs, util, tools, web
 import galaxy.tools.search
 import galaxy.tools.data
+import galaxy.tool_shed
 import galaxy.tool_shed.tool_shed_registry
 from galaxy.web import security
 import galaxy.model

lib/galaxy/model/__init__.py

@@ -2648,7 +2648,7 @@ class APIKeys( object ):
 class ToolShedRepository( object ):
     def __init__( self, id=None, create_time=None, tool_shed=None, name=None, description=None, owner=None, installed_changeset_revision=None,
                   changeset_revision=None, ctx_rev=None, metadata=None, includes_datatypes=False, update_available=False, deleted=False,
-                   uninstalled=False, dist_to_shed=False ):
+                  uninstalled=False, dist_to_shed=False ):
         self.id = id
         self.create_time = create_time
         self.tool_shed = tool_shed
@@ -2664,13 +2664,37 @@ class ToolShedRepository( object ):
         self.deleted = deleted
         self.uninstalled = uninstalled
         self.dist_to_shed = dist_to_shed
+    def repo_path( self, app ):        
+        tool_shed_url = self.tool_shed
+        if tool_shed_url.find( ':' ) > 0:
+            # Eliminate the port, if any, since it will result in an invalid directory name.
+            tool_shed_url = tool_shed_url.split( ':' )[ 0 ]
+        tool_shed = tool_shed_url.rstrip( '/' )
+        for index, shed_tool_conf_dict in enumerate( app.toolbox.shed_tool_confs ):
+            tool_path = shed_tool_conf_dict[ 'tool_path' ]
+            relative_path = os.path.join( tool_path, tool_shed, 'repos', self.owner, self.name, self.installed_changeset_revision )
+            if os.path.exists( relative_path ):
+                return relative_path
+        return None
     @property
     def includes_tools( self ):
         return self.metadata and 'tools' in self.metadata
     @property
+    def includes_tool_dependencies( self ):
+        return self.metadata and 'tool_dependencies' in self.metadata
+    @property
     def includes_workflows( self ):
         return self.metadata and 'workflows' in self.metadata
 
+class ToolDependency( object ):
+    def __init__( self, tool_shed_repository_id=None, installed_changeset_revision=None, name=None, version=None, type=None, uninstalled=False ):
+        self.tool_shed_repository_id = tool_shed_repository_id
+        self.installed_changeset_revision = installed_changeset_revision
+        self.name = name
+        self.version = version
+        self.type = type
+        self.uninstalled = uninstalled
+
 class ToolVersion( object ):
     def __init__( self, id=None, create_time=None, tool_id=None, tool_shed_repository=None ):
         self.id = id

lib/galaxy/model/mapping.py

@@ -390,6 +390,17 @@ ToolShedRepository.table = Table( "tool_shed_repository", metadata,
     Column( "uninstalled", Boolean, default=False ),
     Column( "dist_to_shed", Boolean, default=False ) )
 
+ToolDependency.table = Table( "tool_dependency", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "tool_shed_repository_id", Integer, ForeignKey( "tool_shed_repository.id" ), index=True, nullable=False ),
+    Column( "installed_changeset_revision", TrimmedString( 255 ) ),
+    Column( "name", TrimmedString( 255 ) ),
+    Column( "version", TrimmedString( 40 ) ),
+    Column( "type", TrimmedString( 40 ) ),
+    Column( "uninstalled", Boolean, default=False ) )
+
 ToolVersion.table = Table( "tool_version", metadata,
     Column( "id", Integer, primary_key=True ),
     Column( "create_time", DateTime, default=now ),
@@ -1667,7 +1678,12 @@ assign_mapper( context, Page, Page.table,
 assign_mapper( context, ToolShedRepository, ToolShedRepository.table,
     properties=dict( tool_versions=relation( ToolVersion,
                                              primaryjoin=( ToolShedRepository.table.c.id == ToolVersion.table.c.tool_shed_repository_id ),
-                                             backref='tool_shed_repository' ) ) )
+                                             backref='tool_shed_repository' ),
+                     tool_dependencies=relation( ToolDependency,
+                                                 primaryjoin=( ToolShedRepository.table.c.id == ToolDependency.table.c.tool_shed_repository_id ),
+                                                 backref='tool_shed_repository' ) ) )
+
+assign_mapper( context, ToolDependency, ToolDependency.table )
 
 assign_mapper( context, ToolVersion, ToolVersion.table )
 

lib/galaxy/model/migrate/versions/0099_add_tool_dependency_table.py

@@ -0,0 +1,49 @@
+"""
+Migration script to add the tool_dependency table.
+"""
+from sqlalchemy import *
+from sqlalchemy.orm import *
+from migrate import *
+from migrate.changeset import *
+import sys, logging
+from galaxy.model.custom_types import *
+from sqlalchemy.exc import *
+import datetime
+now = datetime.datetime.utcnow
+
+log = logging.getLogger( __name__ )
+log.setLevel( logging.DEBUG )
+handler = logging.StreamHandler( sys.stdout )
+format = "%(name)s %(levelname)s %(asctime)s %(message)s"
+formatter = logging.Formatter( format )
+handler.setFormatter( formatter )
+log.addHandler( handler )
+
+metadata = MetaData( migrate_engine )
+
+# New table to store information about cloned tool shed repositories.
+ToolDependency_table = Table( "tool_dependency", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "tool_shed_repository_id", Integer, ForeignKey( "tool_shed_repository.id" ), index=True, nullable=False ),
+    Column( "installed_changeset_revision", TrimmedString( 255 ) ),
+    Column( "name", TrimmedString( 255 ) ),
+    Column( "version", TrimmedString( 40 ) ),
+    Column( "type", TrimmedString( 40 ) ),
+    Column( "uninstalled", Boolean, default=False ) )
+
+def upgrade():
+    print __doc__
+    metadata.reflect()
+    try:
+        ToolDependency_table.create()
+    except Exception, e:
+        log.debug( "Creating tool_dependency table failed: %s" % str( e ) )
+
+def downgrade():
+    metadata.reflect()
+    try:
+        ToolDependency_table.drop()
+    except Exception, e:
+        log.debug( "Dropping tool_dependency table failed: %s" % str( e ) )

lib/galaxy/tool_shed/__init__.py

@@ -1,27 +1,47 @@
 """
 Classes encapsulating the management of repositories installed from Galaxy tool sheds.
 """
-import os, logging
-from galaxy.util.shed_util import *
+import os
+import galaxy.util.shed_util
 from galaxy.model.orm import *
 
-log = logging.getLogger(__name__)
+from galaxy import eggs
+import pkg_resources
+
+pkg_resources.require( 'elementtree' )
+from elementtree import ElementTree, ElementInclude
 
 class InstalledRepositoryManager( object ):
     def __init__( self, app ):
         self.app = app
         self.model = self.app.model
         self.sa_session = self.model.context.current
+        self.tool_configs = self.app.config.tool_configs
+        if self.app.config.migrated_tools_config not in self.tool_configs:
+            self.tool_configs.append( self.app.config.migrated_tools_config )
+    def get_repository_install_dir( self, tool_shed_repository ):
+        for tool_config in self.tool_configs:
+            tree = ElementTree.parse( tool_config )
+            root = tree.getroot()
+            ElementInclude.include( root )
+            tool_path = root.get( 'tool_path', None )
+            if tool_path:
+                tool_shed = galaxy.util.shed_util.clean_tool_shed_url( tool_shed_repository.tool_shed )
+                relative_path = os.path.join( tool_path,
+                                              tool_shed,
+                                              'repos',
+                                              tool_shed_repository.owner,
+                                              tool_shed_repository.name,
+                                              tool_shed_repository.installed_changeset_revision )
+                if os.path.exists( relative_path ):
+                    return relative_path
+        return None
     def load_proprietary_datatypes( self ):
         for tool_shed_repository in self.sa_session.query( self.model.ToolShedRepository ) \
                                                    .filter( and_( self.model.ToolShedRepository.table.c.includes_datatypes==True,
                                                                   self.model.ToolShedRepository.table.c.deleted==False ) ) \
                                                    .order_by( self.model.ToolShedRepository.table.c.id ):
-            metadata = tool_shed_repository.metadata
-            datatypes_config = metadata[ 'datatypes_config' ]
-            # We need the repository installation directory, which we can derive from the path to the datatypes config.
-            path_items = datatypes_config.split( 'repos' )
-            relative_install_dir = '%srepos/%s/%s/%s' % \
-                ( path_items[0], tool_shed_repository.owner, tool_shed_repository.name, tool_shed_repository.installed_changeset_revision )
-            load_datatype_items( self.app, tool_shed_repository, relative_install_dir )
+            relative_install_dir = self.get_repository_install_dir( tool_shed_repository )
+            if relative_install_dir:
+                galaxy.util.shed_util.load_datatype_items( self.app, tool_shed_repository, relative_install_dir )
            

lib/galaxy/tool_shed/encoding_util.py

@@ -0,0 +1,41 @@
+import binascii
+from galaxy.util.hash_util import *
+from galaxy.util.json import json_fix
+
+from galaxy import eggs
+import pkg_resources
+
+pkg_resources.require( "simplejson" )
+import simplejson
+
+encoding_sep = '__esep__'
+
+def tool_shed_decode( value ):
+    # Extract and verify hash
+    a, b = value.split( ":" )
+    value = binascii.unhexlify( b )
+    test = hmac_new( 'ToolShedAndGalaxyMustHaveThisSameKey', value )
+    assert a == test
+    # Restore from string
+    values = None
+    try:
+        values = simplejson.loads( value )
+    except Exception, e:
+        log.debug( "Decoding json value from tool shed threw exception: %s" % str( e ) )
+    if values is not None:
+        try:
+            return json_fix( values )
+        except Exception, e:
+            log.debug( "Fixing decoded json value from tool shed threw exception: %s" % str( e ) )
+            fixed_values = values
+    if values is None:
+        values = value
+    return values
+def tool_shed_encode( val ):
+    if isinstance( val, dict ):
+        value = simplejson.dumps( val )
+    else:
+        value = val
+    a = hmac_new( 'ToolShedAndGalaxyMustHaveThisSameKey', value )
+    b = binascii.hexlify( value )
+    return "%s:%s" % ( a, b )

lib/galaxy/tool_shed/install_manager.py

@@ -2,14 +2,12 @@
 Manage automatic installation of tools configured in the xxx.xml files in ~/scripts/migrate_tools (e.g., 0002_tools.xml).
 All of the tools were at some point included in the Galaxy distribution, but are now hosted in the main Galaxy tool shed.
 """
-import logging, urllib2
+import urllib2
 from galaxy.tools import ToolSection
 from galaxy.util.json import from_json_string, to_json_string
 from galaxy.util.shed_util import *
 from galaxy.util.odict import odict
 
-log = logging.getLogger( __name__ )
-
 class InstallManager( object ):
     def __init__( self, app, latest_migration_script_number, tool_shed_install_config, migrated_tools_config ):
         """
@@ -73,7 +71,7 @@ class InstallManager( object ):
                     # Tools outside of sections.
                     file_path = elem.get( 'file', None )
                     if file_path:
-                        path, name = os.path.split( file_path )
+                        name = strip_path( file_path )
                         if name in migrated_tool_configs:
                             if elem not in tool_panel_elems:
                                 tool_panel_elems.append( elem )
@@ -83,7 +81,7 @@ class InstallManager( object ):
                         if section_elem.tag == 'tool':
                             file_path = section_elem.get( 'file', None )
                             if file_path:
-                                path, name = os.path.split( file_path )
+                                name = strip_path( file_path )
                                 if name in migrated_tool_configs:
                                     # Append the section, not the tool.
                                     if elem not in tool_panel_elems:
@@ -100,7 +98,7 @@ class InstallManager( object ):
             if proprietary_tool_panel_elem.tag == 'tool':
                 # The proprietary_tool_panel_elem looks something like <tool file="emboss_5/emboss_antigenic.xml" />.
                 proprietary_tool_config = proprietary_tool_panel_elem.get( 'file' )
-                proprietary_path, proprietary_name = os.path.split( proprietary_tool_config )
+                proprietary_name = strip_path( proprietary_tool_config )
                 if tool_config == proprietary_name:
                     # The tool is loaded outside of any sections.
                     tool_sections.append( None )
@@ -112,15 +110,14 @@ class InstallManager( object ):
                     if section_elem.tag == 'tool':
                         # The section_elem looks something like <tool file="emboss_5/emboss_antigenic.xml" />.
                         proprietary_tool_config = section_elem.get( 'file' )
-                        proprietary_path, proprietary_name = os.path.split( proprietary_tool_config )
+                        proprietary_name = strip_path( proprietary_tool_config )
                         if tool_config == proprietary_name:
                             # The tool is loaded inside of the section_elem.
                             tool_sections.append( ToolSection( proprietary_tool_panel_elem ) )
                             if not is_displayed:
                                 is_displayed = True
         return is_displayed, tool_sections
-    def handle_repository_contents( self, current_working_dir, repository_clone_url, relative_install_dir, repository_elem, repository_name, description,
-                                    changeset_revision, ctx_rev ):
+    def handle_repository_contents( self, repository_clone_url, relative_install_dir, repository_elem, repository_name, description, changeset_revision, ctx_rev ):
         # Generate the metadata for the installed tool shed repository, among other things.  It is critical that the installed repository is
         # updated to the desired changeset_revision before metadata is set because the process for setting metadata uses the repository files on disk.
         # The values for the keys in each of the following dictionaries will be a list to allow for the same tool to be displayed in multiple places
@@ -136,7 +133,7 @@ class InstallManager( object ):
                 tool_panel_dict_for_tool_config = generate_tool_panel_dict_for_tool_config( guid, tool_config, tool_sections=tool_sections )
                 for k, v in tool_panel_dict_for_tool_config.items():
                     tool_panel_dict_for_display[ k ] = v
-        metadata_dict = generate_metadata( self.toolbox, relative_install_dir, repository_clone_url )
+        metadata_dict = generate_metadata_using_disk_files( self.toolbox, relative_install_dir, repository_clone_url )
         # Add a new record to the tool_shed_repository table if one doesn't already exist.  If one exists but is marked
         # deleted, undelete it.  It is critical that this happens before the call to add_to_tool_panel() below because
         # tools will not be properly loaded if the repository is marked deleted.
@@ -150,6 +147,7 @@ class InstallManager( object ):
                                                                       metadata_dict,
                                                                       dist_to_shed=True )
         if 'tools' in metadata_dict:
+            work_dir = make_tmp_directory()
             repository_tools_tups = get_repository_tools_tups( self.app, metadata_dict )
             if repository_tools_tups:
                 sample_files = metadata_dict.get( 'sample_files', [] )
@@ -159,11 +157,20 @@ class InstallManager( object ):
                 repository_tools_tups, sample_files_copied = handle_missing_index_file( self.app, self.tool_path, sample_files, repository_tools_tups )
                 # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance.
                 copy_sample_files( self.app, sample_files, sample_files_copied=sample_files_copied )
-                if 'tool_dependencies_config' in metadata_dict:
+                if 'tool_dependencies' in metadata_dict:
+                    # Get the tool_dependencies.xml file from the repository.
+                    tool_dependencies_config = get_config_from_repository( self.app,
+                                                                           'tool_dependencies.xml',
+                                                                           tool_shed_repository,
+                                                                           changeset_revision,
+                                                                           work_dir )
                     # Install tool dependencies.
-                    status, message = handle_tool_dependencies( self.app, repository_clone_url, metadata_dict[ 'tool_dependencies_config' ] )
+                    status, message = handle_tool_dependencies( app=self.app,
+                                                                tool_shed_repository=tool_shed_repository,
+                                                                installed_changeset_revision=changeset_revision,
+                                                                tool_dependencies_config=tool_dependencies_config )
                     if status != 'ok' and message:
-                        print 'The following error occurred while installing tool dependencies:'
+                        print 'The following error occurred from the InstallManager while installing tool dependencies:'
                         print message
                 add_to_tool_panel( self.app,
                                    repository_name,
@@ -174,8 +181,17 @@ class InstallManager( object ):
                                    self.migrated_tools_config,
                                    tool_panel_dict=tool_panel_dict_for_display,
                                    new_install=True )
-        if 'datatypes_config' in metadata_dict:
-            datatypes_config = os.path.abspath( metadata_dict[ 'datatypes_config' ] )
+            try:
+                shutil.rmtree( work_dir )
+            except:
+                pass
+        if 'datatypes' in metadata_dict:
+            work_dir = make_tmp_directory()
+            datatypes_config = get_config_from_repository( self.app,
+                                                           'datatypes_conf.xml',
+                                                           tool_shed_repository,
+                                                           changeset_revision,
+                                                           work_dir )
             # Load proprietary data types required by tools.  The value of override is not important here since the Galaxy server will be started
             # after this installation completes.
             converter_path, display_path = alter_config_and_load_prorietary_datatypes( self.app, datatypes_config, relative_install_dir, override=False )
@@ -194,6 +210,10 @@ class InstallManager( object ):
             if display_path:
                 # Load proprietary datatype display applications
                 self.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict )
+            try:
+                shutil.rmtree( work_dir )
+            except:
+                pass
         return tool_shed_repository, metadata_dict
     def install_repository( self, repository_elem ):
         # Install a single repository, loading contained tools into the tool panel.
@@ -205,14 +225,12 @@ class InstallManager( object ):
         if self.__isinstalled( clone_dir ):
             print "Skipping automatic install of repository '", name, "' because it has already been installed in location ", clone_dir
         else:
-            current_working_dir = os.getcwd()
             tool_shed_url = self.__get_url_from_tool_shed( self.tool_shed )
             repository_clone_url = os.path.join( tool_shed_url, 'repos', self.repository_owner, name )
             relative_install_dir = os.path.join( clone_dir, name )
             ctx_rev = get_ctx_rev( tool_shed_url, name, self.repository_owner, changeset_revision )
             clone_repository( repository_clone_url, os.path.abspath( relative_install_dir ), ctx_rev )
-            tool_shed_repository, metadata_dict = self.handle_repository_contents( current_working_dir,
-                                                                                   repository_clone_url,
+            tool_shed_repository, metadata_dict = self.handle_repository_contents( repository_clone_url,
                                                                                    relative_install_dir,
                                                                                    repository_elem,
                                                                                    name,
@@ -280,8 +298,7 @@ class InstallManager( object ):
                 if shed_url.endswith( '/' ):
                     shed_url = shed_url.rstrip( '/' )
                 return shed_url
-        # The tool shed from which the repository was originally
-        # installed must no longer be configured in tool_sheds_conf.xml.
+        # The tool shed from which the repository was originally installed must no longer be configured in tool_sheds_conf.xml.
         return None
     def __isinstalled( self, clone_dir ):
         full_path = os.path.abspath( clone_dir )

lib/galaxy/tool_shed/tool_dependencies/common_util.py

@@ -0,0 +1,43 @@
+import os, tarfile, urllib2
+from galaxy.datatypes.checkers import *
+
+def extract_tar( file_name, file_path ):
+    if isgzip( file_name ) or isbz2( file_name ):
+        # Open for reading with transparent compression.
+        tar = tarfile.open( file_name, 'r:*' )
+    else:
+        tar = tarfile.open( file_name )
+    tar.extractall( path=file_path )
+    tar.close()
+def isbz2( file_path ):
+    return is_bz2( file_path )
+def isgzip( file_path ):
+    return is_gzip( file_path )
+def istar( file_path ):
+    return tarfile.is_tarfile( file_path )
+def iszip( file_path ):
+    return check_zip( file_path )
+def tar_extraction_directory( file_path, file_name ):
+    extensions = [ '.tar.gz', '.tgz', '.tar.bz2', '.zip' ]
+    for extension in extensions:
+        if file_name.endswith( extension ):
+            dir_name = file_name[ :-len( extension ) ]
+            full_path = os.path.abspath( os.path.join( file_path, dir_name ) )
+            if os.path.exists( full_path ):
+                return dir_name
+    raise ValueError( 'Could not find directory %s' % full_path )
+def url_download( install_dir, downloaded_file_name, download_url ):
+    file_path = os.path.join( install_dir, downloaded_file_name )
+    src = None
+    dst = None
+    try:
+        src = urllib2.urlopen( download_url )
+        data = src.read()
+        dst = open( file_path,'wb' )
+        dst.write( data )
+    except:
+        if src:
+            src.close()
+        if dst:
+            dst.close()
+    return os.path.abspath( file_path )

lib/galaxy/tool_shed/tool_dependencies/fabric_util.py

@@ -0,0 +1,108 @@
+# For Python 2.5
+from __future__ import with_statement
+
+import os
+from contextlib import contextmanager
+import common_util
+
+from galaxy import eggs
+import pkg_resources
+
+pkg_resources.require( 'fabric' )
+
+from fabric.api import env, lcd, local
+
+def check_fabric_version():
+    version = env.version
+    if int( version.split( "." )[ 0 ] ) < 1:
+        raise NotImplementedError( "Install Fabric version 1.0 or later." )
+def set_galaxy_environment( galaxy_user, tool_dependency_dir, host='localhost', shell='/bin/bash -l -c' ):
+    """General Galaxy environment configuration"""
+    env.user = galaxy_user
+    env.install_dir = tool_dependency_dir
+    env.host_string = host
+    env.shell = shell
+    env.use_sudo = False
+    env.safe_cmd = local
+    return env
+@contextmanager
+def make_tmp_dir():
+    tmp_dir = local( 'echo $TMPDIR' ).strip()
+    if not tmp_dir:
+        home_dir = local( 'echo $HOME' )
+        tmp_dir = os.path.join( home_dir, 'tmp' )
+    work_dir = os.path.join( tmp_dir, 'deploy_tmp' )
+    if not os.path.exists( work_dir ):
+        local( 'mkdir -p %s' % work_dir )
+    yield work_dir
+    if os.path.exists( work_dir ):
+        local( 'rm -rf %s' % work_dir )
+def handle_post_build_processing( tool_dependency_dir, install_dir, package_name=None ):
+    cmd = "echo 'PATH=%s/bin:$PATH' > %s/env.sh;chmod +x %s/env.sh" % ( install_dir, install_dir, install_dir )
+    message = ''
+    output = local( cmd, capture=True )
+    log_results( cmd, output, os.path.join( install_dir, 'env_sh.log' ) )
+    if output.return_code:
+        message = '%s  %s' % ( message, str( output.stderr ) )
+    """
+    Since automatic dependency installation requires a version attribute in the tool's <requirement> tag, we don't have to
+    create a default symlink, but we'll keep this code around for a bit just in case we need it later.
+    if package_name:
+        package_dir = os.path.join( tool_dependency_dir, package_name )
+        package_default = os.path.join( package_dir, 'default' )
+        if not os.path.islink( package_default ):
+            cmd = 'ln -s %s %s' % ( install_dir, package_default )
+            output = local( cmd, capture=True )
+            if output.return_code:
+                message = '%s\n%s' % ( message, str( output.stderr ) )
+    """
+    return message
+def install_and_build_package( params_dict ):
+    """Install a Galaxy tool dependency package either via a url or a mercurial or git clone command."""
+    install_dir = params_dict[ 'install_dir' ]
+    download_url = params_dict.get( 'download_url', None )
+    clone_cmd = params_dict.get( 'clone_cmd', None )
+    build_commands = params_dict.get( 'build_commands', None )
+    package_name = params_dict.get( 'package_name', None )
+    with make_tmp_dir() as work_dir:
+        with lcd( work_dir ):
+            if download_url:
+                downloaded_filename = os.path.split( download_url )[ -1 ]
+                downloaded_file_path = common_util.url_download( work_dir, downloaded_filename, download_url )
+                if common_util.istar( downloaded_file_path ):
+                    common_util.extract_tar( downloaded_file_path, work_dir )
+                    dir = common_util.tar_extraction_directory( work_dir, downloaded_filename )
+                else:
+                    dir = work_dir
+            elif clone_cmd:
+                output = local( clone_cmd, capture=True )
+                log_results( clone_cmd, output, os.path.join( install_dir, 'clone_repository.log' ) )
+                if output.return_code:
+                    return '%s.  ' % str( output.stderr )
+                dir = package_name
+            if build_commands:
+                with lcd( dir ):
+                    for build_command in build_commands:
+                        output = local( build_command, capture=True )
+                        log_results( build_command, output, os.path.join( install_dir, 'build_commands.log' ) )
+                        if output.return_code:
+                            return '%s.  ' % str( output.stderr )
+    return ''
+def log_results( command, fabric_AttributeString, file_path ):
+    """
+    Write attributes of fabric.operations._AttributeString (which is the output of executing command using fabric's local() method)
+    to a specified log file.
+    """
+    if os.path.exists( file_path ):
+        logfile = open( file_path, 'ab' )
+    else:
+        logfile = open( file_path, 'wb' )
+    logfile.write( "#############################################" )
+    logfile.write( '\n%s\nSTDOUT\n' % command )
+    logfile.write( "#############################################" )
+    logfile.write( str( fabric_AttributeString.stdout ) )
+    logfile.write( "#############################################" )
+    logfile.write( '\n%s\nSTDERR\n' % command )
+    logfile.write( "#############################################" )
+    logfile.write( str( fabric_AttributeString.stderr ) )
+    logfile.close()

lib/galaxy/tool_shed/tool_dependencies/install_util.py

@@ -0,0 +1,183 @@
+import sys, os, subprocess, tempfile
+from fabric_util import *
+from galaxy.tool_shed.encoding_util import *
+from galaxy.model.orm import *
+
+from galaxy import eggs
+import pkg_resources
+
+pkg_resources.require( 'elementtree' )
+from elementtree import ElementTree, ElementInclude
+from elementtree.ElementTree import Element, SubElement
+
+def create_or_update_tool_dependency( app, tool_shed_repository, changeset_revision, name, version, type ):
+    sa_session = app.model.context.current
+    tool_dependency = get_tool_dependency_by_shed_changeset_revision( app, tool_shed_repository, name, version, type, changeset_revision )
+    if tool_dependency:
+        tool_dependency.uninstalled = False
+    else:
+        tool_dependency = app.model.ToolDependency( tool_shed_repository_id=tool_shed_repository.id,
+                                                    installed_changeset_revision=changeset_revision,
+                                                    name=name,
+                                                    version=version,
+                                                    type=type )
+    sa_session.add( tool_dependency )
+    sa_session.flush()
+    return tool_dependency
+def get_install_dir( app, repository, installed_changeset_revision, package_name, package_version ):
+    return os.path.abspath( os.path.join( app.config.tool_dependency_dir,
+                                          package_name,
+                                          package_version,
+                                          repository.owner,
+                                          repository.name,
+                                          installed_changeset_revision ) )
+def get_tool_dependency_by_shed_changeset_revision( app, repository, dependency_name, dependency_version, dependency_type, changeset_revision ):
+    sa_session = app.model.context.current
+    return sa_session.query( app.model.ToolDependency ) \
+                     .filter( and_( app.model.ToolDependency.table.c.tool_shed_repository_id == repository.id,
+                                    app.model.ToolDependency.table.c.name == dependency_name,
+                                    app.model.ToolDependency.table.c.version == dependency_version,
+                                    app.model.ToolDependency.table.c.type == dependency_type,
+                                    app.model.ToolDependency.table.c.installed_changeset_revision == changeset_revision ) ) \
+                     .first()
+def install_package( app, elem, tool_shed_repository, installed_changeset_revision ):
+    # The value of package_name should match the value of the "package" type in the tool config's <requirements> tag set, but it's not required.
+    message = ''
+    package_name = elem.get( 'name', None )
+    package_version = elem.get( 'version', None )
+    if package_name and package_version:
+        install_dir = get_install_dir( app, tool_shed_repository, installed_changeset_revision, package_name, package_version )
+        if not_installed( install_dir ):
+            for package_elem in elem:
+                if package_elem.tag == 'proprietary_fabfile':
+                    # TODO: This is not yet working...
+                    # Handle tool dependency installation where the repository includes one or more proprietary fabric scripts.
+                    if not fabric_version_checked:
+                        check_fabric_version()
+                        fabric_version_checked = True
+                    fabfile_name = package_elem.get( 'name', None )
+                    fabfile_path = os.path.abspath( os.path.join( os.path.split( tool_dependencies_config )[ 0 ], fabfile_name ) )
+                    print 'Installing tool dependencies via fabric script ', fabfile_path
+                elif package_elem.tag == 'fabfile':
+                    # Handle tool dependency installation using a fabric script provided by Galaxy.  Example tag set definition:
+                    fabfile_path = None
+                for method_elem in package_elem.findall( 'method' ):
+                    error_message = run_fabric_method( app,
+                                                       method_elem,
+                                                       fabfile_path,
+                                                       app.config.tool_dependency_dir,
+                                                       install_dir,
+                                                       package_name=package_name )
+                    if error_message:
+                        message += '%s' % error_message
+                    else:
+                        tool_dependency = create_or_update_tool_dependency( app,
+                                                                            tool_shed_repository,
+                                                                            installed_changeset_revision,
+                                                                            name=package_name,
+                                                                            version=package_version,
+                                                                            type='package' )
+                        print package_name, 'version', package_version, 'installed in', install_dir
+        else:
+            print '\nSkipping installation of tool dependency', package_name, 'version', package_version, 'since it is installed in', install_dir, '\n'
+    return message
+def not_installed( install_dir ):
+    # TODO: try executing a binary or something in addition to just seeing if the install_dir exists.
+    return not os.path.exists( install_dir )
+def run_fabric_method( app, elem, fabfile_path, tool_dependency_dir, install_dir, package_name=None, **kwd ):
+    """Parse a tool_dependency.xml file's fabfile <method> tag set to build the method parameters and execute the method."""
+    if not os.path.exists( install_dir ):
+        os.makedirs( install_dir )
+    method_name = elem.get( 'name', None )
+    params_dict = dict( install_dir=install_dir )
+    build_commands = []
+    for param_elem in elem:
+        param_name = param_elem.get( 'name' )
+        if param_name:
+            if param_name == 'build_commands':
+                for build_command_elem in param_elem:
+                    build_commands.append( build_command_elem.text.replace( '$INSTALL_DIR', install_dir ) )
+                if build_commands:
+                    params_dict[ 'build_commands' ] = build_commands
+            else:
+                if param_elem.text:
+                    params_dict[ param_name ] = param_elem.text
+    if package_name:
+        params_dict[ 'package_name' ] = package_name
+    if fabfile_path:
+        # TODO: Handle this using the fabric api.
+        # run_proprietary_fabric_method( app, elem, fabfile_path, tool_dependency_dir, install_dir, package_name=package_name )
+        return 'Tool dependency installation using proprietary fabric scripts is not yet supported.  '
+    else:
+        # There is currently only 1 fabric method, install_and_build_package().
+        try:
+            message = install_and_build_package( params_dict )
+            if message:
+                return message
+        except Exception, e:
+            return '%s.  ' % str( e )
+        try:
+            message = handle_post_build_processing( tool_dependency_dir, install_dir, package_name=package_name )
+            if message:
+                return message
+        except:
+            return '%s.  ' % str( e )
+        print package_name, 'installed to', install_dir
+        return ''
+def run_proprietary_fabric_method( app, elem, fabfile_path, tool_dependency_dir, install_dir, package_name=None, **kwd ):
+    """
+    TODO: Handle this using the fabric api.
+    Parse a tool_dependency.xml file's fabfile <method> tag set to build the method parameters and execute the method.
+    """
+    if not os.path.exists( install_dir ):
+        os.makedirs( install_dir )
+    method_name = elem.get( 'name', None )
+    params_str = ''
+    build_commands = []
+    for param_elem in elem:
+        param_name = param_elem.get( 'name' )
+        if param_name:
+            if param_name == 'build_commands':
+                for build_command_elem in param_elem:
+                    build_commands.append( build_command_elem.text.replace( '$INSTALL_DIR', install_dir ) )
+                if build_commands:
+                    params_str += 'build_commands=%s,' % tool_shed_encode( encoding_sep.join( build_commands ) )
+            else:
+                if param_elem.text:
+                    param_value = tool_shed_encode( param_elem.text )
+                    params_str += '%s=%s,' % ( param_name, param_value )
+    if package_name:
+        params_str += 'package_name=%s' % package_name
+    else:
+        params_str = params_str.rstrip( ',' )
+    try:
+        cmd = 'fab -f %s %s:%s' % ( fabfile_path, method_name, params_str )
+        returncode, message = run_subprocess( app, cmd )
+    except Exception, e:
+        return "Exception executing fabric script %s: %s.  " % ( str( fabfile_path ), str( e ) )
+    if returncode:
+        return message    
+    message = handle_post_build_processing( tool_dependency_dir, install_dir, package_name=package_name )
+    if message:
+        return message
+    else:
+        print package_name, 'installed to', install_dir
+        return ''
+def run_subprocess( app, cmd ):
+    env = os.environ
+    PYTHONPATH = env.get( 'PYTHONPATH', '' )
+    if PYTHONPATH:
+        env[ 'PYTHONPATH' ] = '%s:%s' % ( os.path.abspath( os.path.join( app.config.root, 'lib' ) ), PYTHONPATH )
+    else:
+        env[ 'PYTHONPATH' ] = os.path.abspath( os.path.join( app.config.root, 'lib' ) )
+    message = ''
+    tmp_name = tempfile.NamedTemporaryFile().name
+    tmp_stderr = open( tmp_name, 'wb' )
+    proc = subprocess.Popen( cmd, shell=True, env=env, stderr=tmp_stderr.fileno() )
+    returncode = proc.wait()
+    tmp_stderr.close()
+    if returncode:
+        tmp_stderr = open( tmp_name, 'rb' )
+        message = '%s\n' % str( tmp_stderr.read() )
+        tmp_stderr.close()
+    return returncode, message

lib/galaxy/tool_shed/tool_shed_registry.py

@@ -15,10 +15,9 @@ class Registry( object ):
     def __init__( self, root_dir=None, config=None ):
         self.tool_sheds = odict()
         if root_dir and config:
-            # Parse datatypes_conf.xml
+            # Parse tool_sheds_conf.xml
             tree = parse_xml( config )
             root = tree.getroot()
-            # Load datatypes and converters from config
             log.debug( 'Loading references to tool sheds from %s' % config )
             for elem in root.findall( 'tool_shed' ):
                 try:

lib/galaxy/tools/__init__.py

@@ -830,10 +830,8 @@ class Tool:
         if not self.name: 
             raise Exception, "Missing tool 'name'"
         # Get the UNIQUE id for the tool 
-        # TODO: can this be generated automatically?
         if guid is None:
             self.id = root.get( "id" )
-            self.version = root.get( "version" )
         else:
             self.id = guid
         if not self.id: 
@@ -850,8 +848,7 @@ class Tool:
         self.force_history_refresh = util.string_as_bool( root.get( 'force_history_refresh', 'False' ) )
         self.display_interface = util.string_as_bool( root.get( 'display_interface', str( self.display_interface ) ) )
         self.require_login = util.string_as_bool( root.get( 'require_login', str( self.require_login ) ) )
-        # Load input translator, used by datasource tools to change 
-        # names/values of incoming parameters
+        # Load input translator, used by datasource tools to change names/values of incoming parameters
         self.input_translator = root.find( "request_param_translation" )
         if self.input_translator:
             self.input_translator = ToolInputTranslator.from_element( self.input_translator )

lib/galaxy/util/__init__.py

@@ -16,6 +16,7 @@ try:
 except ImportError:
     from md5 import new as md5
 
+from galaxy import eggs
 import pkg_resources
 
 pkg_resources.require( 'docutils' )

lib/galaxy/util/shed_util.py

@@ -1,17 +1,18 @@
-import sys, os, tempfile, shutil, subprocess, logging, string, urllib2
+import sys, os, tempfile, shutil, logging, string, urllib2
 from datetime import date, datetime, timedelta
 from time import strftime, gmtime
 from galaxy import util
 from galaxy.datatypes.checkers import *
 from galaxy.util.json import *
 from galaxy.tools.search import ToolBoxSearch
+from galaxy.tool_shed.tool_dependencies.install_util import install_package
 from galaxy.model.orm import *
 
 from galaxy import eggs
 import pkg_resources
 
 pkg_resources.require( 'mercurial' )
-from mercurial import ui, commands
+from mercurial import hg, ui, commands
 
 pkg_resources.require( 'elementtree' )
 from elementtree import ElementTree, ElementInclude
@@ -26,6 +27,7 @@ MAPPED_CHARS = { '>' :'>',
                  '&' : '&',
                  '\'' : ''' }
 VALID_CHARS = set( string.letters + string.digits + "'\"-=_.()/+*^,:?!#[]%\\$@;{}" )
+NOT_TOOL_CONFIGS = [ 'datatypes_conf.xml', 'tool_dependencies.xml' ]
 
 class ShedCounter( object ):
     def __init__( self, model ):
@@ -203,7 +205,7 @@ def alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_
             if display_path:
                 registration.attrib[ 'proprietary_display_path' ] = display_path
             for relative_path_to_datatype_file_name in datatype_class_modules:
-                relative_head, relative_tail = os.path.split( relative_path_to_datatype_file_name )
+                datatype_file_name_path, datatype_file_name = os.path.split( relative_path_to_datatype_file_name )
                 for elem in registration.findall( 'datatype' ):
                     # Handle 'type' attribute which should be something like one of the following: 
                     # type="gmap:GmapDB"
@@ -216,7 +218,7 @@ def alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_
                             # Handle the case where datatype_module is "galaxy.datatypes.gmap".
                             proprietary_datatype_module = proprietary_datatype_module.split( '.' )[ -1 ]
                         # The value of proprietary_path must be an absolute path due to job_working_directory.
-                        elem.attrib[ 'proprietary_path' ] = os.path.abspath( relative_head )
+                        elem.attrib[ 'proprietary_path' ] = os.path.abspath( datatype_file_name_path )
                         elem.attrib[ 'proprietary_datatype_module' ] = proprietary_datatype_module
             sniffers = datatypes_config_root.find( 'sniffers' )
         else:
@@ -271,9 +273,7 @@ def clean_tool_shed_url( tool_shed_url ):
         return tool_shed_url.split( ':' )[ 0 ]
     return tool_shed_url.rstrip( '/' )
 def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ):
-    """
-    Clone the repository up to the specified changeset_revision.  No subsequent revisions will be present in the cloned repository.
-    """
+    """Clone the repository up to the specified changeset_revision.  No subsequent revisions will be present in the cloned repository."""
     commands.clone( get_configured_ui(),
                     repository_clone_url,
                     dest=repository_file_dir,
@@ -281,12 +281,10 @@ def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ):
                     noupdate=False,
                     rev=[ ctx_rev ] )
 def copy_sample_file( app, filename, dest_path=None ):
-    """
-    Copy xxx.loc.sample to dest_path/xxx.loc.sample and dest_path/xxx.loc.  The default value for dest_path is ~/tool-data.
-    """
+    """Copy xxx.loc.sample to dest_path/xxx.loc.sample and dest_path/xxx.loc.  The default value for dest_path is ~/tool-data."""
     if dest_path is None:
         dest_path = os.path.abspath( app.config.tool_data_path )
-    sample_file_path, sample_file_name = os.path.split( filename )
+    sample_file_name = strip_path( filename )
     copied_file = sample_file_name.replace( '.sample', '' )
     # It's ok to overwrite the .sample version of the file.
     shutil.copy( os.path.abspath( filename ), os.path.join( dest_path, sample_file_name ) )
@@ -319,7 +317,7 @@ def create_or_update_tool_shed_repository( app, name, description, changeset_rev
     tool_shed = tmp_url.split( 'repos' )[ 0 ].rstrip( '/' )
     if not owner:
         owner = get_repository_owner( tmp_url )
-    includes_datatypes = 'datatypes_config' in metadata_dict
+    includes_datatypes = 'datatypes' in metadata_dict
     tool_shed_repository = get_repository_by_shed_name_owner_changeset_revision( app, tool_shed, name, owner, changeset_revision )
     if tool_shed_repository:
         tool_shed_repository.description = description
@@ -387,31 +385,111 @@ def generate_datatypes_metadata( datatypes_config, metadata_dict ):
         if datatypes:
             metadata_dict[ 'datatypes' ] = datatypes
     return metadata_dict
-def generate_metadata( toolbox, relative_install_dir, repository_clone_url ):
+def generate_tool_dependency_metadata( tool_dependencies_config, metadata_dict ):
     """
-    Browse the repository files on disk to generate metadata.  Since we are using disk files, it is imperative that the
-    repository is updated to the desired change set revision before metadata is generated.
+    If the combination of name, version and type of each element is defined in the <requirement> tag for at least one tool in the repository,
+    then update the received metadata_dict with information from the parsed tool_dependencies_config.
     """
-    metadata_dict = {}
-    sample_files = []
-    datatypes_config = None
-    # Find datatypes_conf.xml if it exists.
-    for root, dirs, files in os.walk( relative_install_dir ):
-        if root.find( '.hg' ) < 0:
-            for name in files:
-                if name == 'datatypes_conf.xml':
-                    relative_path = os.path.join( root, name )
-                    datatypes_config = os.path.abspath( relative_path )
+    tree = ElementTree.parse( tool_dependencies_config )
+    root = tree.getroot()
+    ElementInclude.include( root )
+    tool_dependencies_dict = {}
+    if can_generate_tool_dependency_metadata( root, metadata_dict ):
+        for elem in root:
+            if elem.tag == 'package':
+                tool_dependencies_dict = generate_package_dependency_metadata( elem, tool_dependencies_dict )
+            # Handle tool dependency installation via other means here (future).
+        if tool_dependencies_dict:
+            metadata_dict[ 'tool_dependencies' ] = tool_dependencies_dict
+    return metadata_dict
+def can_generate_tool_dependency_metadata( root, metadata_dict ):
+    """
+    Make sure the combination of name, version and type (the type will be the value of elem.tag) of each root element tag in the tool_dependencies.xml
+    file is defined in the <requirement> tag for at least one tool in the repository.
+    """
+    can_generate_dependency_metadata = False
+    for elem in root:
+        can_generate_dependency_metadata = False
+        tool_dependency_name = elem.get( 'name', None )
+        tool_dependency_version = elem.get( 'version', None )
+        tool_dependency_type = elem.tag
+        if tool_dependency_name and tool_dependency_version and tool_dependency_type:
+            for tool_dict in metadata_dict[ 'tools' ]:
+                requirements = tool_dict.get( 'requirements', [] )
+                for requirement_dict in requirements:
+                    requirement_name = requirement_dict.get( 'name', None )
+                    requirement_version = requirement_dict.get( 'version', None )
+                    requirement_type = requirement_dict.get( 'type', None )
+                    if requirement_name == tool_dependency_name and requirement_version == tool_dependency_version and requirement_type == tool_dependency_type:
+                        can_generate_dependency_metadata = True
+                        break
+                if not can_generate_dependency_metadata:
+                    # We've discovered at least 1 combination of name, version and type that is not defined in the <requirement>
+                    # tag for any tool in the repository.
                     break
+            if not can_generate_dependency_metadata:
+                break
+    return can_generate_dependency_metadata
+def generate_package_dependency_metadata( elem, tool_dependencies_dict ):
+    """The value of package_name must match the value of the "package" type in the tool config's <requirements> tag set."""
+    requirements_dict = {}
+    package_name = elem.get( 'name', None )
+    package_version = elem.get( 'version', None )
+    if package_name and package_version:
+        requirements_dict [ 'name' ] = package_name
+        requirements_dict [ 'type' ] = 'package'
+        requirements_dict [ 'version' ] = package_version
+        dependency_key = '%s/%s' % ( package_name, package_version )
+        fabfiles_dict = {}
+        for sub_elem in elem:
+            if sub_elem.tag == 'proprietary_fabfile':
+                requirements_dict = generate_fabfile_metadata( sub_elem, requirements_dict, proprietary=True )
+            elif sub_elem.tag == 'fabfile':
+                requirements_dict = generate_fabfile_metadata( sub_elem, requirements_dict, proprietary=False )
+            elif sub_elem.tag == 'readme':
+                requirements_dict[ 'readme' ] = sub_elem.text
+    if requirements_dict:
+        tool_dependencies_dict[ dependency_key ] = requirements_dict
+    return tool_dependencies_dict
+def generate_fabfile_metadata( elem, requirements_dict, proprietary=False ):
+    """
+    <proprietary_fabfile name="fabfile.py">
+        <method name="install_and_build">
+            <param name="download_url">ftp://emboss.open-bio.org/pub/EMBOSS/old/5.0.0/EMBOSS-5.0.0.tar.gz</param>
+            <param name="download_url">ftp://emboss.open-bio.org/pub/EMBOSS/old/5.0.0/PHYLIP-3.6b.tar.gz</param>
+        </method>
+    </proprietary_fabfile>
+    """
+    fabfiles_dict = {}
+    fabfile_name = elem.get( 'name', None )
+    if fabfile_name:
+        for method_elem in elem.findall( 'method' ):
+            method_name = method_elem.get( 'name', None )
+            if method_name:
+                params_str = ''
+                for param_elem in method_elem.findall( 'param' ):
+                    param_name = param_elem.get( 'name', None )
+                    param_value = param_elem.text
+                    if param_name and param_value:
+                        params_str += '%s=%s,' % ( param_name, param_value )
+                fabfiles_dict[ 'fabfile' ] = fabfile_name
+                fabfiles_dict[ 'method' ] = method_name
+                fabfiles_dict[ 'params' ] = params_str.rstrip( ',' )
+        if fabfiles_dict:
+            if proprietary:
+                key = 'proprietary_fabfiles'
+            else:
+                key = 'fabfiles'
+            requirements_dict[ key ] = fabfiles_dict
+    return requirements_dict
+def generate_metadata_using_disk_files( toolbox, relative_install_dir, repository_clone_url ):
+    """generate metadata using only the repository files on disk - files are not retrieved from the repository manifest."""
+    metadata_dict = {}
+    tool_dependencies_config = None
+    datatypes_config = get_config_from_disk( 'datatypes_conf.xml', relative_install_dir )
     if datatypes_config:
-        metadata_dict[ 'datatypes_config' ] = relative_path
         metadata_dict = generate_datatypes_metadata( datatypes_config, metadata_dict )
-    # Find all special .sample files.
-    for root, dirs, files in os.walk( relative_install_dir ):
-        if root.find( '.hg' ) < 0:
-            for name in files:
-                if name.endswith( '.sample' ):
-                    sample_files.append( os.path.join( root, name ) )
+    sample_files = get_sample_files_from_disk( relative_install_dir )
     if sample_files:
         metadata_dict[ 'sample_files' ] = sample_files
     # Find all tool configs and exported workflows.
@@ -421,7 +499,7 @@ def generate_metadata( toolbox, relative_install_dir, repository_clone_url ):
                 dirs.remove( '.hg' )
             for name in files:
                 # Find all tool configs.
-                if name != 'datatypes_conf.xml' and name.endswith( '.xml' ):
+                if name not in NOT_TOOL_CONFIGS and name.endswith( '.xml' ):
                     full_path = os.path.abspath( os.path.join( root, name ) )
                     if not ( check_binary( full_path ) or check_image( full_path ) or check_gzip( full_path )[ 0 ]
                              or check_bz2( full_path )[ 0 ] or check_zip( full_path ) ):
@@ -450,6 +528,10 @@ def generate_metadata( toolbox, relative_install_dir, repository_clone_url ):
                     exported_workflow_dict = from_json_string( workflow_text )
                     if 'a_galaxy_workflow' in exported_workflow_dict and exported_workflow_dict[ 'a_galaxy_workflow' ] == 'true':
                         metadata_dict = generate_workflow_metadata( relative_path, exported_workflow_dict, metadata_dict )
+    # This step must be done after metadata for tools has been defined.
+    tool_dependencies_config = get_config_from_disk( 'tool_dependencies.xml', relative_install_dir )
+    if tool_dependencies_config:
+        metadata_dict = generate_tool_dependency_metadata( tool_dependencies_config, metadata_dict )
     return metadata_dict
 def generate_tool_guid( repository_clone_url, tool ):
     """
@@ -485,7 +567,7 @@ def generate_tool_metadata( tool_config, tool, repository_clone_url, metadata_di
             outputs = []
             for output in ttb.outputs:
                 name, file_name, extra = output
-                outputs.append( ( name, os.path.split( file_name )[ 1 ] ) )
+                outputs.append( ( name, strip_path( file_name ) ) )
             test_dict = dict( name=ttb.name,
                               required_files=required_files,
                               inputs=inputs,
@@ -597,7 +679,7 @@ def generate_tool_panel_dict_from_shed_tool_conf_entries( trans, repository ):
     for tool_dict in metadata[ 'tools' ]:
         guid = tool_dict[ 'guid' ]
         tool_config = tool_dict[ 'tool_config' ]
-        file_path, file_name = os.path.split( tool_config )
+        file_name = strip_path( tool_config )
         guids_and_configs[ guid ] = file_name
     # Parse the shed_tool_conf file in which all of this repository's tools are defined and generate the tool_panel_dict. 
     tree = util.parse_xml( shed_tool_conf )
@@ -638,7 +720,7 @@ def generate_tool_panel_dict_for_tool_config( guid, tool_config, tool_sections=N
     {<Tool guid> : [{ tool_config : <tool_config_file>, id: <ToolSection id>, version : <ToolSection version>, name : <TooSection name>}]}
     """
     tool_panel_dict = {}
-    file_path, file_name = os.path.split( tool_config )
+    file_name = strip_path( tool_config )
     tool_section_dicts = generate_tool_section_dicts( tool_config=file_name, tool_sections=tool_sections )
     tool_panel_dict[ guid ] = tool_section_dicts
     return tool_panel_dict
@@ -680,6 +762,52 @@ def generate_workflow_metadata( relative_path, exported_workflow_dict, metadata_
     else:
         metadata_dict[ 'workflows' ] = [ ( relative_path, exported_workflow_dict ) ]
     return metadata_dict
+def get_changectx_for_changeset( repo, changeset_revision, **kwd ):
+    """Retrieve a specified changectx from a repository"""
+    for changeset in repo.changelog:
+        ctx = repo.changectx( changeset )
+        if str( ctx ) == changeset_revision:
+            return ctx
+    return None
+def get_config( config_file, repo, repo_dir, ctx, dir ):
+    """Return config_filename if it exists in some changeset of the repository."""
+    # First look on disk.
+    for root, dirs, files in os.walk( repo_dir ):
+        if root.find( '.hg' ) < 0:
+            for name in files:
+                if name == config_file:
+                    dest_file_name = os.path.join( dir, name )
+                    shutil.copy( os.path.abspath( os.path.join( root, name ) ), dest_file_name )
+                    return os.path.abspath( dest_file_name )
+    # Next look in the current change set.
+    for filename in ctx:
+        ctx_file_name = strip_path( filename )
+        if ctx_file_name == config_file:
+            return get_named_tmpfile_from_ctx( ctx, filename, dir=dir )
+    # Finally look in the repository manifest.
+    for changeset in repo.changelog:
+        prev_ctx = repo.changectx( changeset )
+        for ctx_file in prev_ctx.files():
+            ctx_file_name = strip_path( ctx_file )
+            if ctx_file_name == config_file:
+                return get_named_tmpfile_from_ctx( prev_ctx, filename, dir=dir )
+    return None
+def get_config_from_disk( config_file, relative_install_dir ):
+    for root, dirs, files in os.walk( relative_install_dir ):
+        if root.find( '.hg' ) < 0:
+            for name in files:
+                if name == config_file:
+                    return os.path.abspath( os.path.join( root, name ) )
+    return None
+def get_config_from_repository( app, config_file, repository, changeset_revision, dir, install_dir=None ):
+    """Return config_filename if it exists in some changeset of the repository using only the repository and changeset_revision."""
+    if install_dir is None:
+        install_dir = repository.repo_path( app )
+    repo_files_dir = os.path.join( install_dir, repository.name )
+    repo = hg.repository( get_configured_ui(), repo_files_dir )
+    ctx = get_changectx_for_changeset( repo, changeset_revision )
+    config = get_config( config_file, repo, repo_files_dir, ctx, dir )
+    return config
 def get_configured_ui():
     # Configure any desired ui settings.
     _ui = ui.ui()
@@ -701,11 +829,11 @@ def get_converter_and_display_paths( registration_elem, relative_install_dir ):
             for converter in elem.findall( 'converter' ):
                 converter_config = converter.get( 'file', None )
                 if converter_config:
-                    relative_head, relative_tail = os.path.split( converter_config )
+                    converter_config_file_name = strip_path( converter_config )
                     for root, dirs, files in os.walk( relative_install_dir ):
                         if root.find( '.hg' ) < 0:
                             for name in files:
-                                if name == relative_tail:
+                                if name == converter_config_file_name:
                                     # The value of converter_path must be absolute due to job_working_directory.
                                     converter_path = os.path.abspath( root )
                                     break
@@ -718,11 +846,11 @@ def get_converter_and_display_paths( registration_elem, relative_install_dir ):
             for display_app in elem.findall( 'display' ):
                 display_config = display_app.get( 'file', None )
                 if display_config:
-                    relative_head, relative_tail = os.path.split( display_config )
+                    display_config_file_name = strip_path( display_config )
                     for root, dirs, files in os.walk( relative_install_dir ):
                         if root.find( '.hg' ) < 0:
                             for name in files:
-                                if name == relative_tail:
+                                if name == display_config_file_name:
                                     # The value of display_path must be absolute due to job_working_directory.
                                     display_path = os.path.abspath( root )
                                     break
@@ -737,18 +865,6 @@ def get_ctx_rev( tool_shed_url, name, owner, changeset_revision ):
     ctx_rev = response.read()
     response.close()
     return ctx_rev
-def get_shed_tool_conf_dict( app, shed_tool_conf ):
-    """
-    Return the in-memory version of the shed_tool_conf file, which is stored in the config_elems entry
-    in the shed_tool_conf_dict associated with the file.
-    """
-    for index, shed_tool_conf_dict in enumerate( app.toolbox.shed_tool_confs ):
-        if shed_tool_conf == shed_tool_conf_dict[ 'config_filename' ]:
-            return index, shed_tool_conf_dict
-        else:
-            file_path, file_name = os.path.split( shed_tool_conf_dict[ 'config_filename' ] )
-            if shed_tool_conf == file_name:
-                return index, shed_tool_conf_dict
 def get_repository_by_shed_name_owner_changeset_revision( app, tool_shed, name, owner, changeset_revision ):
     sa_session = app.model.context.current
     if tool_shed.find( '//' ) > 0:
@@ -779,6 +895,26 @@ def get_repository_tools_tups( app, metadata_dict ):
             if tool:
                 repository_tools_tups.append( ( relative_path, guid, tool ) )
     return repository_tools_tups
+def get_sample_files_from_disk( relative_install_dir ):
+    sample_files = []
+    for root, dirs, files in os.walk( relative_install_dir ):
+            if root.find( '.hg' ) < 0:
+                for name in files:
+                    if name.endswith( '.sample' ):
+                        sample_files.append( os.path.join( root, name ) )
+    return sample_files
+def get_shed_tool_conf_dict( app, shed_tool_conf ):
+    """
+    Return the in-memory version of the shed_tool_conf file, which is stored in the config_elems entry
+    in the shed_tool_conf_dict associated with the file.
+    """
+    for index, shed_tool_conf_dict in enumerate( app.toolbox.shed_tool_confs ):
+        if shed_tool_conf == shed_tool_conf_dict[ 'config_filename' ]:
+            return index, shed_tool_conf_dict
+        else:
+            file_name = strip_path( shed_tool_conf_dict[ 'config_filename' ] )
+            if shed_tool_conf == file_name:
+                return index, shed_tool_conf_dict
 def get_tool_panel_config_tool_path_install_dir( app, repository ):
     # Return shed-related tool panel config, the tool_path configured in it, and the relative path to the directory where the
     # repository is installed.  This method assumes all repository tools are defined in a single shed-related tool panel config.
@@ -868,7 +1004,7 @@ def get_url_from_repository_tool_shed( app, repository ):
 def handle_missing_data_table_entry( app, tool_path, sample_files, repository_tools_tups ):
     """
     Inspect each tool to see if any have input parameters that are dynamically generated select lists that require entries in the
-    tool_data_table_conf.xml file.
+    tool_data_table_conf.xml file.  This method is not called only from Galaxy (not the tool shed) when a repository is being installed.
     """
     missing_data_table_entry = False
     for index, repository_tools_tup in enumerate( repository_tools_tups ):
@@ -880,8 +1016,8 @@ def handle_missing_data_table_entry( app, tool_path, sample_files, repository_to
         sample_file = None
         # The repository must contain a tool_data_table_conf.xml.sample file that includes all required entries for all tools in the repository.
         for sample_file in sample_files:
-            head, tail = os.path.split( sample_file )
-            if tail == 'tool_data_table_conf.xml.sample':
+            sample_file_name = strip_path( sample_file )
+            if sample_file_name == 'tool_data_table_conf.xml.sample':
                 break
         error, correction_msg = handle_sample_tool_data_table_conf_file( app, sample_file )
         if error:
@@ -892,18 +1028,21 @@ def handle_missing_data_table_entry( app, tool_path, sample_files, repository_to
         repository_tools_tups[ index ] = ( tup_path, guid, repository_tool )
     return repository_tools_tups
 def handle_missing_index_file( app, tool_path, sample_files, repository_tools_tups ):
-    """Inspect each tool to see if it has any input parameters that are dynamically generated select lists that depend on a .loc file."""
+    """
+    Inspect each tool to see if it has any input parameters that are dynamically generated select lists that depend on a .loc file.
+    This method is not called from the tool shed, but from Galaxy when a repository is being installed.
+    """
     sample_files_copied = []
     for index, repository_tools_tup in enumerate( repository_tools_tups ):
         tup_path, guid, repository_tool = repository_tools_tup
         params_with_missing_index_file = repository_tool.params_with_missing_index_file
         for param in params_with_missing_index_file:
             options = param.options
-            missing_file_path, missing_file_name = os.path.split( options.missing_index_file )
+            missing_file_name = strip_path( options.missing_index_file )
             if missing_file_name not in sample_files_copied:
                 # The repository must contain the required xxx.loc.sample file.
                 for sample_file in sample_files:
-                    sample_file_path, sample_file_name = os.path.split( sample_file )
+                    sample_file_name = strip_path( sample_file )
                     if sample_file_name == '%s.sample' % missing_file_name:
                         copy_sample_file( app, sample_file )
                         if options.tool_data_table and options.tool_data_table.missing_index_file:
@@ -916,10 +1055,11 @@ def handle_missing_index_file( app, tool_path, sample_files, repository_tools_tu
     return repository_tools_tups, sample_files_copied
 def handle_sample_tool_data_table_conf_file( app, filename ):
     """
-    Parse the incoming filename and add new entries to the in-memory
-    app.tool_data_tables dictionary as well as appending them to the
+    Parse the incoming filename and add new entries to the in-memory app.tool_data_tables dictionary as well as appending them to the
     shed's tool_data_table_conf.xml file on disk.
     """
+    # TODO: Load an in-memory version of the tool_data_table_conf.xml file, and write it to disk
+    # from the in-memory version only when changes are made.
     error = False
     message = ''
     try:
@@ -927,6 +1067,8 @@ def handle_sample_tool_data_table_conf_file( app, filename ):
     except Exception, e:
         message = str( e )
         error = True
+    """
+    # TODO: eliminate this - the shed should not need to write this to disk...
     if not error:
         # Add an entry to the end of the tool_data_table_conf.xml file.
         tdt_config = "%s/tool_data_table_conf.xml" % app.config.root
@@ -953,42 +1095,32 @@ def handle_sample_tool_data_table_conf_file( app, filename ):
         else:
             message = "The required file named tool_data_table_conf.xml does not exist in the Galaxy install directory."
             error = True
+    """
     return error, message
-def handle_tool_dependencies( current_working_dir, repo_files_dir, repository_tools_tups ):
+def handle_tool_dependencies( app, tool_shed_repository, installed_changeset_revision, tool_dependencies_config ):
     """
-    Inspect each tool to see if it includes a "requirement" that refers to a fabric
-    script.  For those that do, execute the fabric script to install tool dependencies.
+    Install and build tool dependencies defined in the tool_dependencies_config.  This config's tag sets can refer to installation
+    methods in Galaxy's tool_dependencies module or to proprietary fabric scripts contained in the repository.  Future enhancements
+    to handling tool dependencies may provide installation processes in addition to fabric based processes.
+    
+    The dependencies will be installed in:
+    ~/<app.config.tool_dependency_dir>/<package_name>/<package_version>/<repository_owner>/<repository_name>/<installed_changeset_revision>
     """
-    for index, repository_tools_tup in enumerate( repository_tools_tups ):
-        tup_path, guid, repository_tool = repository_tools_tup
-        for requirement in repository_tool.requirements:
-            if requirement.type == 'fabfile':
-                log.debug( 'Executing fabric script to install dependencies for tool "%s"...' % repository_tool.name )
-                fabfile = requirement.fabfile
-                method = requirement.method
-                # Find the relative path to the fabfile.
-                relative_fabfile_path = None
-                for root, dirs, files in os.walk( repo_files_dir ):
-                    for name in files:
-                        if name == fabfile:
-                            relative_fabfile_path = os.path.join( root, name )
-                            break
-                if relative_fabfile_path:
-                    # cmd will look something like: fab -f fabfile.py install_bowtie
-                    cmd = 'fab -f %s %s' % ( relative_fabfile_path, method )
-                    tmp_name = tempfile.NamedTemporaryFile().name
-                    tmp_stderr = open( tmp_name, 'wb' )
-                    os.chdir( repo_files_dir )
-                    proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() )
-                    returncode = proc.wait()
-                    os.chdir( current_working_dir )
-                    tmp_stderr.close()
-                    if returncode != 0:
-                        # TODO: do something more here than logging the problem.
-                        tmp_stderr = open( tmp_name, 'rb' )
-                        error = tmp_stderr.read()
-                        tmp_stderr.close()
-                        log.debug( 'Problem installing dependencies for tool "%s"\n%s' % ( repository_tool.name, error ) )
+    status = 'ok'
+    message = ''
+    # Parse the tool_dependencies.xml config.
+    tree = ElementTree.parse( tool_dependencies_config )
+    root = tree.getroot()
+    ElementInclude.include( root )
+    fabric_version_checked = False
+    for elem in root:
+        if elem.tag == 'package':
+            error_message = install_package( app, elem, tool_shed_repository, installed_changeset_revision )
+            if error_message:
+                message += '  %s' % error_message
+    if message:
+        status = 'error'
+    return status, message
 def handle_tool_versions( app, tool_version_dicts, tool_shed_repository ):
     """
     Using the list of tool_version_dicts retrieved from the tool shed (one per changeset revison up to the currently installed changeset revision),
@@ -1019,7 +1151,13 @@ def handle_tool_versions( app, tool_version_dicts, tool_shed_repository ):
 def load_datatype_items( app, repository, relative_install_dir, deactivate=False ):
     # Load proprietary datatypes.
     metadata = repository.metadata
-    datatypes_config = metadata.get( 'datatypes_config', None )
+    work_dir = make_tmp_directory()
+    datatypes_config = get_config_from_repository( app,
+                                                   'datatypes_conf.xml',
+                                                   repository,
+                                                   repository.changeset_revision,
+                                                   work_dir,
+                                                   install_dir=relative_install_dir )
     if datatypes_config:
         converter_path, display_path = alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir, deactivate=deactivate )
         if converter_path or display_path:
@@ -1037,14 +1175,18 @@ def load_datatype_items( app, repository, relative_install_dir, deactivate=False
         if display_path:
             # Load or deactivate proprietary datatype display applications
             app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict, deactivate=deactivate )
+    try:
+        shutil.rmtree( work_dir )
+    except:
+        pass
 def load_repository_contents( trans, repository_name, description, owner, changeset_revision, ctx_rev, tool_path, repository_clone_url,
-                              relative_install_dir, current_working_dir, tool_shed=None, tool_section=None, shed_tool_conf=None,
-                              install_tool_dependencies=False ):
-    """Generate the metadata for the installed tool shed repository, among other things."""
-    # It is critical that the installed repository is updated to the desired changeset_revision before metadata is set because the
-    # process for setting metadata uses the repository files on disk.  This method is called when an admin is installing a new repository
-    # or reinstalling an uninstalled repository. 
-    metadata_dict = generate_metadata( trans.app.toolbox, relative_install_dir, repository_clone_url )
+                              relative_install_dir, tool_shed=None, tool_section=None, shed_tool_conf=None, install_tool_dependencies=False ):
+    """
+    Generate the metadata for the installed tool shed repository, among other things.  This method is called from Galaxy (never the tool shed)
+    when an admin is installing a new repository or reinstalling an uninstalled repository.
+    """
+    message = ''
+    metadata_dict = generate_metadata_using_disk_files( trans.app.toolbox, relative_install_dir, repository_clone_url )
     # Add a new record to the tool_shed_repository table if one doesn't already exist.  If one exists but is marked deleted, undelete it.  This
     # must happen before the call to add_to_tool_panel() below because tools will not be properly loaded if the repository is marked deleted.
     log.debug( "Adding new row (or updating an existing row) for repository '%s' in the tool_shed_repository table." % repository_name )
@@ -1060,6 +1202,7 @@ def load_repository_contents( trans, repository_name, description, owner, change
         tool_panel_dict = generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section )
         repository_tools_tups = get_repository_tools_tups( trans.app, metadata_dict )
         if repository_tools_tups:
+            work_dir = make_tmp_directory()
             sample_files = metadata_dict.get( 'sample_files', [] )
             # Handle missing data table entries for tool parameters that are dynamically generated select lists.
             repository_tools_tups = handle_missing_data_table_entry( trans.app, tool_path, sample_files, repository_tools_tups )
@@ -1067,7 +1210,21 @@ def load_repository_contents( trans, repository_name, description, owner, change
             repository_tools_tups, sample_files_copied = handle_missing_index_file( trans.app, tool_path, sample_files, repository_tools_tups )
             # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance.
             copy_sample_files( trans.app, sample_files, sample_files_copied=sample_files_copied )
-            handle_tool_dependencies( current_working_dir, relative_install_dir, repository_tools_tups )
+            if install_tool_dependencies and 'tool_dependencies' in metadata_dict:
+                # Get the tool_dependencies.xml file from the repository.
+                tool_dependencies_config = get_config_from_repository( trans.app,
+                                                                       'tool_dependencies.xml',
+                                                                       tool_shed_repository,
+                                                                       changeset_revision,
+                                                                       work_dir )
+                # Install dependencies for repository tools.
+                status, message = handle_tool_dependencies( app=trans.app,
+                                                            tool_shed_repository=tool_shed_repository,
+                                                            installed_changeset_revision=changeset_revision,
+                                                            tool_dependencies_config=tool_dependencies_config )
+                if status != 'ok' and message:
+                    print 'The following error occurred from load_repository_contents while installing tool dependencies:'
+                    print message
             add_to_tool_panel( app=trans.app,
                                repository_name=repository_name,
                                repository_clone_url=repository_clone_url,
@@ -1077,8 +1234,17 @@ def load_repository_contents( trans, repository_name, description, owner, change
                                shed_tool_conf=shed_tool_conf,
                                tool_panel_dict=tool_panel_dict,
                                new_install=True )
-    if 'datatypes_config' in metadata_dict:
-        datatypes_config = os.path.abspath( metadata_dict[ 'datatypes_config' ] )
+            try:
+                shutil.rmtree( work_dir )
+            except:
+                pass
+    if 'datatypes' in metadata_dict:
+        work_dir = make_tmp_directory()
+        datatypes_config = get_config_from_repository( trans.app,
+                                                       'datatypes_conf.xml',
+                                                       tool_shed_repository,
+                                                       changeset_revision,
+                                                       work_dir )
         # Load data types required by tools.
         converter_path, display_path = alter_config_and_load_prorietary_datatypes( trans.app, datatypes_config, relative_install_dir, override=False )
         if converter_path or display_path:
@@ -1096,7 +1262,22 @@ def load_repository_contents( trans, repository_name, description, owner, change
         if display_path:
             # Load proprietary datatype display applications
             trans.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict )
-    return tool_shed_repository, metadata_dict
+        try:
+            shutil.rmtree( work_dir )
+        except:
+            pass
+    return tool_shed_repository, metadata_dict, message
+def make_tmp_directory():
+    tmp_dir = os.getenv( 'TMPDIR', '' )
+    if tmp_dir:
+        tmp_dir = tmp_dir.strip()
+    else:
+        home_dir = os.getenv( 'HOME' )
+        tmp_dir = os.path.join( home_dir, 'tmp' )
+    work_dir = os.path.join( tmp_dir, 'work_tmp' )
+    if not os.path.exists( work_dir ):
+        os.makedirs( work_dir )
+    return work_dir
 def panel_entry_per_tool( tool_section_dict ):
     # Return True if tool_section_dict looks like this.
     # {<Tool guid> : [{ tool_config : <tool_config_file>, id: <ToolSection id>, version : <ToolSection version>, name : <TooSection name>}]}
@@ -1157,6 +1338,10 @@ def remove_from_tool_panel( trans, repository, shed_tool_conf, uninstall ):
     trans.sa_session.flush()
     # Create a list of guids for all tools that will be removed from the in-memory tool panel and config file on disk.
     guids_to_remove = [ k for k in tool_panel_dict.keys() ]
+    # Remove the tools from the toolbox's tools_by_id dictionary.
+    for guid_to_remove in guids_to_remove:
+        if guid_to_remove in trans.app.toolbox.tools_by_id:
+            del trans.app.toolbox.tools_by_id[ guid_to_remove ]
     index, shed_tool_conf_dict = get_shed_tool_conf_dict( trans.app, shed_tool_conf )
     if uninstall:
         # Remove from the shed_tool_conf file on disk.
@@ -1194,7 +1379,7 @@ def remove_from_tool_panel( trans, repository, shed_tool_conf, uninstall ):
                 config_elems_to_remove.append( config_elem )
         elif config_elem.tag == 'tool':
             if config_elem.get( 'guid' ) in guids_to_remove:
-                tool_key = key = 'tool_%s' % str( config_elem.get( 'guid' ) )
+                tool_key = 'tool_%s' % str( config_elem.get( 'guid' ) )
                 if tool_key in trans.app.toolbox.tool_panel:
                     del trans.app.toolbox.tool_panel[ tool_key ]
                 if uninstall:
@@ -1213,6 +1398,9 @@ def remove_from_tool_panel( trans, repository, shed_tool_conf, uninstall ):
     if uninstall:
         # Write the current in-memory version of the integrated_tool_panel.xml file to disk.
         trans.app.toolbox.write_integrated_tool_panel_config_file()
+def strip_path( fpath ):
+    file_path, file_name = os.path.split( fpath )
+    return file_name
 def to_html_escaped( text ):
     """Translates the characters in text to html values"""
     translated = []

lib/galaxy/web/base/controller.py

@@ -1,13 +1,12 @@
 """
 Contains functionality needed in every web interface
 """
-import os, time, logging, re, string, sys, glob, shutil, tempfile, subprocess, binascii
+import os, time, logging, re, string, sys, glob, shutil, tempfile, subprocess
 from datetime import date, datetime, timedelta
 from time import strftime
 from galaxy import config, tools, web, util
 from galaxy.util import inflector
 from galaxy.util.hash_util import *
-from galaxy.util.json import json_fix
 from galaxy.web import error, form, url_for
 from galaxy.model.orm import *
 from galaxy.workflow.modules import *
@@ -2463,32 +2462,3 @@ def get_quota( trans, id ):
     id = trans.security.decode_id( id )
     quota = trans.sa_session.query( trans.model.Quota ).get( id )
     return quota
-def tool_shed_encode( val ):
-    if isinstance( val, dict ):
-        value = simplejson.dumps( val )
-    else:
-        value = val
-    a = hmac_new( 'ToolShedAndGalaxyMustHaveThisSameKey', value )
-    b = binascii.hexlify( value )
-    return "%s:%s" % ( a, b )
-def tool_shed_decode( value ):
-    # Extract and verify hash
-    a, b = value.split( ":" )
-    value = binascii.unhexlify( b )
-    test = hmac_new( 'ToolShedAndGalaxyMustHaveThisSameKey', value )
-    assert a == test
-    # Restore from string
-    values = None
-    try:
-        values = simplejson.loads( value )
-    except Exception, e:
-        log.debug( "Decoding json value from tool shed threw exception: %s" % str( e ) )
-    if values is not None:
-        try:
-            return json_fix( values )
-        except Exception, e:
-            log.debug( "Fixing decoded json value from tool shed threw exception: %s" % str( e ) )
-            fixed_values = values
-    if values is None:
-        values = value
-    return values

lib/galaxy/web/controllers/admin_toolshed.py

@@ -2,6 +2,8 @@ import urllib2
 from galaxy.web.controllers.admin import *
 from galaxy.util.json import from_json_string, to_json_string
 from galaxy.util.shed_util import *
+from galaxy.tool_shed.tool_dependencies.install_util import get_install_dir, not_installed
+from galaxy.tool_shed.encoding_util import *
 from galaxy import eggs, tools
 
 eggs.require( 'mercurial' )
@@ -180,21 +182,43 @@ class AdminToolshed( AdminGalaxy ):
         remove_from_disk_checked = CheckboxField.is_checked( remove_from_disk )
         repository = get_repository( trans, kwd[ 'id' ] )
         shed_tool_conf, tool_path, relative_install_dir = get_tool_panel_config_tool_path_install_dir( trans.app, repository )
+        repository_install_dir = os.path.abspath ( relative_install_dir )
         if params.get( 'deactivate_or_uninstall_repository_button', False ):
             if repository.includes_tools:
                 # Handle tool panel alterations.
                 remove_from_tool_panel( trans, repository, shed_tool_conf, uninstall=remove_from_disk_checked )
             if repository.includes_datatypes:
                 # Deactivate proprietary datatypes.
-                load_datatype_items( trans.app, repository, relative_install_dir, deactivate=True )
+                load_datatype_items( trans.app, repository, repository_install_dir, deactivate=True )
             if remove_from_disk_checked:
-                # Remove the repository from disk.
                 try:
-                    shutil.rmtree( relative_install_dir )
-                    log.debug( "Removed repository installation directory: %s" % str( relative_install_dir ) )
+                    # Remove the repository from disk.
+                    shutil.rmtree( repository_install_dir )
+                    log.debug( "Removed repository installation directory: %s" % str( repository_install_dir ) )
+                    removed = True
                 except Exception, e:
-                    log.debug( "Error removing repository installation directory %s: %s" % ( str( relative_install_dir ), str( e ) ) )
-                repository.uninstalled = True
+                    log.debug( "Error removing repository installation directory %s: %s" % ( str( repository_install_dir ), str( e ) ) )
+                    removed = False
+                if removed:
+                    repository.uninstalled = True
+                    # Remove all installed tool dependencies - this is required when uninstalling the repository.
+                    for tool_dependency in repository.tool_dependencies:
+                        dependency_install_dir = os.path.abspath( os.path.join( trans.app.config.tool_dependency_dir,
+                                                                                tool_dependency.name,
+                                                                                tool_dependency.version,
+                                                                                repository.owner,
+                                                                                repository.name,
+                                                                                tool_dependency.installed_changeset_revision ) )
+                        try:
+                            shutil.rmtree( dependency_install_dir )
+                            log.debug( "Removed tool dependency installation directory: %s" % str( dependency_install_dir ) )
+                            removed = True
+                        except Exception, e:
+                            log.debug( "Error removing tool dependency installation directory %s: %s" % ( str( dependency_install_dir ), str( e ) ) )
+                            removed = False
+                        if removed:
+                            tool_dependency.uninstalled = True
+                            trans.sa_session.add( tool_dependency )
             repository.deleted = True
             trans.sa_session.add( repository )
             trans.sa_session.flush()
@@ -229,6 +253,104 @@ class AdminToolshed( AdminGalaxy ):
         return trans.response.send_redirect( url )
     @web.expose
     @web.require_admin
+    def install_tool_dependencies( self, trans, **kwd ):
+        """Install dependencies for tools included in the repository when the repository is being installed."""
+        message = kwd.get( 'message', ''  )
+        status = kwd.get( 'status', 'done' )
+        tool_shed_url = kwd[ 'tool_shed_url' ]
+        repo_info_dict = kwd[ 'repo_info_dict' ]
+        includes_tools = util.string_as_bool( kwd.get( 'includes_tools', False ) )
+        # Decode the encoded repo_info_dict param value.
+        dict_with_tool_dependencies = tool_shed_decode( repo_info_dict )
+        # The repo_info_dict includes tool dependencies which we need to display so the user knows what will be installed.
+        new_repo_info_dict = {}
+        for name, repo_info_tuple in dict_with_tool_dependencies.items():
+            description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, tool_dependencies = repo_info_tuple
+            # Create a new repo_info_dict by eliminating tool-dependencies from the repo_info_tuple.
+            new_repo_info_dict[ name ] = ( description, repository_clone_url, changeset_revision, ctx_rev )
+        repo_info_dict = tool_shed_encode( new_repo_info_dict )
+        install_tool_dependencies_check_box = CheckboxField( 'install_tool_dependencies', checked=True )
+        return trans.fill_template( '/admin/tool_shed_repository/install_tool_dependencies.mako',
+                                    tool_shed_url=tool_shed_url,
+                                    repo_info_dict=repo_info_dict,
+                                    dict_with_tool_dependencies=dict_with_tool_dependencies,
+                                    includes_tools=includes_tools,
+                                    install_tool_dependencies_check_box=install_tool_dependencies_check_box,
+                                    message=message,
+                                    status=status )
+    @web.expose
+    @web.require_admin
+    def install_missing_tool_dependencies( self, trans, **kwd ):
+        """
+        Install dependencies for tools included in the repository that were not installed when the repository was installed or that are
+        being reinstalled after the repository was uninstalled.
+        """
+        reinstalling = util.string_as_bool( kwd.get( 'reinstalling', False ) )
+        message = kwd.get( 'message', ''  )
+        status = kwd.get( 'status', 'done' )
+        repository = get_repository( trans, kwd[ 'id' ] )
+        install_tool_dependencies = CheckboxField.is_checked( kwd.get( 'install_tool_dependencies', '' ) )
+        if not reinstalling and install_tool_dependencies and kwd.get( 'install_missing_tool_dependencies_button', False ):
+            shed_tool_conf, tool_path, relative_install_dir = get_tool_panel_config_tool_path_install_dir( trans.app, repository )
+            repo_files_dir = os.path.abspath( os.path.join( relative_install_dir, repository.name ) )
+            # Get the tool_dependencies.xml file from the repository.
+            work_dir = make_tmp_directory()
+            tool_dependencies_config = get_config_from_repository( trans.app,
+                                                                   'tool_dependencies.xml',
+                                                                   repository,
+                                                                   repository.changeset_revision,
+                                                                   work_dir,
+                                                                   install_dir=relative_install_dir )
+            status, message = handle_tool_dependencies( app=trans.app,
+                                                        tool_shed_repository=repository,
+                                                        installed_changeset_revision=repository.installed_changeset_revision,
+                                                        tool_dependencies_config=tool_dependencies_config )
+            try:
+                shutil.rmtree( work_dir )
+            except:
+                pass
+            tool_dependencies_missing = status == 'error'
+            return trans.fill_template( '/admin/tool_shed_repository/manage_repository.mako',
+                                        repository=repository,
+                                        description=repository.description,
+                                        repo_files_dir=repo_files_dir,
+                                        tool_dependencies_missing=tool_dependencies_missing,
+                                        message=message,
+                                        status=status )
+        if reinstalling and kwd.get( 'install_missing_tool_dependencies_button', False ):
+            # The user has been presented the option to install tool dependencies, so redirect to reinstall the repository, sending
+            # along the user's choice.
+            return trans.response.send_redirect( web.url_for( controller='admin_toolshed',
+                                                              action='reinstall_repository',
+                                                              **kwd ) )
+        tool_dependencies = repository.metadata[ 'tool_dependencies' ]
+        install_tool_dependencies_check_box = CheckboxField( 'install_tool_dependencies', checked=True )
+        if not reinstalling:
+            # Filter the tool_dependencies dictionary to eliminate successfully installed dependencies.
+            filtered_tool_dependencies = {}
+            for dependency_key, requirements_dict in tool_dependencies.items():
+                name = requirements_dict[ 'name' ]
+                version = requirements_dict[ 'version' ]
+                install_dir = get_install_dir( trans.app, repository, repository.changeset_revision, name, version )
+                if not_installed( install_dir ):
+                    filtered_tool_dependencies[ dependency_key ] = requirements_dict
+            tool_dependencies = filtered_tool_dependencies
+        no_changes = kwd.get( 'no_changes', '' )
+        no_changes_checked = CheckboxField.is_checked( no_changes )
+        new_tool_panel_section = kwd.get( 'new_tool_panel_section', '' )
+        tool_panel_section = kwd.get( 'tool_panel_section', '' )
+        return trans.fill_template( '/admin/tool_shed_repository/install_missing_tool_dependencies.mako',
+                                    repository=repository,
+                                    reinstalling=reinstalling,
+                                    tool_dependencies=tool_dependencies,
+                                    no_changes_checked=no_changes_checked,
+                                    new_tool_panel_section=new_tool_panel_section,
+                                    tool_panel_section=tool_panel_section,
+                                    install_tool_dependencies_check_box=install_tool_dependencies_check_box,
+                                    message=message,
+                                    status=status )
+    @web.expose
+    @web.require_admin
     def install_repository( self, trans, **kwd ):
         if not trans.app.toolbox.shed_tool_confs:
             message = 'The <b>tool_config_file</b> setting in <b>universe_wsgi.ini</b> must include at least one shed tool configuration file name with a '
@@ -246,10 +368,13 @@ class AdminToolshed( AdminGalaxy ):
         new_tool_panel_section = kwd.get( 'new_tool_panel_section', '' )
         tool_panel_section = kwd.get( 'tool_panel_section', '' )
         includes_tools = util.string_as_bool( kwd.get( 'includes_tools', False ) )
+        install_tool_dependencies = kwd.get( 'install_tool_dependencies', '' )
         if not includes_tools or ( includes_tools and kwd.get( 'select_tool_panel_section_button', False ) ):
             if includes_tools:
+                install_tool_dependencies = CheckboxField.is_checked( install_tool_dependencies )
                 shed_tool_conf = kwd[ 'shed_tool_conf' ]
             else:
+                install_tool_dependencies = False
                 # If installing a repository that includes no tools, get the relative tool_path from the file
                 # to which the migrated_tools_config setting points.
                 shed_tool_conf = trans.app.config.migrated_tools_config
@@ -261,8 +386,8 @@ class AdminToolshed( AdminGalaxy ):
                     tool_path = shed_tool_conf_dict[ 'tool_path' ]
                     break
                 else:
-                    head, tail = os.path.split( config_filename )
-                    if tail == shed_tool_conf:
+                    file_name = strip_path( config_filename )
+                    if file_name == shed_tool_conf:
                         tool_path = shed_tool_conf_dict[ 'tool_path' ]
                         break
             if includes_tools and ( new_tool_panel_section or tool_panel_section ):
@@ -290,7 +415,6 @@ class AdminToolshed( AdminGalaxy ):
             # Decode the encoded repo_info_dict param value.
             repo_info_dict = tool_shed_decode( repo_info_dict )
             # Clone the repository to the configured location.
-            current_working_dir = os.getcwd()
             installed_repository_names = []
             for name, repo_info_tuple in repo_info_dict.items():
                 description, repository_clone_url, changeset_revision, ctx_rev = repo_info_tuple
@@ -303,19 +427,22 @@ class AdminToolshed( AdminGalaxy ):
                     clone_repository( repository_clone_url, os.path.abspath( relative_install_dir ), ctx_rev )
                     owner = get_repository_owner( clean_repository_clone_url( repository_clone_url ) )
                     tool_shed = clean_tool_shed_url( tool_shed_url )
-                    tool_shed_repository, metadata_dict = load_repository_contents( trans,
-                                                                                    repository_name=name,
-                                                                                    description=description,
-                                                                                    owner=owner,
-                                                                                    changeset_revision=changeset_revision,
-                                                                                    ctx_rev=ctx_rev,
-                                                                                    tool_path=tool_path,
-                                                                                    repository_clone_url=repository_clone_url,
-                                                                                    relative_install_dir=relative_install_dir,
-                                                                                    current_working_dir=current_working_dir,
-                                                                                    tool_shed=tool_shed,
-                                                                                    tool_section=tool_section,
-                                                                                    shed_tool_conf=shed_tool_conf )
+                    tool_shed_repository, metadata_dict, error_message = load_repository_contents( trans,
+                                                                                                   repository_name=name,
+                                                                                                   description=description,
+                                                                                                   owner=owner,
+                                                                                                   changeset_revision=changeset_revision,
+                                                                                                   ctx_rev=ctx_rev,
+                                                                                                   tool_path=tool_path,
+                                                                                                   repository_clone_url=repository_clone_url,
+                                                                                                   relative_install_dir=relative_install_dir,
+                                                                                                   tool_shed=tool_shed,
+                                                                                                   tool_section=tool_section,
+                                                                                                   shed_tool_conf=shed_tool_conf,
+                                                                                                   install_tool_dependencies=install_tool_dependencies )
+                    if error_message:
+                        message += error_message
+                        status = 'error'
                     if 'tools' in metadata_dict:
                         # Get the tool_versions from the tool shed for each tool in the installed change set.
                         url = '%srepository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % \
@@ -335,12 +462,21 @@ class AdminToolshed( AdminGalaxy ):
             if installed_repository_names:                
                 installed_repository_names.sort()
                 num_repositories_installed = len( installed_repository_names )
-                if tool_section:
-                    message += 'Installed %d %s and all tools were loaded into tool panel section <b>%s</b>:<br/>Installed repositories: ' % \
-                        ( num_repositories_installed, inflector.cond_plural( num_repositories_installed, 'repository' ), tool_section.name )
+                if install_tool_dependencies:
+                    dependency_str = ' along with tool dependencies '
                 else:
-                    message += 'Installed %d %s and all tools were loaded into the tool panel outside of any sections.<br/>Installed repositories: ' % \
-                        ( num_repositories_installed, inflector.cond_plural( num_repositories_installed, 'repository' ) )
+                    dependency_str = ''
+                if tool_section:
+                    message += 'Installed %d %s%sand all tools were loaded into tool panel section <b>%s</b>:<br/>Installed repositories: ' % \
+                        ( num_repositories_installed,
+                          inflector.cond_plural( num_repositories_installed, 'repository' ),
+                          dependency_str,
+                          tool_section.name )
+                else:
+                    message += 'Installed %d %s%s and all tools were loaded into the tool panel outside of any sections.<br/>Installed repositories: ' % \
+                        ( num_repositories_installed,
+                          inflector.cond_plural( num_repositories_installed, 'repository' ),
+                          dependency_str )
                 for i, repo_name in enumerate( installed_repository_names ):
                     if i == len( installed_repository_names ) -1:
                         message += '%s.<br/>' % repo_name
@@ -387,6 +523,7 @@ class AdminToolshed( AdminGalaxy ):
                                     repo_info_dict=repo_info_dict,
                                     shed_tool_conf=shed_tool_conf,
                                     includes_tools=includes_tools,
+                                    install_tool_dependencies=install_tool_dependencies,
                                     shed_tool_conf_select_field=shed_tool_conf_select_field,
                                     tool_panel_section_select_field=tool_panel_section_select_field,
                                     new_tool_panel_section=new_tool_panel_section,
@@ -411,26 +548,57 @@ class AdminToolshed( AdminGalaxy ):
             message = "The repository information has been updated."
         elif params.get( 'set_metadata_button', False ):
             repository_clone_url = generate_clone_url( trans, repository )
-            metadata_dict = generate_metadata( trans.app.toolbox, relative_install_dir, repository_clone_url )
+            metadata_dict = generate_metadata_using_disk_files( trans.app.toolbox, relative_install_dir, repository_clone_url )
             if metadata_dict:
                 repository.metadata = metadata_dict
                 trans.sa_session.add( repository )
                 trans.sa_session.flush()
             message = "Repository metadata has been reset."
+        tool_dependencies_missing = False
+        if repository.includes_tool_dependencies:
+            # See if any tool dependencies need to be installed, get the tool_dependencies.xml file from the repository.
+            work_dir = make_tmp_directory()
+            tool_dependencies_config = get_config_from_repository( trans.app,
+                                                                   'tool_dependencies.xml',
+                                                                   repository,
+                                                                   repository.changeset_revision,
+                                                                   work_dir,
+                                                                   install_dir=relative_install_dir )
+            # Parse the tool_dependencies.xml config.
+            tree = ElementTree.parse( tool_dependencies_config )
+            root = tree.getroot()
+            ElementInclude.include( root )
+            fabric_version_checked = False
+            for elem in root:
+                if elem.tag == 'package':
+                    package_name = elem.get( 'name', None )
+                    package_version = elem.get( 'version', None )
+                    if package_name and package_version:
+                        install_dir = get_install_dir( trans.app, repository, repository.installed_changeset_revision, package_name, package_version )
+                        if not_installed( install_dir ):
+                            tool_dependencies_missing = True
+                            break
+            try:
+                shutil.rmtree( work_dir )
+            except:
+                pass
         return trans.fill_template( '/admin/tool_shed_repository/manage_repository.mako',
                                     repository=repository,
                                     description=description,
                                     repo_files_dir=repo_files_dir,
+                                    tool_dependencies_missing=tool_dependencies_missing,
                                     message=message,
                                     status=status )
     @web.expose
     @web.require_admin
     def reinstall_repository( self, trans, **kwd ):
+        message = kwd.get( 'message', '' )
+        status = kwd.get( 'status', 'done' )
         repository = get_repository( trans, kwd[ 'id' ] )
         no_changes = kwd.get( 'no_changes', '' )
         no_changes_checked = CheckboxField.is_checked( no_changes )
+        install_tool_dependencies = CheckboxField.is_checked( kwd.get( 'install_tool_dependencies', '' ) )
         shed_tool_conf, tool_path, relative_install_dir = get_tool_panel_config_tool_path_install_dir( trans.app, repository )
-        current_working_dir = os.getcwd()
         repository_clone_url = generate_clone_url( trans, repository )
         clone_dir = os.path.join( tool_path, self.__generate_tool_path( repository_clone_url, repository.installed_changeset_revision ) )
         relative_install_dir = os.path.join( clone_dir, repository.name )
@@ -496,25 +664,31 @@ class AdminToolshed( AdminGalaxy ):
                     tool_section = trans.app.toolbox.tool_panel[ section_key ]
                 else:
                     tool_section = None
-        tool_shed_repository, metadata_dict = load_repository_contents( trans,
-                                                                        repository_name=repository.name,
-                                                                        description=repository.description,
-                                                                        owner=repository.owner,
-                                                                        changeset_revision=repository.installed_changeset_revision,
-                                                                        ctx_rev=ctx_rev,
-                                                                        tool_path=tool_path,
-                                                                        repository_clone_url=repository_clone_url,
-                                                                        relative_install_dir=relative_install_dir,
-                                                                        current_working_dir=current_working_dir,
-                                                                        tool_shed=repository.tool_shed,
-                                                                        tool_section=tool_section,
-                                                                        shed_tool_conf=shed_tool_conf )
+        tool_shed_repository, metadata_dict, error_message  = load_repository_contents( trans,
+                                                                                        repository_name=repository.name,
+                                                                                        description=repository.description,
+                                                                                        owner=repository.owner,
+                                                                                        changeset_revision=repository.installed_changeset_revision,
+                                                                                        ctx_rev=ctx_rev,
+                                                                                        tool_path=tool_path,
+                                                                                        repository_clone_url=repository_clone_url,
+                                                                                        relative_install_dir=relative_install_dir,
+                                                                                        tool_shed=repository.tool_shed,
+                                                                                        tool_section=tool_section,
+                                                                                        shed_tool_conf=shed_tool_conf,
+                                                                                        install_tool_dependencies=install_tool_dependencies )
+        if error_message:
+            message += error_message
+            status = 'error'
         repository.uninstalled = False
         repository.deleted = False
         trans.sa_session.add( repository )
         trans.sa_session.flush()
-        message = 'The <b>%s</b> repository has been reinstalled.' % repository.name
-        status = 'done'
+        if install_tool_dependencies:
+            dependency_str = ' along with tool dependencies'
+        else:
+            dependency_str = ' without tool dependencies'
+        message += 'The <b>%s</b> repository has been reinstalled%s.  ' % ( repository.name, dependency_str )
         return trans.response.send_redirect( web.url_for( controller='admin_toolshed',
                                                           action='browse_repositories',
                                                           message=message,
@@ -605,7 +779,6 @@ class AdminToolshed( AdminGalaxy ):
             if changeset_revision == latest_changeset_revision:
                 message = "The cloned tool shed repository named '%s' is current (there are no updates available)." % name
             else:
-                current_working_dir = os.getcwd()
                 shed_tool_conf, tool_path, relative_install_dir = get_tool_panel_config_tool_path_install_dir( trans.app, repository )
                 if relative_install_dir:
                     repo_files_dir = os.path.abspath( os.path.join( relative_install_dir, name ) )
@@ -615,7 +788,7 @@ class AdminToolshed( AdminGalaxy ):
                     update_repository( repo, latest_ctx_rev )
                     # Update the repository metadata.
                     tool_shed = clean_tool_shed_url( tool_shed_url )
-                    metadata_dict = generate_metadata( trans.app.toolbox, relative_install_dir, repository_clone_url )
+                    metadata_dict = generate_metadata_using_disk_files( trans.app.toolbox, relative_install_dir, repository_clone_url )
                     repository.metadata = metadata_dict
                     # Update the repository changeset_revision in the database.
                     repository.changeset_revision = latest_changeset_revision

lib/galaxy/web/controllers/workflow.py

@@ -14,6 +14,7 @@ from galaxy.datatypes.data import Data
 from galaxy.util.odict import odict
 from galaxy.util.sanitize_html import sanitize_html
 from galaxy.util.topsort import topsort, topsort_levels, CycleError
+from galaxy.tool_shed.encoding_util import *
 from galaxy.workflow.modules import *
 from galaxy import model
 from galaxy import util

lib/galaxy/webapps/community/config.py

@@ -45,7 +45,7 @@ class Configuration( object ):
         self.id_secret = kwargs.get( "id_secret", "USING THE DEFAULT IS NOT SECURE!" )
         # Tool stuff
         self.tool_secret = kwargs.get( "tool_secret", "" )
-        self.tool_data_path = resolve_path( kwargs.get( "tool_data_path", "tool-data" ), os.getcwd() )
+        self.tool_data_path = resolve_path( kwargs.get( "tool_data_path", "shed-tool-data" ), os.getcwd() )
         self.tool_data_table_config_path = resolve_path( kwargs.get( 'tool_data_table_config_path', 'tool_data_table_conf.xml' ), self.root )
         self.ftp_upload_dir = kwargs.get( 'ftp_upload_dir', None )
         # Location for dependencies
@@ -107,10 +107,26 @@ class Configuration( object ):
         else:
             return default
     def check( self ):
-        # Check that required directories exist
-        for path in self.root, self.file_path, self.template_path:
-            if not os.path.isdir( path ):
-                raise ConfigurationError("Directory does not exist: %s" % path )
+        # Check that required directories exist.
+        paths_to_check = [ self.root, self.file_path, self.tool_data_path, self.template_path ]
+        for path in paths_to_check:
+            if path not in [ None, False ] and not os.path.isdir( path ):
+                try:
+                    os.makedirs( path )
+                except Exception, e:
+                    raise ConfigurationError( "Unable to create missing directory: %s\n%s" % ( path, e ) )
+        # Create the directories that it makes sense to create.
+        for path in self.file_path, \
+                    self.template_cache, \
+                    os.path.join( self.tool_data_path, 'shared', 'jars' ):
+            if path not in [ None, False ] and not os.path.isdir( path ):
+                try:
+                    os.makedirs( path )
+                except Exception, e:
+                    raise ConfigurationError( "Unable to create missing directory: %s\n%s" % ( path, e ) )
+        # Check that required files exist.
+        if not os.path.isfile( self.datatypes_config ):
+            raise ConfigurationError( "File not found: %s" % self.datatypes_config )
     def is_admin_user( self, user ):
         """
         Determine if the provided user is listed in `admin_users`.

lib/galaxy/webapps/community/controllers/admin.py

@@ -3,7 +3,7 @@ from galaxy.webapps.community import model
 from galaxy.model.orm import *
 from galaxy.web.framework.helpers import time_ago, iff, grids
 from galaxy.util import inflector
-from galaxy.util.shed_util import get_configured_ui
+from galaxy.util.shed_util import get_changectx_for_changeset, get_configured_ui
 from common import *
 from repository import RepositoryListGrid, CategoryListGrid
 
@@ -462,27 +462,38 @@ class AdminController( BaseUIController, Admin ):
                                                    status=status ) )
     @web.expose
     @web.require_admin
-    def reset_all_repository_metadata( self, trans, **kwd ):
+    def reset_metadata_on_all_repositories( self, trans, **kwd ):
         params = util.Params( kwd )
         message = util.restore_text( params.get( 'message', ''  ) )
         status = params.get( 'status', 'done' )
-        if 'reset_all_repository_metadata_button' in kwd:
-            count = 0
+        if 'reset_metadata_on_all_repositories_button' in kwd:
+            successful_count = 0
+            unsuccessful_count = 0
             for repository in trans.sa_session.query( trans.model.Repository ) \
                                               .filter( trans.model.Repository.table.c.deleted == False ):
                 try:
-                    reset_all_repository_metadata( trans, trans.security.encode_id( repository.id ) )
-                    log.debug( "Reset metadata on repository %s" % repository.name )
-                    count += 1
+                    error_message, status = reset_all_metadata_on_repository( trans, trans.security.encode_id( repository.id ) )
+                    if error_message:
+                        log.debug( "Error attempting to reset metadata on repository '%s': %s" % ( repository.name, error_message ) )
+                        unsuccessful_count += 1
+                    else:
+                        log.debug( "Successfully reset metadata on repository %s" % repository.name )
+                        successful_count += 1
                 except Exception, e:
-                    log.debug( "Error attempting to reset metadata on repository %s: %s" % ( repository.name, str( e ) ) )
-            message = "Reset metadata on %d repositories" % count
+                    log.debug( "Error attempting to reset metadata on repository '%s': %s" % ( repository.name, str( e ) ) )
+                    unsuccessful_count += 1
+            message = "Successfully reset metadata on %d %s.  " % ( successful_count,
+                                                                    inflector.cond_plural( successful_count, "repository" ) )            
+            if unsuccessful_count:
+                message += "Error setting metadata on %d %s - see the paster log for details.  " % ( unsuccessful_count,
+                                                                                                     inflector.cond_plural( unsuccessful_count,
+                                                                                                                            "repository" ) )
             trans.response.send_redirect( web.url_for( controller='admin',
                                                        action='browse_repository_metadata',
                                                        webapp='community',
                                                        message=util.sanitize_text( message ),
                                                        status=status ) )
-        return trans.fill_template( '/webapps/community/admin/reset_all_repository_metadata.mako',
+        return trans.fill_template( '/webapps/community/admin/reset_metadata_on_all_repositories.mako',
                                     message=message,
                                     status=status )
     @web.expose

lib/galaxy/webapps/community/controllers/hg.py

@@ -26,7 +26,9 @@ class HgController( BaseUIController ):
                 owner, name = path_info.split( '/' )
                 repository = get_repository_by_name_and_owner( trans, name, owner )
                 if repository:
-                    reset_all_repository_metadata( trans, trans.security.encode_id( repository.id ) )
+                    error_message, status = reset_all_metadata_on_repository( trans, trans.security.encode_id( repository.id ) )
+                    if error_message:
+                        log.debug( "Error resetting all metadata on repository '%s': %s" % ( str( repository.name ), str( error_message ) ) )
         return wsgi_app
 
 def make_web_app():

lib/galaxy/webapps/community/controllers/repository.py

@@ -10,7 +10,7 @@ from galaxy.webapps.community.model import directory_hash_id
 from galaxy.web.framework.helpers import time_ago, iff, grids
 from galaxy.util.json import from_json_string, to_json_string
 from galaxy.model.orm import *
-from galaxy.util.shed_util import get_configured_ui
+from galaxy.util.shed_util import get_changectx_for_changeset, get_configured_ui, make_tmp_directory, NOT_TOOL_CONFIGS, strip_path
 from common import *
 
 from galaxy import eggs
@@ -21,6 +21,7 @@ log = logging.getLogger( __name__ )
 
 MAX_CONTENT_SIZE = 32768
 VALID_REPOSITORYNAME_RE = re.compile( "^[a-z0-9\_]+$" )
+README_FILES = [ 'readme', 'read_me', 'install' ]
     
 class CategoryListGrid( grids.Grid ):
     class NameColumn( grids.TextColumn ):
@@ -805,19 +806,36 @@ class RepositoryController( BaseUIController, ItemRatings ):
         repository = get_repository( trans, repository_id )
         changeset_revision = util.restore_text( params.get( 'changeset_revision', repository.tip ) )
         repository_metadata = get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision )
-        # Tell the caller if the repository includes Galaxy tools so the page
-        # enabling selection of the tool panel section can be displayed.
-        includes_tools = 'tools' in repository_metadata.metadata
+        metadata = repository_metadata.metadata
+        # Tell the caller if the repository includes Galaxy tools so the page enabling selection of the tool panel section can be displayed.
+        includes_tools = 'tools' in metadata
+        includes_tool_dependencies = 'tool_dependencies' in metadata
         # Get the changelog rev for this changeset_revision.
         repo_dir = repository.repo_path
         repo = hg.repository( get_configured_ui(), repo_dir )
         ctx = get_changectx_for_changeset( repo, changeset_revision )
         repo_info_dict = {}
-        repo_info_dict[ repository.name ] = ( repository.description, repository_clone_url, changeset_revision, str( ctx.rev() ) )
+        if includes_tool_dependencies:
+            repo_info_dict[ repository.name ] = ( repository.description,
+                                                  repository_clone_url,
+                                                  changeset_revision,
+                                                  str( ctx.rev() ),
+                                                  repository.user.username,
+                                                  metadata[ 'tool_dependencies' ] )
+        else:
+            repo_info_dict[ repository.name ] = ( repository.description,
+                                                  repository_clone_url,
+                                                  changeset_revision,
+                                                  str( ctx.rev() ) )
         encoded_repo_info_dict = encode( repo_info_dict )
-        # Redirect back to local Galaxy to perform install.
-        url = '%sadmin_toolshed/install_repository?tool_shed_url=%s&repo_info_dict=%s&includes_tools=%s' % \
-            ( galaxy_url, url_for( '/', qualified=True ), encoded_repo_info_dict, str( includes_tools ) )
+        if includes_tool_dependencies:
+            # Redirect back to local Galaxy to present the option to install tool dependencies.
+            url = '%sadmin_toolshed/install_tool_dependencies?tool_shed_url=%s&repo_info_dict=%s&includes_tools=%s' % \
+                ( galaxy_url, url_for( '/', qualified=True ), encoded_repo_info_dict, str( includes_tools ) )
+        else:
+            # Redirect back to local Galaxy to perform install.
+            url = '%sadmin_toolshed/install_repository?tool_shed_url=%s&repo_info_dict=%s&includes_tools=%s' % \
+                ( galaxy_url, url_for( '/', qualified=True ), encoded_repo_info_dict, str( includes_tools ) )
         return trans.response.send_redirect( url )
     @web.expose
     def get_ctx_rev( self, trans, **kwd ):
@@ -838,11 +856,15 @@ class RepositoryController( BaseUIController, ItemRatings ):
         repository_name = kwd[ 'name' ]
         repository_owner = kwd[ 'owner' ]
         changeset_revision = kwd[ 'changeset_revision' ]
+        valid_filenames = [ r for r in README_FILES ]
+        for r in README_FILES:
+            valid_filenames.append( '%s.txt' % r )
+        valid_filenames.append( '%s.txt' % repository_name )
         repository = get_repository_by_name_and_owner( trans, repository_name, repository_owner )
         repo_dir = repository.repo_path
         for root, dirs, files in os.walk( repo_dir ):
             for name in files:
-                if name.lower() in [ 'readme', 'readme.txt', 'read_me', 'read_me.txt', '%s.txt' % repository_name ]:
+                if name.lower() in valid_filenames:
                     f = open( os.path.join( root, name ), 'r' )
                     text = f.read()
                     f.close()
@@ -851,9 +873,8 @@ class RepositoryController( BaseUIController, ItemRatings ):
     @web.expose
     def get_tool_versions( self, trans, **kwd ):
         """
-        For each valid /downloadable change set (up to the received changeset_revision) in the
-        repository's change log, append the change set's tool_versions dictionary to the list
-        that will be returned.
+        For each valid /downloadable change set (up to the received changeset_revision) in the repository's change log, append the change
+        set's tool_versions dictionary to the list that will be returned.
         """
         name = kwd[ 'name' ]
         owner = kwd[ 'owner' ]
@@ -909,59 +930,33 @@ class RepositoryController( BaseUIController, ItemRatings ):
                                                                                  trans.security.encode_id( repository.id ),
                                                                                  changeset_revision )
             if repository_metadata:
-                # If changeset_revision is in the repository_metadata table for this
-                # repository, then we know there are no additional updates for the tools.
+                # If changeset_revision is in the repository_metadata table for this repository, then we know there are no additional updates
+                # for the tools.
                 if from_update_manager:
                     return no_update
                 else:
                     # Return the same value for changeset_revision and latest_changeset_revision.
                     url += changeset_revision
             else:
-                # The changeset_revision column in the repository_metadata table has been
-                # updated with a new changeset_revision value since the repository was cloned.
-                # Load each tool in the repository's changeset_revision to generate a list of
-                # tool guids, since guids differentiate tools by id and version.
+                # TODO: Re-engineer this to define the change set for update to be the one just before the next change set in the repository_metadata
+                # table for this repository.
+                # The changeset_revision column in the repository_metadata table has been updated with a new changeset_revision value since the
+                # repository was cloned.  Load each tool in the repository's changeset_revision to generate a list of tool guids, since guids
+                # differentiate tools by id and version.
                 ctx = get_changectx_for_changeset( repo, changeset_revision )
-                if ctx is not None:        
+                if ctx is not None:
+                    work_dir = make_tmp_directory()
                     tool_guids = []
                     for filename in ctx:
                         # Find all tool configs in this repository changeset_revision.
-                        if filename != 'datatypes_conf.xml' and filename.endswith( '.xml' ):
-                            fctx = ctx[ filename ]
-                            # Write the contents of the old tool config to a temporary file.
-                            fh = tempfile.NamedTemporaryFile( 'w' )
-                            tmp_filename = fh.name
-                            fh.close()
-                            fh = open( tmp_filename, 'w' )
-                            fh.write( fctx.data() )
-                            fh.close()
-                            if not ( check_binary( tmp_filename ) or check_image( tmp_filename ) or check_gzip( tmp_filename )[ 0 ]
-                                     or check_bz2( tmp_filename )[ 0 ] or check_zip( tmp_filename ) ):
-                                try:
-                                    # Make sure we're looking at a tool config and not a display application config or something else.
-                                    element_tree = util.parse_xml( tmp_filename )
-                                    element_tree_root = element_tree.getroot()
-                                    is_tool = element_tree_root.tag == 'tool'
-                                except Exception, e:
-                                    log.debug( "Error parsing %s, exception: %s" % ( tmp_filename, str( e ) ) )
-                                    is_tool = False
-                                if is_tool:
-                                    try:
-                                        tool = load_tool( trans, tmp_filename )
-                                        valid = True
-                                    except:
-                                        valid = False
-                                    if valid and tool is not None:
-                                        tool_guids.append( generate_tool_guid( trans, repository, tool ) )
-                            try:
-                                os.unlink( tmp_filename )
-                            except:
-                                pass
+                        if filename not in NOT_TOOL_CONFIGS and filename.endswith( '.xml' ):
+                            valid, tool = load_tool_from_tmp_directory( trans, repo, repo_dir, ctx, filename, work_dir )
+                            if valid and tool is not None:
+                                tool_guids.append( generate_tool_guid( trans, repository, tool ) )
                     tool_guids.sort()
                     if tool_guids:
-                        # Compare our list of tool guids against those in each repository_metadata record
-                        # for the repository to find the repository_metadata record with the changeset_revision
-                        # value we want to pass back to the caller.
+                        # Compare our list of tool guids against those in each repository_metadata record for the repository to find the
+                        # repository_metadata record with the changeset_revision value we want to pass back to the caller.
                         found = False
                         for repository_metadata in get_repository_metadata_by_repository_id( trans, trans.security.encode_id( repository.id ) ):
                             metadata = repository_metadata.metadata
@@ -989,6 +984,10 @@ class RepositoryController( BaseUIController, ItemRatings ):
                         if from_update_manager:
                             return no_update
                         url += changeset_revision
+                    try:
+                        shutil.rmtree( work_dir )
+                    except:
+                        pass
         url += '&latest_ctx_rev=%s' % str( latest_ctx.rev() )
         return trans.response.send_redirect( url )
     @web.expose
@@ -1241,7 +1240,7 @@ class RepositoryController( BaseUIController, ItemRatings ):
         current_working_dir = os.getcwd()
         # Update repository files for browsing.
         update_repository( repo )
-        is_malicious = change_set_is_malicious( trans, id, repository.tip )
+        is_malicious = changeset_is_malicious( trans, id, repository.tip )
         return trans.fill_template( '/webapps/community/repository/browse_repository.mako',
                                     repo=repo,
                                     repository=repository,
@@ -1322,9 +1321,7 @@ class RepositoryController( BaseUIController, ItemRatings ):
                     try:
                         commands.remove( repo.ui, repo, selected_file, force=True )
                     except Exception, e:
-                        # I never have a problem with commands.remove on a Mac, but in the test/production
-                        # tool shed environment, it throws an exception whenever I delete all files from a
-                        # repository.  If this happens, we'll try the following.
+                        log.debug( "Error removing files using the mercurial API, so trying a different approach, the error was: %s" % str( e ))
                         relative_selected_file = selected_file.split( 'repo_%d' % repository.id )[1].lstrip( '/' )
                         repo.dirstate.remove( relative_selected_file )
                         repo.dirstate.write()
@@ -1346,42 +1343,24 @@ class RepositoryController( BaseUIController, ItemRatings ):
                 # Commit the change set.
                 if not commit_message:
                     commit_message = 'Deleted selected files'
-                try:
-                    commands.commit( repo.ui, repo, repo_dir, user=trans.user.username, message=commit_message )
-                except Exception, e:
-                    # I never have a problem with commands.commit on a Mac, but in the test/production
-                    # tool shed environment, it occasionally throws a "TypeError: array item must be char"
-                    # exception.  If this happens, we'll try the following.
-                    repo.dirstate.write()
-                    repo.commit( user=trans.user.username, text=commit_message )
+                commands.commit( repo.ui, repo, repo_dir, user=trans.user.username, message=commit_message )
                 handle_email_alerts( trans, repository )
                 # Update the repository files for browsing.
                 update_repository( repo )
                 # Get the new repository tip.
                 repo = hg.repository( get_configured_ui(), repo_dir )
-                if tip != repository.tip:
-                    message = "The selected files were deleted from the repository."
+                if tip == repository.tip:
+                    message += 'No changes to repository.  '
+                    kwd[ 'message' ] = message
+                    
                 else:
-                    message = 'No changes to repository.'
-                # Set metadata on the repository tip.
-                error_message, status = set_repository_metadata( trans, id, repository.tip, **kwd )
-                if error_message:
-                    # If there is an error, display it.
-                    message = '%s<br/>%s' % ( message, error_message )
-                    return trans.response.send_redirect( web.url_for( controller='repository',
-                                                                      action='manage_repository',
-                                                                      id=id,
-                                                                      message=message,
-                                                                      status=status ) )
-                else:
-                    # If no error occurred in setting metadata on the repository tip, reset metadata on all
-                    # changeset revisions for the repository.  This will result in a more standardized set of
-                    # valid repository revisions that can be installed.
-                    reset_all_repository_metadata( trans, id, **kwd )
+                    message += 'The selected files were deleted from the repository.  '
+                    kwd[ 'message' ] = message
+                    set_repository_metadata_due_to_new_tip( trans, id, repository, **kwd )
             else:
                 message = "Select at least 1 file to delete from the repository before clicking <b>Delete selected files</b>."
                 status = "error"
-        is_malicious = change_set_is_malicious( trans, id, repository.tip )
+        is_malicious = changeset_is_malicious( trans, id, repository.tip )
         return trans.fill_template( '/webapps/community/repository/browse_repository.mako',
                                     repo=repo,
                                     repository=repository,
@@ -1436,7 +1415,7 @@ class RepositoryController( BaseUIController, ItemRatings ):
         else:
             repository_metadata_id = None
             metadata = None
-        is_malicious = change_set_is_malicious( trans, id, repository.tip )
+        is_malicious = changeset_is_malicious( trans, id, repository.tip )
         if is_malicious:
             if trans.app.security_agent.can_push( trans.user, repository ):
                 message += malicious_error_can_push
@@ -1640,7 +1619,7 @@ class RepositoryController( BaseUIController, ItemRatings ):
                             'has_metadata' : has_metadata }
             # Make sure we'll view latest changeset first.
             changesets.insert( 0, change_dict )
-        is_malicious = change_set_is_malicious( trans, id, repository.tip )
+        is_malicious = changeset_is_malicious( trans, id, repository.tip )
         return trans.fill_template( '/webapps/community/repository/view_changelog.mako', 
                                     repository=repository,
                                     changesets=changesets,
@@ -1669,7 +1648,7 @@ class RepositoryController( BaseUIController, ItemRatings ):
         diffs = []
         for diff in patch.diff( repo, node1=ctx_parent.node(), node2=ctx.node() ):
             diffs.append( to_html_escaped( diff ) )
-        is_malicious = change_set_is_malicious( trans, id, repository.tip )
+        is_malicious = changeset_is_malicious( trans, id, repository.tip )
         return trans.fill_template( '/webapps/community/repository/view_changeset.mako', 
                                     repository=repository,
                                     ctx=ctx,
@@ -1712,7 +1691,7 @@ class RepositoryController( BaseUIController, ItemRatings ):
         avg_rating, num_ratings = self.get_ave_item_rating_data( trans.sa_session, repository, webapp_model=trans.model )
         display_reviews = util.string_as_bool( params.get( 'display_reviews', False ) )
         rra = self.get_user_item_rating( trans.sa_session, trans.user, repository, webapp_model=trans.model )
-        is_malicious = change_set_is_malicious( trans, id, repository.tip )
+        is_malicious = changeset_is_malicious( trans, id, repository.tip )
         return trans.fill_template( '/webapps/community/repository/rate_repository.mako', 
                                     repository=repository,
                                     avg_rating=avg_rating,
@@ -1845,9 +1824,12 @@ class RepositoryController( BaseUIController, ItemRatings ):
                                                           status=status ) )
     @web.expose
     def reset_all_metadata( self, trans, id, **kwd ):
-        reset_all_repository_metadata( trans, id, **kwd )
-        message = "All repository metadata has been reset."
-        status = 'done'
+        error_message, status = reset_all_metadata_on_repository( trans, id, **kwd )
+        if error_message:
+            message = error_message
+        else:
+            message = "All repository metadata has been reset."
+            status = 'done'
         return trans.response.send_redirect( web.url_for( controller='repository',
                                                           action='manage_repository',
                                                           id=id,
@@ -1862,7 +1844,7 @@ class RepositoryController( BaseUIController, ItemRatings ):
         repository = get_repository( trans, repository_id )
         tool, message = load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_config )
         tool_state = self.__new_state( trans )
-        is_malicious = change_set_is_malicious( trans, repository_id, repository.tip )
+        is_malicious = changeset_is_malicious( trans, repository_id, repository.tip )
         try:
             return trans.fill_template( "/webapps/community/repository/tool_form.mako",
                                         repository=repository,
@@ -1900,43 +1882,34 @@ class RepositoryController( BaseUIController, ItemRatings ):
         repo = hg.repository( get_configured_ui(), repo_dir )
         ctx = get_changectx_for_changeset( repo, changeset_revision )
         invalid_message = ''
-        if changeset_revision == repository.tip:
-            for root, dirs, files in os.walk( repo_dir ):
-                found = False
-                for name in files:
-                    if name == tool_config:
-                        tool_config_path = os.path.join( root, name )
-                        found = True
-                        break
-                if found:
-                    break
-            metadata_dict, invalid_files = generate_metadata_for_repository_tip( trans, repository_id, ctx, changeset_revision, repo, repo_dir )
-        else:
-            for filename in ctx:
-                if filename == tool_config:
-                    fctx = ctx[ filename ]
-                    # Write the contents of datatypes_config.xml to a temporary file.
-                    fh = tempfile.NamedTemporaryFile( 'w' )
-                    tool_config_path = fh.name
-                    fh.close()
-                    fh = open( tool_config_path, 'w' )
-                    fh.write( fctx.data() )
-                    fh.close()
-                    break
-            metadata_dict, invalid_files = generate_metadata_for_changeset_revision( trans, repository_id, ctx, changeset_revision, repo_dir )
+        work_dir = make_tmp_directory()
+        for filename in ctx:
+            ctx_file_name = strip_path( filename )
+            if ctx_file_name == tool_config:
+                tool_config_path = get_named_tmpfile_from_ctx( ctx, filename, dir=work_dir )
+                break
+        metadata_dict, invalid_files = generate_metadata_for_changeset_revision( trans,
+                                                                                 repo,
+                                                                                 repository_id,
+                                                                                 ctx,
+                                                                                 changeset_revision,
+                                                                                 repo_dir,
+                                                                                 updating_tip=changeset_revision==repository.tip )
         for invalid_file_tup in invalid_files:
             invalid_tool_config, invalid_msg = invalid_file_tup
-            if tool_config == invalid_tool_config:
+            invalid_tool_config_name = strip_path( invalid_tool_config )
+            if tool_config == invalid_tool_config_name:
                 invalid_message = invalid_msg
                 break 
-        tool, message = load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_config_path )
+        tool, error_message = load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_config )
+        if error_message:
+            message += error_message
         tool_state = self.__new_state( trans )
-        is_malicious = change_set_is_malicious( trans, repository_id, repository.tip )
-        if changeset_revision != repository.tip:
-            try:
-                os.unlink( tool_config_path )
-            except:
-                pass
+        is_malicious = changeset_is_malicious( trans, repository_id, repository.tip )
+        try:
+            shutil.rmtree( work_dir )
+        except:
+            pass
         try:
             if invalid_message:
                 message = invalid_message
@@ -1997,7 +1970,7 @@ class RepositoryController( BaseUIController, ItemRatings ):
                     except:
                         tool = None
                     break
-        is_malicious = change_set_is_malicious( trans, repository_id, repository.tip )
+        is_malicious = changeset_is_malicious( trans, repository_id, repository.tip )
         changeset_revision_select_field = build_changeset_revision_select_field( trans,
                                                                                  repository,
                                                                                  selected_value=changeset_revision,

lib/galaxy/webapps/community/controllers/upload.py

@@ -126,16 +126,14 @@ class UploadController( BaseUIController ):
                     full_path = full_path.encode( 'ascii', 'replace' )
                     commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message )
                     if full_path.endswith( 'tool_data_table_conf.xml.sample' ):
-                        # Handle the special case where a tool_data_table_conf.xml.sample
-                        # file is being uploaded by parsing the file and adding new entries
-                        # to the in-memory trans.app.tool_data_tables dictionary as well as
-                        # appending them to the shed's tool_data_table_conf.xml file on disk.
+                        # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded by parsing the file and adding new entries
+                        # to the in-memory trans.app.tool_data_tables dictionary.
                         error, error_message = handle_sample_tool_data_table_conf_file( trans.app, full_path )
                         if error:
                             message = '%s<br/>%s' % ( message, error_message )
-                    if full_path.endswith( '.loc.sample' ):
-                        # Handle the special case where a xxx.loc.sample file is being uploaded by copying it to ~/tool-data/xxx.loc.
-                        copy_sample_file( trans.app, full_path )
+                    #if full_path.endswith( '.loc.sample' ):
+                    #    # Handle the special case where a xxx.loc.sample file is being uploaded by copying it to ~/tool-data/xxx.loc.
+                    #    copy_sample_file( trans.app, full_path )
                     # See if the content of the change set was valid.
                     admin_only = len( repository.downloadable_revisions ) != 1
                     handle_email_alerts( trans, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only )
@@ -143,37 +141,24 @@ class UploadController( BaseUIController ):
                     # Update the repository files for browsing.
                     update_repository( repo )
                     # Get the new repository tip.
-                    if tip != repository.tip:
+                    if tip == repository.tip:
+                        message = 'No changes to repository.  '
+                    else:
                         if ( isgzip or isbz2 ) and uncompress_file:
                             uncompress_str = ' uncompressed and '
                         else:
                             uncompress_str = ' '
-                        message = "The file '%s' has been successfully%suploaded to the repository." % ( uploaded_file_filename, uncompress_str )
+                        message = "The file '%s' has been successfully%suploaded to the repository.  " % ( uploaded_file_filename, uncompress_str )
                         if istar and ( undesirable_dirs_removed or undesirable_files_removed ):
                             items_removed = undesirable_dirs_removed + undesirable_files_removed
-                            message += "  %d undesirable items (.hg .svn .git directories, .DS_Store, hgrc files, etc) were removed from the archive." % items_removed
+                            message += "  %d undesirable items (.hg .svn .git directories, .DS_Store, hgrc files, etc) were removed from the archive.  " % items_removed
                         if istar and remove_repo_files_not_in_tar and files_to_remove:
                             if upload_point is not None:
-                                message += "  %d files were removed from the repository relative to the selected upload point '%s'." % ( len( files_to_remove ), upload_point )
+                                message += "  %d files were removed from the repository relative to the selected upload point '%s'.  " % ( len( files_to_remove ), upload_point )
                             else:
-                                message += "  %d files were removed from the repository root." % len( files_to_remove )
-                    else:
-                        message = 'No changes to repository.'      
-                    # Set metadata on the repository tip.
-                    error_message, status = set_repository_metadata( trans, repository_id, repository.tip, content_alert_str=content_alert_str, **kwd )
-                    if error_message:
-                        # If there is an error, display it.
-                        message = '%s<br/>%s' % ( message, error_message )
-                        return trans.response.send_redirect( web.url_for( controller='repository',
-                                                                          action='manage_repository',
-                                                                          id=repository_id,
-                                                                          message=message,
-                                                                          status=status ) )
-                    else:
-                        # If no error occurred in setting metadata on the repository tip, reset metadata on all
-                        # changeset revisions for the repository.  This will result in a more standardized set of
-                        # valid repository revisions that can be installed.
-                        reset_all_repository_metadata( trans, repository_id, **kwd )
+                                message += "  %d files were removed from the repository root.  " % len( files_to_remove )
+                        kwd[ 'message' ] = message
+                        set_repository_metadata_due_to_new_tip( trans, repository_id, repository, content_alert_str=content_alert_str, **kwd )
                     trans.response.send_redirect( web.url_for( controller='repository',
                                                                action='browse_repository',
                                                                id=repository_id,
@@ -278,16 +263,14 @@ class UploadController( BaseUIController ):
                     content_alert_str += self.__check_file_content( filename_in_archive )
                 commands.add( repo.ui, repo, filename_in_archive )
                 if filename_in_archive.endswith( 'tool_data_table_conf.xml.sample' ):
-                    # Handle the special case where a tool_data_table_conf.xml.sample
-                    # file is being uploaded by parsing the file and adding new entries
-                    # to the in-memory trans.app.tool_data_tables dictionary as well as
-                    # appending them to the shed's tool_data_table_conf.xml file on disk.
+                    # Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded by parsing the file and adding new entries
+                    # to the in-memory trans.app.tool_data_tables dictionary.
                     error, message = handle_sample_tool_data_table_conf_file( trans.app, filename_in_archive )
                     if error:
                         return False, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed
-                if filename_in_archive.endswith( '.loc.sample' ):
-                    # Handle the special case where a xxx.loc.sample file is being uploaded by copying it to ~/tool-data/xxx.loc.
-                    copy_sample_file( trans.app, filename_in_archive )
+                #if filename_in_archive.endswith( '.loc.sample' ):
+                #    # Handle the special case where a xxx.loc.sample file is being uploaded by copying it to ~/tool-data/xxx.loc.
+                #    copy_sample_file( trans.app, filename_in_archive )
             commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message )
             # See if the content of the change set was valid.
             admin_only = len( repository.downloadable_revisions ) != 1

templates/admin/tool_shed_repository/browse_repository.mako

@@ -2,8 +2,6 @@
 <%namespace file="/message.mako" import="render_msg" />
 <%namespace file="/admin/tool_shed_repository/common.mako" import="*" />
 
-<% from galaxy.web.base.controller import tool_shed_encode, tool_shed_decode %>
-
 <br/><br/>
 <ul class="manage-table-actions">
     <li><a class="action-button" id="repository-${repository.id}-popup" class="menubutton">Repository Actions</a></li>

templates/admin/tool_shed_repository/deactivate_or_uninstall_repository.mako

@@ -55,6 +55,11 @@
                         * The repository's tools will not be loaded into the tool panel.
                     </div>
                 %endif
+                %if repository.includes_tool_dependencies:
+                    <div class="toolParamHelp" style="clear: both;">
+                        * The repository's installed tool dependencies will remain on disk.
+                    </div>
+                %endif
                 %if repository.includes_datatypes:
                     <div class="toolParamHelp" style="clear: both;">
                         * The repository's datatypes, datatype converters and display applications will be eliminated from the datatypes registry.
@@ -73,6 +78,14 @@
                         * The repository's tool tag sets will be removed from the tool config file in which they are defined.
                     </div>
                 %endif
+                %if repository.includes_tool_dependencies:
+                    <div class="toolParamHelp" style="clear: both;">
+                        * The repository's installed tool dependencies will be removed from disk.
+                    </div>
+                    <div class="toolParamHelp" style="clear: both;">
+                        * Each associated tool dependency record's uninstalled column in the tool_dependency database table will be set to True.
+                    </div>
+                %endif
                 %if repository.includes_datatypes:
                     <div class="toolParamHelp" style="clear: both;">
                         * The repository's datatypes, datatype converters and display applications will be eliminated from the datatypes registry.

templates/admin/tool_shed_repository/install_missing_tool_dependencies.mako

@@ -0,0 +1,90 @@
+<%inherit file="/base.mako"/>
+<%namespace file="/message.mako" import="render_msg" />
+
+<% import os %>
+
+%if message:
+    ${render_msg( message, status )}
+%endif
+
+<div class="warningmessage">
+    <p>
+        Galaxy will attempt to install the missing tool dependencies listed below.  Each of these dependencies may require their own build
+        requirements (e.g., CMake, g++, etc).  Galaxy will not attempt to install these build requirements, so if any are missing from your
+        environment tool dependency installation may partially fail.  If this happens, you can install the missing build requirements and
+        have Galaxy attempt to install the tool dependencies again.
+    </p> 
+</div>
+<br/>
+<div class="warningmessage">
+    <p>
+        Installation may take a while.  <b>Always wait until a message is displayed in your browser after clicking the <b>Go</b> button below.</b>
+        If you get bored, watching your Galaxy server's paster log will help pass the time.
+    </p>
+    <p>
+        Information about the tool dependency installation process will be saved in various files named with a ".log" extension in the directory: 
+        ${trans.app.config.tool_dependency_dir}/<i>package name</i>/<i>package version</i>/${repository.owner}/${repository.name}/${repository.changeset_revision}
+    </p>
+</div>
+<br/>
+
+<div class="toolForm">
+    <div class="toolFormBody">
+        <form name="install_missing_tool_dependencies" id="install_missing_tool_dependencies" action="${h.url_for( controller='admin_toolshed', action='install_missing_tool_dependencies', id=trans.security.encode_id( repository.id ), tool_panel_section=tool_panel_section, new_tool_panel_section=new_tool_panel_section, reinstalling=reinstalling )}" method="post" >
+            <div style="clear: both"></div>
+            <div class="form-row">
+                <label>Install tool dependencies?</label>
+                ${install_tool_dependencies_check_box.get_html()}
+                <div class="toolParamHelp" style="clear: both;">
+                    Un-check to skip installation of these tool dependencies.
+                </div>
+                ## Fake the no_changes_check_box value.
+                %if no_changes_checked:
+                    <input type="hidden" id="no_changes" name="no_changes" value="true" checked="checked"><input type="hidden" name="no_changes" value="true">
+                %else:
+                    <input type="hidden" name="no_changes" value="true">
+                %endif
+            </div>
+            <div style="clear: both"></div>
+            <div class="form-row">
+                <table class="grid">
+                    <tr><td colspan="4" bgcolor="#D8D8D8"><b>Tool dependencies</b></td></tr>
+                    <tr>
+                        <th>Name</th>
+                        <th>Version</th>
+                        <th>Type</th>
+                        <th>Install directory</th>
+                    </tr>
+                    %for dependency_key, requirements_dict in tool_dependencies.items():
+                        <%
+                            name = requirements_dict[ 'name' ]
+                            version = requirements_dict[ 'version' ]
+                            type = requirements_dict[ 'type' ]
+                            install_dir = os.path.join( trans.app.config.tool_dependency_dir,
+                                                        name,
+                                                        version,
+                                                        repository.owner,
+                                                        repository.name,
+                                                        repository.changeset_revision )
+                            readme_text = requirements_dict.get( 'readme', None )
+                        %>
+                            <tr>
+                                <td>${name}</td>
+                                <td>${version}</td>
+                                <td>${type}</td>
+                                <td>${install_dir}</td>
+                            </tr>
+                            %if readme_text:
+                                <tr><td colspan="4" bgcolor="#FFFFCC">${name} ${version} requirements and installation information</td></tr>
+                                <tr><td colspan="4"><pre>${readme_text}</pre></td></tr>
+                            %endif
+                    %endfor
+                </table>
+                <div style="clear: both"></div>
+            </div>
+            <div class="form-row">
+                <input type="submit" name="install_missing_tool_dependencies_button" value="Go"/>
+            </div>
+        </form>
+    </div>
+</div>

templates/admin/tool_shed_repository/install_tool_dependencies.mako

@@ -0,0 +1,84 @@
+<%inherit file="/base.mako"/>
+<%namespace file="/message.mako" import="render_msg" />
+
+<% import os %>
+
+%if message:
+    ${render_msg( message, status )}
+%endif
+
+<div class="warningmessage">
+    <p>
+        The tool dependencies listed below can be automatically installed with the repository.  Installing them provides significant
+        benefits and Galaxy includes various features to manage them.
+    </p>
+    <p>
+        Each of these dependencies may require their own build requirements (e.g., CMake, g++, etc).  Galaxy will not attempt to install
+        these build requirements, so if any are missing from your environment tool dependency installation may partially fail.  The
+        repository and all of it's contents will be installed in any case.
+    </p>
+    <p>
+        If tool dependency installation fails in any way, you can install the missing build requirements and have Galaxy attempt to install
+        the tool dependencies again using the <b>Install tool dependencies</b> pop-up menu option on the <b>Manage repository</b> page.
+    </p> 
+</div>
+
+<div class="toolForm">
+    <div class="toolFormBody">
+        <form name="install_tool_dependenceies" id="install_tool_dependenceies" action="${h.url_for( controller='admin_toolshed', action='install_repository', tool_shed_url=tool_shed_url, repo_info_dict=repo_info_dict, includes_tools=includes_tools )}" method="post" >
+            <div style="clear: both"></div>
+            <div class="form-row">
+                <label>Install tool dependencies?</label>
+                ${install_tool_dependencies_check_box.get_html()}
+                <div class="toolParamHelp" style="clear: both;">
+                    Un-check to skip automatic installation of these tool dependencies.
+                </div>
+            </div>
+            <div style="clear: both"></div>
+            <div class="form-row">
+                <table class="grid">
+                    <tr><td colspan="4" bgcolor="#D8D8D8"><b>Tool dependencies</b></td></tr>
+                    <tr>
+                        <th>Name</th>
+                        <th>Version</th>
+                        <th>Type</th>
+                        <th>Install directory</th>
+                    </tr>
+                    %for repository_name, repo_info_tuple in dict_with_tool_dependencies.items():
+                        <%
+                            description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, tool_dependencies = repo_info_tuple
+                        %>
+                        %for dependency_key, requirements_dict in tool_dependencies.items():
+                            <%
+                                name = requirements_dict[ 'name' ]
+                                version = requirements_dict[ 'version' ]
+                                type = requirements_dict[ 'type' ]
+                                install_dir = os.path.join( trans.app.config.tool_dependency_dir,
+                                                            name,
+                                                            version,
+                                                            repository_owner,
+                                                            repository_name,
+                                                            changeset_revision )
+                                readme_text = requirements_dict.get( 'readme', None )
+                            %>
+                                <tr>
+                                    <td>${name}</td>
+                                    <td>${version}</td>
+                                    <td>${type}</td>
+                                    <td>${install_dir}</td>
+                                </tr>
+                                %if readme_text:
+                                    <tr><td colspan="4" bgcolor="#FFFFCC">${name} ${version} requirements and installation information</td></tr>
+                                    <tr><td colspan="4"><pre>${readme_text}</pre></td></tr>
+                                %endif
+                        %endfor
+                    %endfor
+                </table>
+                <div style="clear: both"></div>
+            </div>
+            <div class="form-row">
+                <input type="submit" name="install_tool_dependenceies_button" value="Continue"/>
+            </div>
+        </form>
+    </div>
+</div>

templates/admin/tool_shed_repository/manage_repository.mako

@@ -11,6 +11,9 @@
         %if repository.includes_tools:
             <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='set_tool_versions', id=trans.security.encode_id( repository.id ) )}">Set tool versions</a>
         %endif
+        %if tool_dependencies_missing:
+            <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='install_missing_tool_dependencies', id=trans.security.encode_id( repository.id ) )}">Install tool dependencies</a>
+        %endif
         <a class="action-button" href="${h.url_for( controller='admin_toolshed', action='deactivate_or_uninstall_repository', id=trans.security.encode_id( repository.id ) )}">Deactivate or Uninstall</a>
     </div>
 </ul>

templates/admin/tool_shed_repository/reselect_tool_panel_section.mako

@@ -8,7 +8,11 @@
 <div class="toolForm">
     <div class="toolFormTitle">Choose the tool panel section to contain the installed tools (optional)</div>
     <div class="toolFormBody">
-        <form name="reselect_tool_panel_section" id="reselect_tool_panel_section" action="${h.url_for( controller='admin_toolshed', action='reinstall_repository', id=trans.security.encode_id( repository.id ) )}" method="post" >
+        %if repository.includes_tool_dependencies:
+            <form name="reselect_tool_panel_section" id="reselect_tool_panel_section" action="${h.url_for( controller='admin_toolshed', action='install_missing_tool_dependencies', id=trans.security.encode_id( repository.id ), reinstalling=True )}" method="post" >
+        %else:
+            <form name="reselect_tool_panel_section" id="reselect_tool_panel_section" action="${h.url_for( controller='admin_toolshed', action='reinstall_repository', id=trans.security.encode_id( repository.id ) )}" method="post" >
+        %endif
             <div class="form-row">
                 ${no_changes_check_box.get_html()}
                 <label style="display: inline;">No changes</label>

templates/admin/tool_shed_repository/select_tool_panel_section.mako

@@ -6,26 +6,38 @@
 %endif
 
 <div class="warningmessage">
-    The core Galaxy development team does not maintain the contents of many Galaxy tool shed repositories.  Some repository tools
-    may include code that produces malicious behavior, so be aware of what you are installing.  
-    <p/>
-    If you discover a repository that causes problems after installation, contact <a href="http://wiki.g2.bx.psu.edu/Support" target="_blank">Galaxy support</a>,
-    sending all necessary information, and appropriate action will be taken.
-    <p/>
-    <a href="http://wiki.g2.bx.psu.edu/Tool%20Shed#Contacting_the_owner_of_a_repository" target="_blank">Contact the repository owner</a> for general questions 
-    or concerns.
+    <p>
+        The core Galaxy development team does not maintain the contents of many Galaxy tool shed repositories.  Some repository tools
+        may include code that produces malicious behavior, so be aware of what you are installing.  
+    </p>
+    <p>
+        If you discover a repository that causes problems after installation, contact <a href="http://wiki.g2.bx.psu.edu/Support" target="_blank">Galaxy support</a>,
+        sending all necessary information, and appropriate action will be taken.
+    </p>
+    <p>
+        <a href="http://wiki.g2.bx.psu.edu/Tool%20Shed#Contacting_the_owner_of_a_repository" target="_blank">Contact the repository owner</a> for 
+        general questions or concerns.
+    </p>
 </div>
 <br/>
 <div class="warningmessage">
-    Installation may take a while, depending upon the size of the repository contents.  Wait until a message is displayed in your 
-    browser after clicking the <b>Install</b> button below.
+    <p>
+        Installation may take a while with large repositories or if you elect to install tool dependencies.  <b>Always wait until a message is
+        displayed in your browser after clicking the <b>Install</b> button below.</b>  If you get bored, watching your Galaxy server's paster log 
+        will help pass the time.
+    </p>
+    <p>
+        If installing tool dependencies, information about installation process will be saved in various files named with a ".log" 
+        extension in the directory: 
+        ${trans.app.config.tool_dependency_dir}/<i>package name</i>/<i>package version</i>/<i>repository owner</i>/<i>repository name</i>/<i>repository changeset revision</i>
+    </p>
 </div>
 <br/>
 
 <div class="toolForm">
     <div class="toolFormTitle">Choose the tool panel section to contain the installed tools (optional)</div>
     <div class="toolFormBody">
-        <form name="select_tool_panel_section" id="select_tool_panel_section" action="${h.url_for( controller='admin_toolshed', action='install_repository', tool_shed_url=tool_shed_url, repo_info_dict=repo_info_dict, includes_tools=includes_tools )}" method="post" >
+        <form name="select_tool_panel_section" id="select_tool_panel_section" action="${h.url_for( controller='admin_toolshed', action='install_repository', tool_shed_url=tool_shed_url, repo_info_dict=repo_info_dict, includes_tools=includes_tools, install_tool_dependencies=install_tool_dependencies )}" method="post" >
             %if shed_tool_conf_select_field:
                 <div class="form-row">
                     <label>Shed tool configuration file:</label>

templates/webapps/community/admin/index.mako

@@ -55,7 +55,7 @@
                             <a target="galaxy_main" href="${h.url_for( controller='admin', action='browse_repositories', webapp='community' )}">Browse all repositories</a>
                         </div>
                         <div class="toolTitle">
-                            <a target="galaxy_main" href="${h.url_for( controller='admin', action='reset_all_repository_metadata', webapp='community' )}">Reset all metadata</a>
+                            <a target="galaxy_main" href="${h.url_for( controller='admin', action='reset_metadata_on_all_repositories', webapp='community' )}">Reset all metadata</a>
                         </div>
                         <div class="toolTitle">
                             <a target="galaxy_main" href="${h.url_for( controller='admin', action='browse_repository_metadata', webapp='community' )}">Browse metadata</a>

templates/webapps/community/admin/reset_metadata_on_all_repositories.mako

@@ -7,12 +7,12 @@
 
 <div class="toolForm">
     <div class="toolFormTitle">Reset metadata on each change set of the repositories in this tool shed</div>
-        <form name="reset_all_repository_metadata" id="reset_all_repository_metadata" action="${h.url_for( controller='admin', action='reset_all_repository_metadata' )}" method="post" >
+        <form name="reset_metadata_on_all_repositories" id="reset_metadata_on_all_repositories" action="${h.url_for( controller='admin', action='reset_metadata_on_all_repositories' )}" method="post" >
             <div class="form-row">
                 Click the button below to reset metadata on each change set of the repositories in this tool shed.
             </div>
             <div class="form-row">
-                <input type="submit" name="reset_all_repository_metadata_button" value="Reset all repository metadata"/>
+                <input type="submit" name="reset_metadata_on_all_repositories_button" value="Reset metadata on all repositories"/>
             </div>
         </form>
     </div>

templates/webapps/community/repository/common.mako

@@ -91,6 +91,39 @@
             <div class="toolFormTitle">Preview tools and inspect metadata by tool version</div>
             <div class="toolFormBody">
                 %if metadata:
+                    %if 'tool_dependencies' in metadata:
+                        <div class="form-row">
+                            <table width="100%">
+                                <tr bgcolor="#D8D8D8" width="100%">
+                                    <td><b>The following tool dependencies can optionally be automatically installed</i></td>
+                                </tr>
+                            </table>
+                        </div>
+                        <div style="clear: both"></div>
+                        <div class="form-row">
+                            <% tool_dependencies = metadata[ 'tool_dependencies' ] %>
+                            <table class="grid">
+                                <tr>
+                                    <td><b>name</b></td>
+                                    <td><b>type</b></td>
+                                    <td><b>version</b></td>
+                                </tr>
+                                %for dependency_key, requirements_dict in tool_dependencies.items():
+                                    <%
+                                        name = requirements_dict[ 'name' ]
+                                        type = requirements_dict[ 'type' ]
+                                        version = requirements_dict[ 'version' ]
+                                    %>
+                                    <tr>
+                                        <td>${name}</td>
+                                        <td>${type}</td>
+                                        <td>${version}</td>
+                                    </tr>
+                                %endfor
+                            </table>
+                        </div>
+                        <div style="clear: both"></div>
+                    %endif
                     %if 'tools' in metadata:
                         <div class="form-row">
                             <table width="100%">
@@ -256,18 +289,6 @@
                         <div style="clear: both"></div>
                     %endif
                 %endif
-                %if can_set_metadata:
-                    <form name="set_metadata" action="${h.url_for( controller='repository', action='set_metadata', id=trans.security.encode_id( repository.id ), ctx_str=changeset_revision )}" method="post">
-                        <div class="form-row">
-                            <div style="float: left; width: 250px; margin-right: 10px;">
-                                <input type="submit" name="set_metadata_button" value="Reset metadata"/>
-                            </div>
-                            <div class="toolParamHelp" style="clear: both;">
-                                Inspect the repository and reset the above attributes for the repository tip.
-                            </div>
-                        </div>
-                    </form>
-                %endif
             </div>
         </div>
     %endif

templates/webapps/community/repository/view_tool_metadata.mako

@@ -165,6 +165,37 @@
                     <div style="clear: both"></div>
                 </div>
             %endif
+            <%
+                if 'requirements' in metadata:
+                    requirements = metadata[ 'requirements' ]
+                else:
+                    requirements = None
+            %>
+            %if requirements:
+                <div class="form-row">
+                    <label>Requirements:</label>
+                    <table class="grid">
+                        <tr>
+                            <td><b>name</b></td>
+                            <td><b>version</b></td>
+                            <td><b>type</b></td>
+                        </tr>
+                        %for requirement_dict in requirements:
+                            <%
+                                requirement_name = requirement_dict[ 'name' ] or 'not provided'
+                                requirement_version = requirement_dict[ 'version' ] or 'not provided'
+                                requirement_type = requirement_dict[ 'type' ] or 'not provided'
+                            %>
+                            <tr>
+                                <td>${requirement_name}</td>
+                                <td>${requirement_version}</td>
+                                <td>${requirement_type}</td>
+                            </tr>
+                        %endfor
+                    </table>
+                    <div style="clear: both"></div>
+                </div>
+            %endif
             %if 'version_string_cmd' in metadata:
                 <div class="form-row">
                     <label>Version command string:</label>
@@ -199,25 +230,6 @@
                     <div style="clear: both"></div>
                 </div>
             %endif
-            <%
-                if 'requirements' in metadata:
-                    requirements = metadata[ 'requirements' ]
-                else:
-                    requirements = None
-            %>
-            %if requirements:
-                <%
-                    requirements_str = ''
-                    for requirement_dict in metadata[ 'requirements' ]:
-                        requirements_str += '%s (%s), ' % ( requirement_dict[ 'name' ], requirement_dict[ 'type' ] )
-                    requirements_str = requirements_str.rstrip( ', ' )
-                %>
-                <div class="form-row">
-                    <label>Requirements:</label>
-                    ${requirements_str}
-                    <div style="clear: both"></div>
-                </div>
-            %endif
             <%
                 if 'tests' in metadata:
                     tests = metadata[ 'tests' ]

templates/webapps/community/repository/view_workflow.mako

@@ -16,7 +16,6 @@
     can_upload = can_push
     can_download = in_tool_shed and not is_new and ( not is_malicious or can_push )
     can_browse_contents = in_tool_shed and not is_new
-    can_set_metadata = in_tool_shed and not is_new
     can_rate = in_tool_shed and not is_new and trans.user and repository.user != trans.user
     can_view_change_log = in_tool_shed and not is_new
     if can_push: