Merge.

job_conf.xml.sample_advanced

@@ -6,7 +6,19 @@
           -->
         <plugin id="local" type="runner" load="galaxy.jobs.runners.local:LocalJobRunner"/>
         <plugin id="pbs" type="runner" load="galaxy.jobs.runners.pbs:PBSJobRunner" workers="2"/>
-        <plugin id="drmaa" type="runner" load="galaxy.jobs.runners.drmaa:DRMAAJobRunner"/>
+        <plugin id="drmaa" type="runner" load="galaxy.jobs.runners.drmaa:DRMAAJobRunner">
+            <!-- Different DRMs handle successfully completed jobs differently,
+                 these options can be changed to handle such differences and
+                 are explained in detail on the Galaxy wiki. Defaults are shown -->
+            <param id="invalidjobexception_state">ok</param>
+            <param id="invalidjobexception_retries">0</param>
+            <param id="internalexception_state">ok</param>
+            <param id="internalexception_retries">0</param>
+        </plugin>
+        <plugin id="sge" type="runner" load="galaxy.jobs.runners.drmaa:DRMAAJobRunner">
+            <!-- Override the $DRMAA_LIBRARY_PATH environment variable -->
+            <param id="drmaa_library_path">/sge/lib/libdrmaa.so</param>
+        </plugin>
         <plugin id="lwr" type="runner" load="galaxy.jobs.runners.lwr:LwrJobRunner">
           <!-- More information on LWR can be found at https://lwr.readthedocs.org -->
           <!-- Uncomment following line to use libcurl to perform HTTP calls (defaults to urllib) -->
@@ -14,6 +26,7 @@
         </plugin>
         <plugin id="cli" type="runner" load="galaxy.jobs.runners.cli:ShellJobRunner" />
         <plugin id="condor" type="runner" load="galaxy.jobs.runners.condor:CondorJobRunner" />
+        <plugin id="slurm" type="runner" load="galaxy.jobs.runners.slurm:SlurmJobRunner" />
     </plugins>
     <handlers default="handlers">
         <!-- Additional job handlers - the id should match the name of a
@@ -21,6 +34,15 @@
          -->
         <handler id="handler0" tags="handlers"/>
         <handler id="handler1" tags="handlers"/>
+        <!-- Handlers will load all plugins defined in the <plugins> collection
+             above by default, but can be limited to a subset using <plugin>
+             tags. This is useful for heterogenous environments where the DRMAA
+             plugin would need to be loaded more than once with different
+             configs.
+         -->
+        <handler id="sge_handler">
+            <plugin id="sge"/>
+        </handler>
         <handler id="special_handler0" tags="special_handlers"/>
         <handler id="special_handler1" tags="special_handlers"/>
         <handler id="trackster_handler"/>

lib/galaxy/jobs/__init__.py

@@ -86,6 +86,7 @@ class JobConfiguration( object ):
         self.app = app
         self.runner_plugins = []
         self.handlers = {}
+        self.handler_runner_plugins = {}
         self.default_handler_id = None
         self.destinations = {}
         self.destination_tags = {}
@@ -138,6 +139,10 @@ class JobConfiguration( object ):
                 else:
                     log.debug("Read definition for handler '%s'" % id)
                     self.handlers[id] = (id,)
+                    for plugin in handler.findall('plugin'):
+                        if id not in self.handler_runner_plugins:
+                            self.handler_runner_plugins[id] = []
+                        self.handler_runner_plugins[id].append( plugin.get('id') )
                     if handler.get('tags', None) is not None:
                         for tag in [ x.strip() for x in handler.get('tags').split(',') ]:
                             if tag in self.handlers:
@@ -420,13 +425,19 @@ class JobConfiguration( object ):
         """
         return self.destinations.get(id_or_tag, None)
 
-    def get_job_runner_plugins(self):
+    def get_job_runner_plugins(self, handler_id):
         """Load all configured job runner plugins
 
         :returns: list of job runner plugins
         """
         rval = {}
-        for runner in self.runner_plugins:
+        if handler_id in self.handler_runner_plugins:
+            plugins_to_load = [ rp for rp in self.runner_plugins if rp['id'] in self.handler_runner_plugins[handler_id] ]
+            log.info( "Handler '%s' will load specified runner plugins: %s", handler_id, ', '.join( [ rp['id'] for rp in plugins_to_load ] ) )
+        else:
+            plugins_to_load = self.runner_plugins
+            log.info( "Handler '%s' will load all configured runner plugins", handler_id )
+        for runner in plugins_to_load:
             class_names = []
             module = None
             id = runner['id']
@@ -477,7 +488,7 @@ class JobConfiguration( object ):
                 try:
                     rval[id] = runner_class( self.app, runner[ 'workers' ], **runner.get( 'kwds', {} ) )
                 except TypeError:
-                    log.warning( "Job runner '%s:%s' has not been converted to a new-style runner" % ( module_name, class_name ) )
+                    log.exception( "Job runner '%s:%s' has not been converted to a new-style runner or encountered TypeError on load" % ( module_name, class_name ) )
                     rval[id] = runner_class( self.app )
                 log.debug( "Loaded job runner '%s:%s' as '%s'" % ( module_name, class_name, id ) )
         return rval

lib/galaxy/jobs/handler.py

@@ -565,7 +565,7 @@ class JobHandlerStopQueue( object ):
 class DefaultJobDispatcher( object ):
     def __init__( self, app ):
         self.app = app
-        self.job_runners = self.app.job_config.get_job_runner_plugins()
+        self.job_runners = self.app.job_config.get_job_runner_plugins( self.app.config.server_name )
         # Once plugins are loaded, all job destinations that were created from
         # URLs can have their URL params converted to the destination's param
         # dict by the plugin.

lib/galaxy/jobs/runners/__init__.py

@@ -22,13 +22,39 @@ log = logging.getLogger( __name__ )
 
 STOP_SIGNAL = object()
 
+
+class RunnerParams( object ):
+
+    def __init__( self, specs = None, params = None ):
+        self.specs = specs or dict()
+        self.params = params or dict()
+        for name, value in self.params.items():
+            assert name in self.specs, 'Invalid job runner parameter for this plugin: %s' % name
+            if 'map' in self.specs[ name ]:
+                try:
+                    self.params[ name ] = self.specs[ name ][ 'map' ]( value )
+                except Exception, e:
+                    raise Exception( 'Job runner parameter "%s" value "%s" could not be converted to the correct type: %s' % ( name, value, e ) )
+            if 'valid' in self.specs[ name ]:
+                assert self.specs[ name ][ 'valid' ]( value ), 'Job runner parameter %s failed validation' % name
+
+    def __getattr__( self, name ):
+        return self.params.get( name, self.specs[ name ][ 'default' ] )
+
+
 class BaseJobRunner( object ):
-    def __init__( self, app, nworkers ):
+    def __init__( self, app, nworkers, **kwargs ):
         """Start the job runner
         """
         self.app = app
         self.sa_session = app.model.context
         self.nworkers = nworkers
+        runner_param_specs = dict( recheck_missing_job_retries = dict( map = int, valid = lambda x: x >= 0, default = 0 ) )
+        if 'runner_param_specs' in kwargs:
+            runner_param_specs.update( kwargs.pop( 'runner_param_specs' ) )
+        if kwargs:
+            log.debug( 'Loading %s with params: %s', self.runner_name, kwargs )
+        self.runner_params = RunnerParams( specs = runner_param_specs, params = kwargs )
 
     def _init_worker_threads(self):
         """Start ``nworkers`` worker threads.
@@ -115,7 +141,7 @@ class BaseJobRunner( object ):
                 job_wrapper.cleanup()
             return False
         elif job_state != model.Job.states.QUEUED:
-            log.info( "(%d) Job is in state %s, skipping execution"  % ( job_id, job_state ) )
+            log.info( "(%s) Job is in state %s, skipping execution"  % ( job_id, job_state ) )
             # cleanup may not be safe in all states
             return False
 
@@ -226,6 +252,10 @@ class BaseJobRunner( object ):
         options.update(**kwds)
         return job_script(**options)
 
+    def _complete_terminal_job( self, ajs, **kwargs ):
+        if ajs.job_wrapper.get_state() != model.Job.states.DELETED:
+            self.work_queue.put( ( self.finish_job, ajs ) )
+
 
 class AsynchronousJobState( object ):
     """
@@ -287,8 +317,8 @@ class AsynchronousJobRunner( BaseJobRunner ):
     to the correct methods (queue, finish, cleanup) at appropriate times..
     """
 
-    def __init__( self, app, nworkers ):
-        super( AsynchronousJobRunner, self ).__init__( app, nworkers )
+    def __init__( self, app, nworkers, **kwargs ):
+        super( AsynchronousJobRunner, self ).__init__( app, nworkers, **kwargs )
         # 'watched' and 'queue' are both used to keep track of jobs to watch.
         # 'queue' is used to add new watched jobs, and can be called from
         # any thread (usually by the 'queue_job' method). 'watched' must only

lib/galaxy/jobs/runners/drmaa.py

@@ -16,27 +16,12 @@ from galaxy.jobs import JobDestination
 from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner
 
 eggs.require( "drmaa" )
-# We foolishly named this file the same as the name exported by the drmaa
-# library... 'import drmaa' imports itself.
-drmaa = __import__( "drmaa" )
 
 log = logging.getLogger( __name__ )
 
 __all__ = [ 'DRMAAJobRunner' ]
 
-drmaa_state = {
-    drmaa.JobState.UNDETERMINED: 'process status cannot be determined',
-    drmaa.JobState.QUEUED_ACTIVE: 'job is queued and active',
-    drmaa.JobState.SYSTEM_ON_HOLD: 'job is queued and in system hold',
-    drmaa.JobState.USER_ON_HOLD: 'job is queued and in user hold',
-    drmaa.JobState.USER_SYSTEM_ON_HOLD: 'job is queued and in user and system hold',
-    drmaa.JobState.RUNNING: 'job is running',
-    drmaa.JobState.SYSTEM_SUSPENDED: 'job is system suspended',
-    drmaa.JobState.USER_SUSPENDED: 'job is user suspended',
-    drmaa.JobState.DONE: 'job finished normally',
-    drmaa.JobState.FAILED: 'job finished, but failed',
-}
-
+drmaa = None
 
 DRMAA_jobTemplate_attributes = [ 'args', 'remoteCommand', 'outputPath', 'errorPath', 'nativeSpecification',
                                  'jobName', 'email', 'project' ]
@@ -48,8 +33,50 @@ class DRMAAJobRunner( AsynchronousJobRunner ):
     """
     runner_name = "DRMAARunner"
 
-    def __init__( self, app, nworkers ):
+    def __init__( self, app, nworkers, **kwargs ):
         """Start the job runner"""
+
+        global drmaa
+
+        runner_param_specs = dict(
+            drmaa_library_path = dict( map = str, default = os.environ.get( 'DRMAA_LIBRARY_PATH', None ) ),
+            invalidjobexception_state = dict( map = str, valid = lambda x: x in ( model.Job.states.OK, model.Job.states.ERROR ), default = model.Job.states.OK ),
+            invalidjobexception_retries = dict( map = int, valid = lambda x: int >= 0, default = 0 ),
+            internalexception_state = dict( map = str, valid = lambda x: x in ( model.Job.states.OK, model.Job.states.ERROR ), default = model.Job.states.OK ),
+            internalexception_retries = dict( map = int, valid = lambda x: int >= 0, default = 0 ) )
+
+        if 'runner_param_specs' not in kwargs:
+            kwargs[ 'runner_param_specs' ] = dict()
+        kwargs[ 'runner_param_specs' ].update( runner_param_specs )
+
+        super( DRMAAJobRunner, self ).__init__( app, nworkers, **kwargs )
+
+        # This allows multiple drmaa runners (although only one per handler) in the same job config file
+        if 'drmaa_library_path' in kwargs:
+            log.info( 'Overriding DRMAA_LIBRARY_PATH due to runner plugin parameter: %s', self.runner_params.drmaa_library_path )
+            os.environ['DRMAA_LIBRARY_PATH'] = self.runner_params.drmaa_library_path
+
+        # We foolishly named this file the same as the name exported by the drmaa
+        # library... 'import drmaa' imports itself.
+        drmaa = __import__( "drmaa" )
+
+        # Subclasses may need access to state constants
+        self.drmaa_job_states = drmaa.JobState
+
+        # Descriptive state strings pulled from the drmaa lib itself
+        self.drmaa_job_state_strings = {
+            drmaa.JobState.UNDETERMINED: 'process status cannot be determined',
+            drmaa.JobState.QUEUED_ACTIVE: 'job is queued and active',
+            drmaa.JobState.SYSTEM_ON_HOLD: 'job is queued and in system hold',
+            drmaa.JobState.USER_ON_HOLD: 'job is queued and in user hold',
+            drmaa.JobState.USER_SYSTEM_ON_HOLD: 'job is queued and in user and system hold',
+            drmaa.JobState.RUNNING: 'job is running',
+            drmaa.JobState.SYSTEM_SUSPENDED: 'job is system suspended',
+            drmaa.JobState.USER_SUSPENDED: 'job is user suspended',
+            drmaa.JobState.DONE: 'job finished normally',
+            drmaa.JobState.FAILED: 'job finished, but failed',
+        }
+
         self.ds = drmaa.Session()
         self.ds.initialize()
 
@@ -58,7 +85,6 @@ class DRMAAJobRunner( AsynchronousJobRunner ):
         self.external_killJob_script = app.config.drmaa_external_killjob_script
         self.userid = None
 
-        super( DRMAAJobRunner, self ).__init__( app, nworkers )
         self._init_monitor_thread()
         self._init_worker_threads()
 
@@ -137,8 +163,10 @@ class DRMAAJobRunner( AsynchronousJobRunner ):
                 job_wrapper.cleanup()
             return
 
-        log.debug( "(%s) submitting file %s" % ( galaxy_id_tag, ajs.job_file ) )
-        log.debug( "(%s) command is: %s" % ( galaxy_id_tag, command_line ) )
+        log.debug( "(%s) submitting file %s", galaxy_id_tag, ajs.job_file )
+        log.debug( "(%s) command is: %s", galaxy_id_tag, command_line )
+        if native_spec:
+            log.debug( "(%s) native specification is: %s", galaxy_id_tag, native_spec )
 
         # runJob will raise if there's a submit problem
         if self.external_runJob_script is None:
@@ -175,6 +203,20 @@ class DRMAAJobRunner( AsynchronousJobRunner ):
         # Add to our 'queue' of jobs to monitor
         self.monitor_queue.put( ajs )
 
+    def _complete_terminal_job( self, ajs, drmaa_state, **kwargs ):
+        """
+        Handle a job upon its termination in the DRM. This method is meant to
+        be overridden by subclasses to improve post-mortem and reporting of
+        failures.
+        """
+        if drmaa_state == drmaa.JobState.FAILED:
+            if ajs.job_wrapper.get_state() != model.Job.states.DELETED:
+                ajs.stop_job = False
+                ajs.fail_message = "The cluster DRM system terminated this job"
+                self.work_queue.put( ( self.fail_job, ajs ) )
+        elif drmaa_state == drmaa.JobState.DONE:
+            super( DRMAAJobRunner, self )._complete_terminal_job( ajs )
+
     def check_watched_items( self ):
         """
         Called by the monitor thread to look at each watched job and deal
@@ -188,16 +230,27 @@ class DRMAAJobRunner( AsynchronousJobRunner ):
             try:
                 assert external_job_id not in ( None, 'None' ), '(%s/%s) Invalid job id' % ( galaxy_id_tag, external_job_id )
                 state = self.ds.jobStatus( external_job_id )
-            # InternalException was reported to be necessary on some DRMs, but
-            # this could cause failures to be detected as completion!  Please
-            # report if you experience problems with this.
-            except ( drmaa.InvalidJobException, drmaa.InternalException ), e:
-                # we should only get here if an orphaned job was put into the queue at app startup
-                log.info( "(%s/%s) job left DRM queue with following message: %s" % ( galaxy_id_tag, external_job_id, e ) )
-                self.work_queue.put( ( self.finish_job, ajs ) )
+            except ( drmaa.InternalException, drmaa.InvalidJobException ), e:
+                ecn = e.__class__.__name__
+                retry_param = ecn.lower() + '_retries'
+                state_param = ecn.lower() + '_state'
+                retries = getattr( ajs, retry_param, 0 )
+                if self.runner_params[ retry_param ] > 0:
+                    if retries < self.runner_params[ retry_param ]:
+                        # will retry check on next iteration
+                        setattr( ajs, retry_param, retries + 1 )
+                        continue
+                if self.runner_params[ state_param ] == model.Job.states.OK:
+                    log.info( "(%s/%s) job left DRM queue with following message: %s", galaxy_id_tag, external_job_id, e )
+                    self.work_queue.put( ( self.finish_job, ajs ) )
+                elif self.runner_params[ state_param ] == model.Job.states.ERROR:
+                    log.info( "(%s/%s) job check resulted in %s after %s tries: %s", galaxy_id_tag, external_job_id, ecn, retries, e )
+                    self.work_queue.put( ( self.fail_job, ajs ) )
+                else:
+                    raise Exception( "%s is set to an invalid value (%s), this should not be possible. See galaxy.jobs.drmaa.__init__()", state_param, self.runner_params[ state_param ] )
                 continue
             except drmaa.DrmCommunicationException, e:
-                log.warning( "(%s/%s) unable to communicate with DRM: %s" % ( galaxy_id_tag, external_job_id, e ))
+                log.warning( "(%s/%s) unable to communicate with DRM: %s", galaxy_id_tag, external_job_id, e )
                 new_watched.append( ajs )
                 continue
             except Exception, e:
@@ -208,19 +261,12 @@ class DRMAAJobRunner( AsynchronousJobRunner ):
                 self.work_queue.put( ( self.fail_job, ajs ) )
                 continue
             if state != old_state:
-                log.debug( "(%s/%s) state change: %s" % ( galaxy_id_tag, external_job_id, drmaa_state[state] ) )
+                log.debug( "(%s/%s) state change: %s" % ( galaxy_id_tag, external_job_id, self.drmaa_job_state_strings[state] ) )
             if state == drmaa.JobState.RUNNING and not ajs.running:
                 ajs.running = True
                 ajs.job_wrapper.change_state( model.Job.states.RUNNING )
-            if state == drmaa.JobState.FAILED:
-                if ajs.job_wrapper.get_state() != model.Job.states.DELETED:
-                    ajs.stop_job = False
-                    ajs.fail_message = "The cluster DRM system terminated this job"
-                    self.work_queue.put( ( self.fail_job, ajs ) )
-                continue
-            if state == drmaa.JobState.DONE:
-                if ajs.job_wrapper.get_state() != model.Job.states.DELETED:
-                    self.work_queue.put( ( self.finish_job, ajs ) )
+            if state in ( drmaa.JobState.FAILED, drmaa.JobState.DONE ):
+                self._complete_terminal_job( ajs, drmaa_state = state )
                 continue
             ajs.old_state = state
             new_watched.append( ajs )

lib/galaxy/jobs/runners/slurm.py

@@ -0,0 +1,57 @@
+"""
+SLURM job control via the DRMAA API.
+"""
+
+import time
+import logging
+import subprocess
+
+from galaxy import model
+from galaxy.jobs.runners.drmaa import DRMAAJobRunner
+
+log = logging.getLogger( __name__ )
+
+__all__ = [ 'SlurmJobRunner' ]
+
+
+class SlurmJobRunner( DRMAAJobRunner ):
+    runner_name = "SlurmRunner"
+
+    def _complete_terminal_job( self, ajs, drmaa_state, **kwargs ):
+        def __get_jobinfo():
+            scontrol_out = subprocess.check_output( ( 'scontrol', '-o', 'show', 'job', ajs.job_id ) )
+            return dict( [ out_param.split( '=', 1 ) for out_param in scontrol_out.split() ] )
+        if drmaa_state == self.drmaa_job_states.FAILED:
+            try:
+                job_info = __get_jobinfo()
+                sleep = 1
+                while job_info['JobState'] == 'COMPLETING':
+                    log.debug( '(%s/%s) Waiting %s seconds for failed job to exit COMPLETING state for post-mortem', ajs.job_wrapper.get_id_tag(), ajs.job_id, sleep )
+                    time.sleep( sleep )
+                    sleep *= 2
+                    if sleep > 64:
+                        ajs.fail_message = "This job failed and the system timed out while trying to determine the cause of the failure."
+                        break
+                    job_info = __get_jobinfo()
+                if job_info['JobState'] == 'TIMEOUT':
+                    ajs.fail_message = "This job was terminated because it ran longer than the maximum allowed job run time."
+                elif job_info['JobState'] == 'NODE_FAIL':
+                    log.warning( '(%s/%s) Job failed due to node failure, attempting resubmission', ajs.job_wrapper.get_id_tag(), ajs.job_id )
+                    ajs.job_wrapper.change_state( model.Job.states.QUEUED, info = 'Job was resubmitted due to node failure' )
+                    try:
+                        self.queue_job( ajs.job_wrapper )
+                        return
+                    except:
+                        ajs.fail_message = "This job failed due to a cluster node failure, and an attempt to resubmit the job failed."
+                elif job_info['JobState'] == 'CANCELLED':
+                    ajs.fail_message = "This job failed because it was cancelled by an administrator."
+                else:
+                    ajs.fail_message = "This job failed for reasons that could not be determined."
+                ajs.fail_message += '\nPlease click the bug icon to report this problem if you need help.'
+                ajs.stop_job = False
+                self.work_queue.put( ( self.fail_job, ajs ) )
+            except Exception, e:
+                log.exception( '(%s/%s) Unable to inspect failed slurm job using scontrol, job will be unconditionally failed: %s', ajs.job_wrapper.get_id_tag(), ajs.job_id, e )
+                super( SlurmJobRunner, self )._complete_terminal_job( ajs, drmaa_state = drmaa_state )
+        elif drmaa_state == self.drmaa_job_states.DONE:
+            super( SlurmJobRunner, self )._complete_terminal_job( ajs, drmaa_state = drmaa_state )

lib/galaxy/model/__init__.py

@@ -18,6 +18,7 @@ import json
 import socket
 import time
 from string import Template
+from itertools import ifilter
 
 import galaxy.datatypes
 import galaxy.datatypes.registry
@@ -42,6 +43,11 @@ datatypes_registry = galaxy.datatypes.registry.Registry()
 # Default Value Required for unit tests
 datatypes_registry.load_datatypes()
 
+# When constructing filters with in for a fixed set of ids, maximum
+# number of items to place in the IN statement. Different databases
+# are going to have different limits so it is likely best to not let
+# this be unlimited - filter in Python if over this limit.
+MAX_IN_FILTER_LENGTH = 100
 
 class NoConverterException(Exception):
     def __init__(self, value):
@@ -892,6 +898,33 @@ class History( object, Dictifiable, UsesAnnotations ):
             rval = galaxy.datatypes.data.nice_size( rval )
         return rval
 
+    def contents_iter( self, **kwds ):
+        """
+        Fetch filtered list of contents of history.
+        """
+        python_filter = None
+        db_session = object_session( self )
+        assert db_session != None
+        query = db_session.query( HistoryDatasetAssociation ).filter( HistoryDatasetAssociation.table.c.history_id == self.id )
+        query = query.order_by( HistoryDatasetAssociation.table.c.hid.asc() )
+        deleted = galaxy.util.string_as_bool_or_none( kwds.get( 'deleted', None ) )
+        if deleted is not None:
+            query = query.filter( HistoryDatasetAssociation.deleted == bool( kwds['deleted'] ) )
+        visible = galaxy.util.string_as_bool_or_none( kwds.get( 'visible', None ) )
+        if visible is not None:
+            query = query.filter( HistoryDatasetAssociation.visible == bool( kwds['visible'] ) )
+        if 'ids' in kwds:
+            ids = kwds['ids']
+            max_in_filter_length = kwds.get('max_in_filter_length', MAX_IN_FILTER_LENGTH)
+            if len(ids) < max_in_filter_length:
+                query = query.filter( HistoryDatasetAssociation.id.in_(ids) )
+            else:
+                python_filter = lambda hda: hda.id in ids
+        if python_filter:
+            return ifilter(python_filter, query)
+        else:
+            return query
+
     def copy_tags_from(self,target_user,source_history):
         for src_shta in source_history.tags:
             new_shta = src_shta.copy()

lib/galaxy/webapps/galaxy/api/history_contents.py

@@ -51,47 +51,28 @@ class HistoryContentsController( BaseAPIController, UsesHistoryDatasetAssociatio
             else:
                 history = self.get_history( trans, history_id, check_ownership=True, check_accessible=True )
 
-            # if ids, return _FULL_ data (as show) for each id passed
+            contents_kwds = {}
             if ids:
-                ids = ids.split( ',' )
-                for index, hda in enumerate( history.datasets ):
-                    encoded_hda_id = trans.security.encode_id( hda.id )
-                    if encoded_hda_id in ids:
-                        #TODO: share code with show
-                        rval.append( self._detailed_hda_dict( trans, hda ) )
-
-            # if no ids passed, return a _SUMMARY_ of _all_ datasets in the history
+                ids = map( lambda id: trans.security.decode_id( id ), ids.split( ',' ) )
+                contents_kwds[ 'ids' ] = ids
+                # If explicit ids given, always used detailed result.
+                details = 'all'
             else:
+                contents_kwds[ 'deleted' ] = kwd.get( 'deleted', None )
+                contents_kwds[ 'visible' ] = kwd.get( 'visible', None )
                 # details param allows a mixed set of summary and detailed hdas
                 #TODO: this is getting convoluted due to backwards compat
                 details = kwd.get( 'details', None ) or []
                 if details and details != 'all':
                     details = util.listify( details )
 
-                # by default return all datasets - even if deleted or hidden (defaulting the next switches to None)
-                #   if specified return those datasets that match the setting
-                # backwards compat
-                return_deleted = util.string_as_bool_or_none( kwd.get( 'deleted', None ) )
-                return_visible = util.string_as_bool_or_none( kwd.get( 'visible', None ) )
-
-                for hda in history.datasets:
-                    # if either return_ setting has been requested (!= None), skip hdas that don't match the request
-                    if return_deleted is not None:
-                        if( ( return_deleted and not hda.deleted )
-                        or  ( not return_deleted and hda.deleted ) ):
-                            continue
-                    if return_visible is not None:
-                        if( ( return_visible and not hda.visible )
-                        or  ( not return_visible and hda.visible ) ):
-                            continue
-
-                    encoded_hda_id = trans.security.encode_id( hda.id )
-                    if( ( encoded_hda_id in details )
-                    or  ( details == 'all' ) ):
-                        rval.append( self._detailed_hda_dict( trans, hda ) )
-                    else:
-                        rval.append( self._summary_hda_dict( trans, history_id, hda ) )
-
+            for hda in history.contents_iter( **contents_kwds ):
+                encoded_hda_id = trans.security.encode_id( hda.id )
+                detailed = details == 'all' or ( encoded_hda_id in details )
+                if detailed:
+                    rval.append( self._detailed_hda_dict( trans, hda ) )
+                else:
+                    rval.append( self._summary_hda_dict( trans, history_id, hda ) )
         except Exception, e:
             # for errors that are not specific to one hda (history lookup or summary list)
             rval = "Error in history API at listing contents: " + str( e )

lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py

@@ -19,11 +19,6 @@ import tool_shed.util.shed_util_common as suc
 
 log = logging.getLogger( __name__ )
 
-def default_tool_shed_repository_value_mapper( trans, tool_shed_repository ):
-    value_mapper={ 'id' : trans.security.encode_id( tool_shed_repository.id ),
-                   'error_message' : tool_shed_repository.error_message or '' }
-    return value_mapper
-
 def get_message_for_no_shed_tool_config():
     # This Galaxy instance is not configured with a shed-related tool panel configuration file.
     message = 'The tool_config_file setting in universe_wsgi.ini must include at least one shed tool configuration file name with a <toolbox> '
@@ -48,8 +43,8 @@ class ToolShedRepositoriesController( BaseAPIController ):
         :param id: the encoded id of the ToolShedRepository object
         """
         # Example URL: http://localhost:8763/api/tool_shed_repositories/f2db41e1fa331b3e/exported_workflows
-        # Since exported workflows are dictionaries with very few attributes that differentiate them from each other, we'll build the
-        # list based on the following dictionary of those few attributes.
+        # Since exported workflows are dictionaries with very few attributes that differentiate them from each
+        # other, we'll build the list based on the following dictionary of those few attributes.
         exported_workflows = []
         repository = suc.get_tool_shed_repository_by_id( trans, id )
         metadata = repository.metadata
@@ -58,17 +53,23 @@ class ToolShedRepositoriesController( BaseAPIController ):
         else:
             exported_workflow_tups = []
         for index, exported_workflow_tup in enumerate( exported_workflow_tups ):
-            # The exported_workflow_tup looks like ( relative_path, exported_workflow_dict ), where the value of relative_path is the location
-            # on disk (relative to the root of the installed repository) where the exported_workflow_dict file (.ga file) is located.
+            # The exported_workflow_tup looks like ( relative_path, exported_workflow_dict ), where the value of
+            # relative_path is the location on disk (relative to the root of the installed repository) where the
+            # exported_workflow_dict file (.ga file) is located.
             exported_workflow_dict = exported_workflow_tup[ 1 ]
             annotation = exported_workflow_dict.get( 'annotation', '' )
             format_version = exported_workflow_dict.get( 'format-version', '' )
             workflow_name = exported_workflow_dict.get( 'name', '' )
-            # Since we don't have an in-memory object with an id, we'll identify the exported workflow via it's location (i.e., index) in the list.
+            # Since we don't have an in-memory object with an id, we'll identify the exported workflow via it's
+            # location (i.e., index) in the list.
             display_dict = dict( index=index, annotation=annotation, format_version=format_version, workflow_name=workflow_name )
             exported_workflows.append( display_dict )
         return exported_workflows
 
+    def __get_value_mapper( self, trans ):
+        value_mapper = { 'id' : trans.security.encode_id }
+        return value_mapper
+
     @web.expose_api
     def import_workflow( self, trans, payload, **kwd ):
         """
@@ -96,13 +97,11 @@ class ToolShedRepositoriesController( BaseAPIController ):
         # Since we don't have an in-memory object with an id, we'll identify the exported workflow via it's location (i.e., index) in the list.
         exported_workflow = exported_workflows[ int( index ) ]
         workflow_name = exported_workflow[ 'workflow_name' ]
-        workflow, status, message = workflow_util.import_workflow( trans, repository, workflow_name )
+        workflow, status, error_message = workflow_util.import_workflow( trans, repository, workflow_name )
         if status == 'error':
-            log.error( message, exc_info=True )
-            trans.response.status = 500
-            return message
-        else:
-            return workflow.to_dict( view='element' )
+            log.debug( error_message )
+            return {}
+        return workflow.to_dict( view='element' )
 
     @web.expose_api
     def import_workflows( self, trans, **kwd ):
@@ -125,11 +124,9 @@ class ToolShedRepositoriesController( BaseAPIController ):
         imported_workflow_dicts = []
         for exported_workflow_dict in exported_workflows:
             workflow_name = exported_workflow_dict[ 'workflow_name' ]
-            workflow, status, message = workflow_util.import_workflow( trans, repository, workflow_name )
+            workflow, status, error_message = workflow_util.import_workflow( trans, repository, workflow_name )
             if status == 'error':
-                log.error( message, exc_info=True )
-                trans.response.status = 500
-                return message
+                log.debug( error_message )
             else:
                 imported_workflow_dicts.append( workflow.to_dict( view='element' ) )
         return imported_workflow_dicts
@@ -142,22 +139,15 @@ class ToolShedRepositoriesController( BaseAPIController ):
         """
         # Example URL: http://localhost:8763/api/tool_shed_repositories
         tool_shed_repository_dicts = []
-        try:
-            query = trans.install_model.context.query( trans.app.install_model.ToolShedRepository ) \
-                                    .order_by( trans.app.install_model.ToolShedRepository.table.c.name ) \
-                                    .all()
-            for tool_shed_repository in query:
-                tool_shed_repository_dict = tool_shed_repository.to_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, tool_shed_repository ) )
-                tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories',
-                                                                  action='show',
-                                                                  id=trans.security.encode_id( tool_shed_repository.id ) )
-                tool_shed_repository_dicts.append( tool_shed_repository_dict )
-            return tool_shed_repository_dicts
-        except Exception, e:
-            message = "Error in the tool_shed_repositories API in index: %s" % str( e )
-            log.error( message, exc_info=True )
-            trans.response.status = 500
-            return message
+        for tool_shed_repository in trans.install_model.context.query( trans.app.install_model.ToolShedRepository ) \
+                                                               .order_by( trans.app.install_model.ToolShedRepository.table.c.name ):
+            tool_shed_repository_dict = \
+                tool_shed_repository.to_dict( value_mapper=self.__get_value_mapper( trans ) )
+            tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories',
+                                                              action='show',
+                                                              id=trans.security.encode_id( tool_shed_repository.id ) )
+            tool_shed_repository_dicts.append( tool_shed_repository_dict )
+        return tool_shed_repository_dicts
 
     @web.expose_api
     def install_repository_revision( self, trans, payload, **kwd ):
@@ -208,8 +198,7 @@ class ToolShedRepositoriesController( BaseAPIController ):
         # Make sure this Galaxy instance is configured with a shed-related tool panel configuration file.
         if not suc.have_shed_tool_conf_for_install( trans ):
             message = get_message_for_no_shed_tool_config()
-            log.error( message, exc_info=True )
-            trans.response.status = 500
+            log.debug( message )
             return dict( status='error', error=message )
         # Make sure the current user's API key proves he is an admin user in this Galaxy instance.
         if not trans.user_is_admin():
@@ -225,18 +214,20 @@ class ToolShedRepositoriesController( BaseAPIController ):
         except Exception, e:
             message = "Error attempting to retrieve installation information from tool shed %s for revision %s of repository %s owned by %s: %s" % \
                 ( str( tool_shed_url ), str( changeset_revision ), str( name ), str( owner ), str( e ) )
-            log.error( message, exc_info=True )
-            trans.response.status = 500
+            log.debug( message )
             return dict( status='error', error=message )
         if raw_text:
+            # If successful, the response from get_repository_revision_install_info will be 3
+            # dictionaries, a dictionary defining the Repository, a dictionary defining the
+            # Repository revision (RepositoryMetadata), and a dictionary including the additional
+            # information required to install the repository.
             items = json.from_json_string( raw_text )
             repository_revision_dict = items[ 1 ]
             repo_info_dict = items[ 2 ]
         else:
             message = "Unable to retrieve installation information from tool shed %s for revision %s of repository %s owned by %s: %s" % \
                 ( str( tool_shed_url ), str( changeset_revision ), str( name ), str( owner ), str( e ) )
-            log.error( message, exc_info=True )
-            trans.response.status = 500
+            log.debug( message )
             return dict( status='error', error=message )
         repo_info_dicts = [ repo_info_dict ]
         # Make sure the tool shed returned everything we need for installing the repository.
@@ -345,7 +336,7 @@ class ToolShedRepositoriesController( BaseAPIController ):
                                                                   tool_path,
                                                                   install_tool_dependencies,
                                                                   reinstalling=False )
-                    tool_shed_repository_dict = tool_shed_repository.as_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, tool_shed_repository ) )
+                    tool_shed_repository_dict = tool_shed_repository.as_dict( value_mapper=self.__get_value_mapper( trans ) )
                     tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories',
                                                                       action='show',
                                                                       id=trans.security.encode_id( tool_shed_repository.id ) )
@@ -394,8 +385,7 @@ class ToolShedRepositoriesController( BaseAPIController ):
         if not suc.have_shed_tool_conf_for_install( trans ):
             # This Galaxy instance is not configured with a shed-related tool panel configuration file.
             message = get_message_for_no_shed_tool_config()
-            log.error( message, exc_info=True )
-            trans.response.status = 500
+            log.debug( message )
             return dict( status='error', error=message )
         if not trans.user_is_admin():
             raise HTTPForbidden( detail='You are not authorized to install a tool shed repository into this Galaxy instance.' )
@@ -410,8 +400,7 @@ class ToolShedRepositoriesController( BaseAPIController ):
             len( changeset_revisions ) != num_specified_repositories:
             message = 'Error in tool_shed_repositories API in install_repository_revisions: the received parameters must be ordered '
             message += 'lists so that positional values in tool_shed_urls, names, owners and changeset_revisions are associated.'
-            log.error( message, exc_info=True )
-            trans.response.status = 500
+            log.debug( message )
             return dict( status='error', error=message )
         # Get the information about the Galaxy components (e.g., tool pane section, tool config file, etc) that will contain information
         # about each of the repositories being installed.
@@ -482,7 +471,7 @@ class ToolShedRepositoriesController( BaseAPIController ):
                 repair_dict = repository_util.repair_tool_shed_repository( trans,
                                                                            repository,
                                                                            encoding_util.tool_shed_encode( repo_info_dict ) )
-                repository_dict = repository.to_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, repository ) )
+                repository_dict = repository.to_dict( value_mapper=self.__get_value_mapper( trans ) )
                 repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories',
                                                         action='show',
                                                         id=trans.security.encode_id( repository.id ) )
@@ -502,39 +491,39 @@ class ToolShedRepositoriesController( BaseAPIController ):
 
         :param key: the API key of the Galaxy admin user.
         """
-        try:
-            start_time = strftime( "%Y-%m-%d %H:%M:%S" )
-            results = dict( start_time=start_time,
-                            successful_count=0,
-                            unsuccessful_count=0,
-                            repository_status=[] )
-            # Make sure the current user's API key proves he is an admin user in this Galaxy instance.
-            if not trans.user_is_admin():
-                raise HTTPForbidden( detail='You are not authorized to reset metadata on repositories installed into this Galaxy instance.' )
-            query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=False, order=False )
-            # Now reset metadata on all remaining repositories.
-            for repository in query:
-                repository_id = trans.security.encode_id( repository.id )
-                try:
-                    invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_installed_repository( trans, repository_id )
-                    if invalid_file_tups:
-                        message = tool_util.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, None, as_html=False )
-                        results[ 'unsuccessful_count' ] += 1
-                    else:
-                        message = "Successfully reset metadata on repository %s owned by %s" % ( str( repository.name ), str( repository.owner ) )
-                        results[ 'successful_count' ] += 1
-                except Exception, e:
-                    message = "Error resetting metadata on repository %s owned by %s: %s" % ( str( repository.name ), str( repository.owner ), str( e ) )
+        start_time = strftime( "%Y-%m-%d %H:%M:%S" )
+        results = dict( start_time=start_time,
+                        successful_count=0,
+                        unsuccessful_count=0,
+                        repository_status=[] )
+        # Make sure the current user's API key proves he is an admin user in this Galaxy instance.
+        if not trans.user_is_admin():
+            raise HTTPForbidden( detail='You are not authorized to reset metadata on repositories installed into this Galaxy instance.' )
+        query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=False, order=False )
+        # Now reset metadata on all remaining repositories.
+        for repository in query:
+            repository_id = trans.security.encode_id( repository.id )
+            try:
+                invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_installed_repository( trans, repository_id )
+                if invalid_file_tups:
+                    message = tool_util.generate_message_for_invalid_tools( trans,
+                                                                            invalid_file_tups,
+                                                                            repository,
+                                                                            None,
+                                                                            as_html=False )
                     results[ 'unsuccessful_count' ] += 1
-                results[ 'repository_status' ].append( message )
-            stop_time = strftime( "%Y-%m-%d %H:%M:%S" )
-            results[ 'stop_time' ] = stop_time
-            return json.to_json_string( results, sort_keys=True, indent=4 * ' ' )
-        except Exception, e:
-            message = "Error in the Galaxy tool_shed_repositories API in reset_metadata_on_installed_repositories: %s" % str( e )
-            log.error( message, exc_info=True )
-            trans.response.status = 500
-            return message
+                else:
+                    message = "Successfully reset metadata on repository %s owned by %s" % \
+                        ( str( repository.name ), str( repository.owner ) )
+                    results[ 'successful_count' ] += 1
+            except Exception, e:
+                message = "Error resetting metadata on repository %s owned by %s: %s" % \
+                    ( str( repository.name ), str( repository.owner ), str( e ) )
+                results[ 'unsuccessful_count' ] += 1
+            results[ 'repository_status' ].append( message )
+        stop_time = strftime( "%Y-%m-%d %H:%M:%S" )
+        results[ 'stop_time' ] = stop_time
+        return json.to_json_string( results, sort_keys=True, indent=4 * ' ' )
 
     @web.expose_api
     def show( self, trans, id, **kwd ):
@@ -545,15 +534,12 @@ class ToolShedRepositoriesController( BaseAPIController ):
         :param id: the encoded id of the ToolShedRepository object
         """
         # Example URL: http://localhost:8763/api/tool_shed_repositories/df7a1f0c02a5b08e
-        try:
-            tool_shed_repository = suc.get_tool_shed_repository_by_id( trans, id )
-            tool_shed_repository_dict = tool_shed_repository.as_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, tool_shed_repository ) )
-            tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories',
-                                                              action='show',
-                                                              id=trans.security.encode_id( tool_shed_repository.id ) )
-            return tool_shed_repository_dict
-        except Exception, e:
-            message = "Error in tool_shed_repositories API in index: " + str( e )
-            log.error( message, exc_info=True )
-            trans.response.status = 500
-            return message
+        tool_shed_repository = suc.get_tool_shed_repository_by_id( trans, id )
+        if tool_shed_repository is None:
+            log.debug( "Unable to locate tool_shed_repository record for id %s." % ( str( id ) ) )
+            return {}
+        tool_shed_repository_dict = tool_shed_repository.as_dict( value_mapper=self.__get_value_mapper( trans ) )
+        tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories',
+                                                          action='show',
+                                                          id=trans.security.encode_id( tool_shed_repository.id ) )
+        return tool_shed_repository_dict

lib/galaxy/webapps/tool_shed/api/repositories.py

@@ -36,22 +36,27 @@ class RepositoriesController( BaseAPIController ):
         :param name: the name of the Repository
         :param owner: the owner of the Repository
 
-        Returns the ordered list of changeset revision hash strings that are associated with installable revisions.  As in the changelog, the
-        list is ordered oldest to newest.
+        Returns the ordered list of changeset revision hash strings that are associated with installable revisions.
+        As in the changelog, the list is ordered oldest to newest.
         """
         # Example URL: http://localhost:9009/api/repositories/get_installable_revisions?name=add_column&owner=test
-        try:
+        if name and owner:
             # Get the repository information.
             repository = suc.get_repository_by_name_and_owner( trans.app, name, owner )
+            if repository is None:
+                error_message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: "
+                error_message += "cannot locate repository %s owned by %s." % ( str( name ), str( owner ) )
+                log.debug( error_message )
+                return []
             repo_dir = repository.repo_path( trans.app )
             repo = hg.repository( suc.get_configured_ui(), repo_dir )
             ordered_installable_revisions = suc.get_ordered_metadata_changeset_revisions( repository, repo, downloadable=True )
             return ordered_installable_revisions
-        except Exception, e:
-            message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: %s" % str( e )
-            log.error( message, exc_info=True )
-            trans.response.status = 500
-            return message
+        else:
+            error_message = "Error in the Tool Shed repositories API in get_ordered_installable_revisions: "
+            error_message += "invalid name %s or owner %s received." % ( str( name ), str( owner ) )
+            log.debug( error_message )
+            return []
 
     @web.expose_api_anonymous
     def get_repository_revision_install_info( self, trans, name, owner, changeset_revision, **kwd ):
@@ -106,49 +111,65 @@ class RepositoriesController( BaseAPIController ):
             ]
         }
         """
-        repository_value_mapper = { 'id' : trans.security.encode_id,
-                                    'user_id' : trans.security.encode_id }
-        # Example URL: http://localhost:9009/api/repositories/get_repository_revision_install_info?name=add_column&owner=test&changeset_revision=3a08cc21466f
-        try:
+        # Example URL: 
+        # http://<xyz>/api/repositories/get_repository_revision_install_info?name=<n>&owner=<o>&changeset_revision=<cr>
+        if name and owner and changeset_revision:
             # Get the repository information.
             repository = suc.get_repository_by_name_and_owner( trans.app, name, owner )
+            if repository is None:
+                log.debug( 'Cannot locate repository %s owned by %s' % ( str( name ), str( owner ) ) )
+                return {}, {}, {}
             encoded_repository_id = trans.security.encode_id( repository.id )
-            repository_dict = repository.to_dict( view='element', value_mapper=repository_value_mapper )
+            repository_dict = repository.to_dict( view='element',
+                                                  value_mapper=self.__get_value_mapper( trans ) )
             repository_dict[ 'url' ] = web.url_for( controller='repositories',
                                                     action='show',
                                                     id=encoded_repository_id )
             # Get the repository_metadata information.
-            repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, encoded_repository_id, changeset_revision )
+            repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans,
+                                                                                     encoded_repository_id,
+                                                                                     changeset_revision )
             if not repository_metadata:
-                # The changeset_revision column in the repository_metadata table has been updated with a new value value, so find the
-                # changeset_revision to which we need to update.
+                # The changeset_revision column in the repository_metadata table has been updated with a new
+                # value value, so find the changeset_revision to which we need to update.
                 repo_dir = repository.repo_path( trans.app )
                 repo = hg.repository( suc.get_configured_ui(), repo_dir )
                 new_changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision )
-                repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, encoded_repository_id, new_changeset_revision )
+                repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans,
+                                                                                         encoded_repository_id,
+                                                                                         new_changeset_revision )
                 changeset_revision = new_changeset_revision
             if repository_metadata:
                 encoded_repository_metadata_id = trans.security.encode_id( repository_metadata.id )
                 repository_metadata_dict = repository_metadata.to_dict( view='collection',
-                                                                              value_mapper=self.__get_value_mapper( trans, repository_metadata ) )
+                                                                        value_mapper=self.__get_value_mapper( trans ) )
                 repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions',
                                                                  action='show',
                                                                  id=encoded_repository_metadata_id )
                 # Get the repo_info_dict for installing the repository.
-                repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, \
-                    has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td = \
+                repo_info_dict, \
+                includes_tools, \
+                includes_tool_dependencies, \
+                includes_tools_for_display_in_tool_panel, \
+                has_repository_dependencies, \
+                has_repository_dependencies_only_if_compiling_contained_td = \
                     repository_util.get_repo_info_dict( trans, encoded_repository_id, changeset_revision )
                 return repository_dict, repository_metadata_dict, repo_info_dict
             else:
-                message = "Unable to locate repository_metadata record for repository id %d and changeset_revision %s" % ( repository.id, changeset_revision )
-                log.error( message, exc_info=True )
-                trans.response.status = 500
+                log.debug( "Unable to locate repository_metadata record for repository id %s and changeset_revision %s" % \
+                    ( str( repository.id ), str( changeset_revision ) ) )
                 return repository_dict, {}, {}
-        except Exception, e:
-            message = "Error in the Tool Shed repositories API in get_repository_revision_install_info: %s" % str( e )
-            log.error( message, exc_info=True )
-            trans.response.status = 500
-            return message
+        else:
+            debug_msg = "Error in the Tool Shed repositories API in get_repository_revision_install_info: "
+            debug_msg += "Invalid name %s or owner %s or changeset_revision %s received." % \
+                ( str( name ), str( owner ), str( changeset_revision ) )
+            log.debug( debug_msg )
+            return {}, {}, {}
+
+    def __get_value_mapper( self, trans ):
+        value_mapper = { 'id' : trans.security.encode_id,
+                         'repository_id' : trans.security.encode_id }
+        return value_mapper
 
     @web.expose_api
     def import_capsule( self, trans, payload, **kwd ):
@@ -177,29 +198,27 @@ class RepositoriesController( BaseAPIController ):
                              uploaded_file=None,
                              capsule_file_name=None )
         if os.path.getsize( os.path.abspath( capsule_file_name ) ) == 0:
-            message = 'Your capsule file is empty.'
-            log.error( message, exc_info=True )
-            trans.response.status = 500
-            return message
+            log.debug( 'Your capsule file %s is empty.' % str( capsule_file_name ) )
+            return {}
         try:
             # Open for reading with transparent compression.
             tar_archive = tarfile.open( capsule_file_path, 'r:*' )
         except tarfile.ReadError, e:
-            message = 'Error opening file %s: %s' % ( str( capsule_file_name ), str( e ) )
-            log.error( message, exc_info=True )
-            trans.response.status = 500
-            return message
+            log.debug( 'Error opening capsule file %s: %s' % ( str( capsule_file_name ), str( e ) ) )
+            return {}
         capsule_dict[ 'tar_archive' ] = tar_archive
         capsule_dict[ 'capsule_file_name' ] = capsule_file_name
         capsule_dict = import_util.extract_capsule_files( trans, **capsule_dict )
         capsule_dict = import_util.validate_capsule( trans, **capsule_dict )
         status = capsule_dict.get( 'status', 'error' )
         if status == 'error':
-            message = 'The capsule contents are invalid and cannpt be imported:<br/>%s' % str( capsule_dict.get( 'error_message', '' ) )
-            log.error( message, exc_info=True )
-            trans.response.status = 500
-            return message
+            log.debug( 'The capsule contents are invalid and cannpt be imported:<br/>%s' % \
+                str( capsule_dict.get( 'error_message', '' ) ) )
+            return {}
         encoded_file_path = capsule_dict.get( 'encoded_file_path', None )
+        if encoded_file_path is None:
+            log.debug( 'The capsule_dict %s is missing the required encoded_file_path entry.' % str( capsule_dict ) )
+            return {}
         file_path = encoding_util.tool_shed_decode( encoded_file_path )
         export_info_file_path = os.path.join( file_path, 'export_info.xml' )
         export_info_dict = import_util.get_export_info_dict( export_info_file_path )
@@ -216,12 +235,14 @@ class RepositoriesController( BaseAPIController ):
             # Add the capsule_file_name and encoded_file_path to the repository_status_info_dict.
             repository_status_info_dict[ 'capsule_file_name' ] = capsule_file_name
             repository_status_info_dict[ 'encoded_file_path' ] = encoded_file_path
-            import_results_tups = repository_maintenance_util.create_repository_and_import_archive( trans,
-                                                                                                    repository_status_info_dict,
-                                                                                                    import_results_tups )
+            import_results_tups = \
+                repository_maintenance_util.create_repository_and_import_archive( trans,
+                                                                                  repository_status_info_dict,
+                                                                                  import_results_tups )
         import_util.check_status_and_reset_downloadable( trans, import_results_tups )
         suc.remove_dir( file_path )
-        # NOTE: the order of installation is defined in import_results_tups, but order will be lost when transferred to return_dict.
+        # NOTE: the order of installation is defined in import_results_tups, but order will be lost
+        # when transferred to return_dict.
         return_dict = {}
         for import_results_tup in import_results_tups:
             ok, name_owner, message = import_results_tup
@@ -237,28 +258,19 @@ class RepositoriesController( BaseAPIController ):
         GET /api/repositories
         Displays a collection (list) of repositories.
         """
-        value_mapper = { 'id' : trans.security.encode_id,
-                         'user_id' : trans.security.encode_id }
         # Example URL: http://localhost:9009/api/repositories
         repository_dicts = []
-        deleted = util.string_as_bool( deleted )
-        try:
-            query = trans.sa_session.query( trans.app.model.Repository ) \
-                                    .filter( trans.app.model.Repository.table.c.deleted == deleted ) \
-                                    .order_by( trans.app.model.Repository.table.c.name ) \
-                                    .all()
-            for repository in query:
-                repository_dict = repository.to_dict( view='collection', value_mapper=value_mapper )
-                repository_dict[ 'url' ] = web.url_for( controller='repositories',
-                                                        action='show',
-                                                        id=trans.security.encode_id( repository.id ) )
-                repository_dicts.append( repository_dict )
-            return repository_dicts
-        except Exception, e:
-            message = "Error in the Tool Shed repositories API in index: %s" % str( e )
-            log.error( message, exc_info=True )
-            trans.response.status = 500
-            return message
+        deleted = util.asbool( deleted )
+        for repository in trans.sa_session.query( trans.app.model.Repository ) \
+                                          .filter( trans.app.model.Repository.table.c.deleted == deleted ) \
+                                          .order_by( trans.app.model.Repository.table.c.name ):
+            repository_dict = repository.to_dict( view='collection',
+                                                  value_mapper=self.__get_value_mapper( trans ) )
+            repository_dict[ 'url' ] = web.url_for( controller='repositories',
+                                                    action='show',
+                                                    id=trans.security.encode_id( repository.id ) )
+            repository_dicts.append( repository_dict )
+        return repository_dicts
 
     @web.expose_api
     def repository_ids_for_setting_metadata( self, trans, my_writable=False, **kwd ):
@@ -273,28 +285,22 @@ class RepositoriesController( BaseAPIController ):
                                        in addition to those repositories of type tool_dependency_definition.  This param is ignored
                                        if the current user is not an admin user, in which case this same restriction is automatic.
         """
-        try:
-            if trans.user_is_admin():
-                my_writable = util.asbool( my_writable )
-            else:
-                my_writable = True
-            handled_repository_ids = []
-            repository_ids = []
-            query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False )
-            # Make sure repositories of type tool_dependency_definition are first in the list.
-            for repository in query:
-                if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids:
-                    repository_ids.append( trans.security.encode_id( repository.id ) )
-            # Now add all remaining repositories to the list.
-            for repository in query:
-                if repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids:
-                    repository_ids.append( trans.security.encode_id( repository.id ) )
-            return repository_ids
-        except Exception, e:
-            message = "Error in the Tool Shed repositories API in repository_ids_for_setting_metadata: %s" % str( e )
-            log.error( message, exc_info=True )
-            trans.response.status = 500
-            return message
+        if trans.user_is_admin():
+            my_writable = util.asbool( my_writable )
+        else:
+            my_writable = True
+        handled_repository_ids = []
+        repository_ids = []
+        query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False )
+        # Make sure repositories of type tool_dependency_definition are first in the list.
+        for repository in query:
+            if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids:
+                repository_ids.append( trans.security.encode_id( repository.id ) )
+        # Now add all remaining repositories to the list.
+        for repository in query:
+            if repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids:
+                repository_ids.append( trans.security.encode_id( repository.id ) )
+        return repository_ids
 
     @web.expose_api
     def reset_metadata_on_repositories( self, trans, payload, **kwd ):
@@ -318,6 +324,7 @@ class RepositoriesController( BaseAPIController ):
         :param skip_file (optional): A local file name that contains the encoded repository ids associated with repositories to skip.
                                      This param can be used as an alternative to the above encoded_ids_to_skip.
         """
+
         def handle_repository( trans, repository, results ):
             log.debug( "Resetting metadata on repository %s" % str( repository.name ) )
             repository_id = trans.security.encode_id( repository.id )
@@ -335,53 +342,48 @@ class RepositoriesController( BaseAPIController ):
             status = '%s : %s' % ( str( repository.name ), message )
             results[ 'repository_status' ].append( status )
             return results
-        try:
-            start_time = strftime( "%Y-%m-%d %H:%M:%S" )
-            results = dict( start_time=start_time,
-                            repository_status=[],
-                            successful_count=0,
-                            unsuccessful_count=0 )
-            handled_repository_ids = []
-            encoded_ids_to_skip = payload.get( 'encoded_ids_to_skip', [] )
-            skip_file = payload.get( 'skip_file', None )
-            if skip_file and os.path.exists( skip_file ) and not encoded_ids_to_skip:
-                # Load the list of encoded_ids_to_skip from the skip_file.
-                # Contents of file must be 1 encoded repository id per line.
-                lines = open( skip_file, 'rb' ).readlines()
-                for line in lines:
-                    if line.startswith( '#' ):
-                        # Skip comments.
-                        continue
-                    encoded_ids_to_skip.append( line.rstrip( '\n' ) )
-            if trans.user_is_admin():
-                my_writable = util.asbool( payload.get( 'my_writable', False ) )
-            else:
-                my_writable = True
-            query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False )
-            # First reset metadata on all repositories of type repository_dependency_definition.
-            for repository in query:
-                encoded_id = trans.security.encode_id( repository.id )
-                if encoded_id in encoded_ids_to_skip:
-                    log.debug( "Skipping repository with id %s because it is in encoded_ids_to_skip %s" % \
-                               ( str( repository.id ), str( encoded_ids_to_skip ) ) )
-                elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids:
-                    results = handle_repository( trans, repository, results )
-            # Now reset metadata on all remaining repositories.
-            for repository in query:
-                encoded_id = trans.security.encode_id( repository.id )
-                if encoded_id in encoded_ids_to_skip:
-                    log.debug( "Skipping repository with id %s because it is in encoded_ids_to_skip %s" % \
-                               ( str( repository.id ), str( encoded_ids_to_skip ) ) )
-                elif repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids:
-                    results = handle_repository( trans, repository, results )
-            stop_time = strftime( "%Y-%m-%d %H:%M:%S" )
-            results[ 'stop_time' ] = stop_time
-            return json.to_json_string( results, sort_keys=True, indent=4 * ' ' )
-        except Exception, e:
-            message = "Error in the Tool Shed repositories API in reset_metadata_on_repositories: %s" % str( e )
-            log.error( message, exc_info=True )
-            trans.response.status = 500
-            return message
+
+        start_time = strftime( "%Y-%m-%d %H:%M:%S" )
+        results = dict( start_time=start_time,
+                        repository_status=[],
+                        successful_count=0,
+                        unsuccessful_count=0 )
+        handled_repository_ids = []
+        encoded_ids_to_skip = payload.get( 'encoded_ids_to_skip', [] )
+        skip_file = payload.get( 'skip_file', None )
+        if skip_file and os.path.exists( skip_file ) and not encoded_ids_to_skip:
+            # Load the list of encoded_ids_to_skip from the skip_file.
+            # Contents of file must be 1 encoded repository id per line.
+            lines = open( skip_file, 'rb' ).readlines()
+            for line in lines:
+                if line.startswith( '#' ):
+                    # Skip comments.
+                    continue
+                encoded_ids_to_skip.append( line.rstrip( '\n' ) )
+        if trans.user_is_admin():
+            my_writable = util.asbool( payload.get( 'my_writable', False ) )
+        else:
+            my_writable = True
+        query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False )
+        # First reset metadata on all repositories of type repository_dependency_definition.
+        for repository in query:
+            encoded_id = trans.security.encode_id( repository.id )
+            if encoded_id in encoded_ids_to_skip:
+                log.debug( "Skipping repository with id %s because it is in encoded_ids_to_skip %s" % \
+                           ( str( repository.id ), str( encoded_ids_to_skip ) ) )
+            elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids:
+                results = handle_repository( trans, repository, results )
+        # Now reset metadata on all remaining repositories.
+        for repository in query:
+            encoded_id = trans.security.encode_id( repository.id )
+            if encoded_id in encoded_ids_to_skip:
+                log.debug( "Skipping repository with id %s because it is in encoded_ids_to_skip %s" % \
+                           ( str( repository.id ), str( encoded_ids_to_skip ) ) )
+            elif repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids:
+                results = handle_repository( trans, repository, results )
+        stop_time = strftime( "%Y-%m-%d %H:%M:%S" )
+        results[ 'stop_time' ] = stop_time
+        return json.to_json_string( results, sort_keys=True, indent=4 * ' ' )
 
     @web.expose_api
     def reset_metadata_on_repository( self, trans, payload, **kwd ):
@@ -395,6 +397,7 @@ class RepositoriesController( BaseAPIController ):
         The following parameters must be included in the payload.
         :param repository_id: the encoded id of the repository on which metadata is to be reset.
         """
+
         def handle_repository( trans, start_time, repository ):
             results = dict( start_time=start_time,
                             repository_status=[] )
@@ -410,21 +413,16 @@ class RepositoriesController( BaseAPIController ):
             status = '%s : %s' % ( str( repository.name ), message )
             results[ 'repository_status' ].append( status )
             return results
-        try:
-            repository_id = payload.get( 'repository_id', None )
-            if repository_id is not None:
-                repository = suc.get_repository_in_tool_shed( trans, repository_id )
-                start_time = strftime( "%Y-%m-%d %H:%M:%S" )
-                log.debug( "%s...resetting metadata on repository %s" % ( start_time, str( repository.name ) ) )
-                results = handle_repository( trans, start_time, repository )
-                stop_time = strftime( "%Y-%m-%d %H:%M:%S" )
-                results[ 'stop_time' ] = stop_time
-            return json.to_json_string( results, sort_keys=True, indent=4 * ' ' )
-        except Exception, e:
-            message = "Error in the Tool Shed repositories API in reset_metadata_on_repositories: %s" % str( e )
-            log.error( message, exc_info=True )
-            trans.response.status = 500
-            return message
+
+        repository_id = payload.get( 'repository_id', None )
+        if repository_id is not None:
+            repository = suc.get_repository_in_tool_shed( trans, repository_id )
+            start_time = strftime( "%Y-%m-%d %H:%M:%S" )
+            log.debug( "%s...resetting metadata on repository %s" % ( start_time, str( repository.name ) ) )
+            results = handle_repository( trans, start_time, repository )
+            stop_time = strftime( "%Y-%m-%d %H:%M:%S" )
+            results[ 'stop_time' ] = stop_time
+        return json.to_json_string( results, sort_keys=True, indent=4 * ' ' )
 
     @web.expose_api_anonymous
     def show( self, trans, id, **kwd ):
@@ -434,27 +432,14 @@ class RepositoriesController( BaseAPIController ):
 
         :param id: the encoded id of the Repository object
         """
-        value_mapper = { 'id' : trans.security.encode_id,
-                         'user_id' : trans.security.encode_id }
         # Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135
-        try:
-            repository = suc.get_repository_in_tool_shed( trans, id )
-            repository_dict = repository.to_dict( view='element', value_mapper=value_mapper )
-            repository_dict[ 'url' ] = web.url_for( controller='repositories',
-                                                    action='show',
-                                                    id=trans.security.encode_id( repository.id ) )
-            return repository_dict
-        except Exception, e:
-            message = "Error in the Tool Shed repositories API in show: %s" % str( e )
-            log.error( message, exc_info=True )
-            trans.response.status = 500
-            return message
-
-    def __get_value_mapper( self, trans, repository_metadata ):
-        value_mapper = { 'id' : trans.security.encode_id,
-                         'repository_id' : trans.security.encode_id }
-        if repository_metadata.time_last_tested is not None:
-            # For some reason the Dictifiable.to_dict() method in ~/galaxy/model/item_attrs.py requires
-            # a function rather than a mapped value, so just pass the time_ago function here.
-            value_mapper[ 'time_last_tested' ] = time_ago
-        return value_mapper
+        repository = suc.get_repository_in_tool_shed( trans, id )
+        if repository is None:
+            log.debug( "Unable to locate repository record for id %s." % ( str( id ) ) )
+            return {}
+        repository_dict = repository.to_dict( view='element',
+                                              value_mapper=self.__get_value_mapper( trans ) )
+        repository_dict[ 'url' ] = web.url_for( controller='repositories',
+                                                action='show',
+                                                id=trans.security.encode_id( repository.id ) )
+        return repository_dict

lib/galaxy/webapps/tool_shed/api/repository_revisions.py

@@ -1,6 +1,5 @@
 import datetime
 import logging
-from galaxy.web.framework.helpers import time_ago
 from tool_shed.util import metadata_util
 from galaxy import web
 from galaxy import util
@@ -48,61 +47,29 @@ class RepositoryRevisionsController( BaseAPIController ):
         if not changeset_revision:
             raise HTTPBadRequest( detail="Missing required parameter 'changeset_revision'." )
         export_repository_dependencies = payload.get( 'export_repository_dependencies', False )
-        try:
-            # We'll currently support only gzip-compressed tar archives.
-            file_type = 'gz'
-            export_repository_dependencies = util.string_as_bool( export_repository_dependencies )
-            # Get the repository information.
-            repository = suc.get_repository_by_name_and_owner( trans.app, name, owner )
-            repository_id = trans.security.encode_id( repository.id )
-            response = export_util.export_repository( trans,
-                                                      tool_shed_url,
-                                                      repository_id,
-                                                      str( repository.name ),
-                                                      changeset_revision,
-                                                      file_type,
-                                                      export_repository_dependencies,
-                                                      api=True )
-            return response
-        except Exception, e:
-            message = "Error in the Tool Shed repository_revisions API in export: %s" % str( e )
-            log.error( message, exc_info=True )
-            trans.response.status = 500
-            return message
+        # We'll currently support only gzip-compressed tar archives.
+        file_type = 'gz'
+        export_repository_dependencies = util.asbool( export_repository_dependencies )
+        # Get the repository information.
+        repository = suc.get_repository_by_name_and_owner( trans.app, name, owner )
+        if repository is None:
+            error_message = 'Cannot locate repository with name %s and owner %s,' % ( str( name ), str( owner ) )
+            log.debug( error_message )
+            return None, error_message
+        repository_id = trans.security.encode_id( repository.id )
+        return export_util.export_repository( trans,
+                                              tool_shed_url,
+                                              repository_id,
+                                              str( repository.name ),
+                                              changeset_revision,
+                                              file_type,
+                                              export_repository_dependencies,
+                                              api=True )
 
-    @web.expose_api_anonymous
-    def repository_dependencies( self, trans, id, **kwd ):
-        """
-        GET /api/repository_revisions/{encoded repository_metadata id}/repository_dependencies
-        Displays information about a repository_metadata record in the Tool Shed.
-
-        :param id: the encoded id of the `RepositoryMetadata` object
-        """
-        # Example URL: http://localhost:9009/api/repository_revisions/repository_dependencies/bb125606ff9ea620
+    def __get_value_mapper( self, trans ):
         value_mapper = { 'id' : trans.security.encode_id,
-                         'user_id' : trans.security.encode_id }
-        repository_dependencies_dicts = []
-        try:
-            repository_metadata = metadata_util.get_repository_metadata_by_id( trans, id )
-            metadata = repository_metadata.metadata
-            if metadata and 'repository_dependencies' in metadata:
-                rd_tups = metadata[ 'repository_dependencies' ][ 'repository_dependencies' ]
-                for rd_tup in rd_tups:
-                    tool_shed, name, owner, changeset_revision = rd_tup[ 0:4 ]
-                    repository_dependency = suc.get_repository_by_name_and_owner( trans.app, name, owner )
-                    repository_dependency_dict = repository_dependency.to_dict( view='element', value_mapper=value_mapper )
-                    # We have to add the changeset_revision of of the repository dependency.
-                    repository_dependency_dict[ 'changeset_revision' ] = changeset_revision
-                    repository_dependency_dict[ 'url' ] = web.url_for( controller='repositories',
-                                                                       action='show',
-                                                                       id=trans.security.encode_id( repository_dependency.id ) )
-                    repository_dependencies_dicts.append( repository_dependency_dict )
-            return repository_dependencies_dicts
-        except Exception, e:
-            message = "Error in the Tool Shed repository_revisions API in repository_dependencies: %s" % str( e )
-            log.error( message, exc_info=True )
-            trans.response.status = 500
-            return message
+                         'repository_id' : trans.security.encode_id }
+        return value_mapper
 
     @web.expose_api_anonymous
     def index( self, trans, **kwd ):
@@ -117,59 +84,94 @@ class RepositoryRevisionsController( BaseAPIController ):
         # Filter by downloadable if received.
         downloadable =  kwd.get( 'downloadable', None )
         if downloadable is not None:
-            clause_list.append( trans.model.RepositoryMetadata.table.c.downloadable == util.string_as_bool( downloadable ) )
+            clause_list.append( trans.model.RepositoryMetadata.table.c.downloadable == util.asbool( downloadable ) )
         # Filter by malicious if received.
         malicious =  kwd.get( 'malicious', None )
         if malicious is not None:
-            clause_list.append( trans.model.RepositoryMetadata.table.c.malicious == util.string_as_bool( malicious ) )
+            clause_list.append( trans.model.RepositoryMetadata.table.c.malicious == util.asbool( malicious ) )
         # Filter by tools_functionally_correct if received.
         tools_functionally_correct = kwd.get( 'tools_functionally_correct', None )
         if tools_functionally_correct is not None:
-            clause_list.append( trans.model.RepositoryMetadata.table.c.tools_functionally_correct == util.string_as_bool( tools_functionally_correct ) )
+            clause_list.append( trans.model.RepositoryMetadata.table.c.tools_functionally_correct == util.asbool( tools_functionally_correct ) )
         # Filter by missing_test_components if received.
         missing_test_components = kwd.get( 'missing_test_components', None )
         if missing_test_components is not None:
-            clause_list.append( trans.model.RepositoryMetadata.table.c.missing_test_components == util.string_as_bool( missing_test_components ) )
+            clause_list.append( trans.model.RepositoryMetadata.table.c.missing_test_components == util.asbool( missing_test_components ) )
         # Filter by do_not_test if received.
         do_not_test = kwd.get( 'do_not_test', None )
         if do_not_test is not None:
-            clause_list.append( trans.model.RepositoryMetadata.table.c.do_not_test == util.string_as_bool( do_not_test ) )
+            clause_list.append( trans.model.RepositoryMetadata.table.c.do_not_test == util.asbool( do_not_test ) )
         # Filter by includes_tools if received.
         includes_tools = kwd.get( 'includes_tools', None )
         if includes_tools is not None:
-            clause_list.append( trans.model.RepositoryMetadata.table.c.includes_tools == util.string_as_bool( includes_tools ) )
+            clause_list.append( trans.model.RepositoryMetadata.table.c.includes_tools == util.asbool( includes_tools ) )
         # Filter by test_install_error if received.
         test_install_error = kwd.get( 'test_install_error', None )
         if test_install_error is not None:
-            clause_list.append( trans.model.RepositoryMetadata.table.c.test_install_error == util.string_as_bool( test_install_error ) )
+            clause_list.append( trans.model.RepositoryMetadata.table.c.test_install_error == util.asbool( test_install_error ) )
         # Filter by skip_tool_test if received.
         skip_tool_test = kwd.get( 'skip_tool_test', None )
         if skip_tool_test is not None:
-            skip_tool_test = util.string_as_bool( skip_tool_test )
+            skip_tool_test = util.asbool( skip_tool_test )
             skipped_metadata_ids_subquery = select( [ trans.app.model.SkipToolTest.table.c.repository_metadata_id ] )
             if skip_tool_test:
                 clause_list.append( trans.model.RepositoryMetadata.id.in_( skipped_metadata_ids_subquery ) )
             else:
                 clause_list.append( not_( trans.model.RepositoryMetadata.id.in_( skipped_metadata_ids_subquery ) ) )
-        # Generate and execute the query.
-        try:
-            query = trans.sa_session.query( trans.app.model.RepositoryMetadata ) \
-                                    .filter( and_( *clause_list ) ) \
-                                    .order_by( trans.app.model.RepositoryMetadata.table.c.repository_id.desc() ) \
-                                    .all()
-            for repository_metadata in query:
-                repository_metadata_dict = repository_metadata.to_dict( view='collection',
-                                                                        value_mapper=self.__get_value_mapper( trans, repository_metadata ) )
-                repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions',
-                                                                 action='show',
-                                                                 id=trans.security.encode_id( repository_metadata.id ) )
-                repository_metadata_dicts.append( repository_metadata_dict )
-            return repository_metadata_dicts
-        except Exception, e:
-            message = "Error in the Tool Shed repository_revisions API in index: " + str( e )
-            log.error( message, exc_info=True )
-            trans.response.status = 500
-            return message
+        for repository_metadata in trans.sa_session.query( trans.app.model.RepositoryMetadata ) \
+                                                   .filter( and_( *clause_list ) ) \
+                                                   .order_by( trans.app.model.RepositoryMetadata.table.c.repository_id.desc() ):
+            repository_metadata_dict = repository_metadata.to_dict( view='collection',
+                                                                    value_mapper=self.__get_value_mapper( trans ) )
+            repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions',
+                                                             action='show',
+                                                             id=trans.security.encode_id( repository_metadata.id ) )
+            repository_metadata_dicts.append( repository_metadata_dict )
+        return repository_metadata_dicts
+
+    @web.expose_api_anonymous
+    def repository_dependencies( self, trans, id, **kwd ):
+        """
+        GET /api/repository_revisions/{encoded repository_metadata id}/repository_dependencies
+        Displays information about a repository_metadata record in the Tool Shed.
+
+        :param id: the encoded id of the `RepositoryMetadata` object
+        """
+        # Example URL: http://localhost:9009/api/repository_revisions/repository_dependencies/bb125606ff9ea620
+        repository_dependencies_dicts = []
+        repository_metadata = metadata_util.get_repository_metadata_by_id( trans, id )
+        if repository_metadata is None:
+            log.debug( 'Invalid repository_metadata id received: %s' % str( id ) )
+            return repository_dependencies_dicts
+        metadata = repository_metadata.metadata
+        if metadata is None:
+            log.debug( 'The repository_metadata record with id %s has no metadata.' % str ( id ) )
+            return repository_dependencies_dicts
+        if 'repository_dependencies' in metadata:
+            rd_tups = metadata[ 'repository_dependencies' ][ 'repository_dependencies' ]
+            for rd_tup in rd_tups:
+                tool_shed, name, owner, changeset_revision = rd_tup[ 0:4 ]
+                repository_dependency = suc.get_repository_by_name_and_owner( trans.app, name, owner )
+                if repository_dependency is None:
+                    log.dbug( 'Cannot locate repository dependency %s owned by %s.' % ( name, owner ) )
+                    continue
+                repository_dependency_id = trans.security.encode_id( repository_dependency.id )
+                repository_dependency_repository_metadata = \
+                    suc.get_repository_metadata_by_changeset_revision( trans, repository_dependency_id, changeset_revision )
+                if repository_dependency_repository_metadata is None:
+                    log.debug( 'Cannot locate repository_metadata with id %s for repository dependency %s owned by %s.' % \
+                        ( str( repository_dependency_id ), str( name ), str( owner ) ) )
+                    continue
+                repository_dependency_repository_metadata_id = trans.security.encode_id( repository_dependency_repository_metadata.id )
+                repository_dependency_dict = repository_dependency.to_dict( view='element',
+                                                                            value_mapper=self.__get_value_mapper( trans ) )
+                # We have to add the changeset_revision of of the repository dependency.
+                repository_dependency_dict[ 'changeset_revision' ] = changeset_revision
+                repository_dependency_dict[ 'url' ] = web.url_for( controller='repositories',
+                                                                   action='show',
+                                                                   id=repository_dependency_repository_metadata_id )
+                repository_dependencies_dicts.append( repository_dependency_dict )
+        return repository_dependencies_dicts
 
     @web.expose_api_anonymous
     def show( self, trans, id, **kwd ):
@@ -180,19 +182,16 @@ class RepositoryRevisionsController( BaseAPIController ):
         :param id: the encoded id of the `RepositoryMetadata` object
         """
         # Example URL: http://localhost:9009/api/repository_revisions/bb125606ff9ea620
-        try:
-            repository_metadata = metadata_util.get_repository_metadata_by_id( trans, id )
-            repository_metadata_dict = repository_metadata.to_dict( view='element',
-                                                                    value_mapper=self.__get_value_mapper( trans, repository_metadata ) )
-            repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions',
-                                                             action='show',
-                                                             id=trans.security.encode_id( repository_metadata.id ) )
-            return repository_metadata_dict
-        except Exception, e:
-            message = "Error in the Tool Shed repository_revisions API in show: %s" % str( e )
-            log.error( message, exc_info=True )
-            trans.response.status = 500
-            return message
+        repository_metadata = metadata_util.get_repository_metadata_by_id( trans, id )
+        if repository_metadata is None:
+            log.debug( 'Cannot locate repository_metadata with id %s' % str( id ) )
+            return {}
+        repository_metadata_dict = repository_metadata.to_dict( view='element',
+                                                                value_mapper=self.__get_value_mapper( trans ) )
+        repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions',
+                                                         action='show',
+                                                         id=trans.security.encode_id( repository_metadata.id ) )
+        return repository_metadata_dict
 
     @web.expose_api
     def update( self, trans, payload, **kwd ):
@@ -201,41 +200,32 @@ class RepositoryRevisionsController( BaseAPIController ):
         Updates the value of specified columns of the repository_metadata table based on the key / value pairs in payload.
         """
         repository_metadata_id = kwd.get( 'id', None )
-        try:
-            repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id )
-            flush_needed = False
-            for key, new_value in payload.items():
-                if key == 'time_last_tested':
-                    repository_metadata.time_last_tested = datetime.datetime.utcnow()
-                    flush_needed = True
-                elif hasattr( repository_metadata, key ):
-                    # log information when setting attributes associated with the Tool Shed's install and test framework.
-                    if key in [ 'do_not_test', 'includes_tools', 'missing_test_components', 'test_install_error',
-                                'tools_functionally_correct' ]:
-                        log.debug( 'Setting repository_metadata table column %s to value %s for changeset_revision %s via the Tool Shed API.' % \
-                            ( str( key ), str( new_value ), str( repository_metadata.changeset_revision ) ) )
-                    setattr( repository_metadata, key, new_value )
-                    flush_needed = True
-            if flush_needed:
-                trans.sa_session.add( repository_metadata )
-                trans.sa_session.flush()
-        except Exception, e:
-            message = "Error in the Tool Shed repository_revisions API in update: %s" % str( e )
-            log.error( message, exc_info=True )
-            trans.response.status = 500
-            return message
+        if repository_metadata_id is None:
+            raise HTTPBadRequest( detail="Missing required parameter 'id'." )
+        repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id )
+        if repository_metadata is None:
+            log.debug( 'Cannot locate repository_metadata with id %s' % str( repository_metadata_id ) )
+            return {}
+        flush_needed = False
+        for key, new_value in payload.items():
+            if key == 'time_last_tested':
+                repository_metadata.time_last_tested = datetime.datetime.utcnow()
+                flush_needed = True
+            elif hasattr( repository_metadata, key ):
+                # log information when setting attributes associated with the Tool Shed's install and test framework.
+                if key in [ 'do_not_test', 'includes_tools', 'missing_test_components', 'test_install_error',
+                            'tools_functionally_correct' ]:
+                    log.debug( 'Setting repository_metadata column %s to value %s for changeset_revision %s via the Tool Shed API.' % \
+                        ( str( key ), str( new_value ), str( repository_metadata.changeset_revision ) ) )
+                setattr( repository_metadata, key, new_value )
+                flush_needed = True
+        if flush_needed:
+            trans.sa_session.add( repository_metadata )
+            trans.sa_session.flush()
+            trans.sa_session.refresh( repository_metadata )
         repository_metadata_dict = repository_metadata.to_dict( view='element',
-                                                                value_mapper=self.__get_value_mapper( trans, repository_metadata ) )
+                                                                value_mapper=self.__get_value_mapper( trans ) )
         repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions',
                                                          action='show',
                                                          id=trans.security.encode_id( repository_metadata.id ) )
         return repository_metadata_dict
-
-    def __get_value_mapper( self, trans, repository_metadata ):
-        value_mapper = { 'id' : trans.security.encode_id,
-                         'repository_id' : trans.security.encode_id }
-        if repository_metadata.time_last_tested is not None:
-            # For some reason the Dictifiable.to_dict() method in ~/galaxy/model/item_attrs.py requires
-            # a function rather than a mapped value, so just pass the time_ago function here.
-            value_mapper[ 'time_last_tested' ] = time_ago
-        return value_mapper

lib/galaxy/webapps/tool_shed/controllers/hg.py

@@ -2,6 +2,7 @@ import os, logging
 from galaxy import web
 from galaxy.web.base.controller import BaseUIController
 from tool_shed.util.shed_util_common import get_repository_by_name_and_owner
+from tool_shed.util.shed_util_common import update_repository
 from tool_shed.util.metadata_util import set_repository_metadata
 
 from galaxy import eggs
@@ -9,6 +10,8 @@ eggs.require('mercurial')
 import mercurial.__version__
 from mercurial.hgweb.hgwebdir_mod import hgwebdir
 from mercurial.hgweb.request import wsgiapplication
+from mercurial import hg
+from mercurial import ui
 
 log = logging.getLogger(__name__)
 
@@ -36,6 +39,11 @@ class HgController( BaseUIController ):
                 repository = get_repository_by_name_and_owner( trans.app, name, owner )
                 if repository:
                     if hg_version >= '2.2.3':
+                        # Update the repository on disk to the tip revision, because the web upload form uses the on-disk working
+                        # directory. If the repository is not updated on disk, pushing from the command line and then uploading 
+                        # via the web interface will result in a new head being created.
+                        repo = hg.repository( ui.ui(), repository.repo_path( trans.app ) )
+                        update_repository( repo, ctx_rev=None )
                         # Set metadata using the repository files on disk.
                         error_message, status = set_repository_metadata( trans, repository )
                         if status == 'ok' and error_message:

lib/tool_shed/galaxy_install/install_manager.py

@@ -567,14 +567,17 @@ class InstallManager( object ):
 
     def order_repositories_for_installation( self, tool_shed_repositories, repository_dependencies_dict ):
         """
-        Some repositories may have repository dependencies that are required to be installed before the dependent repository.  This method will
-        inspect the list of repositories about to be installed and make sure to order them appropriately.  For each repository about to be installed,
-        if required repositories are not contained in the list of repositories about to be installed, then they are not considered.  Repository
-        dependency definitions that contain circular dependencies should not result in an infinite loop, but obviously prior installation will not be
-        handled for one or more of the repositories that require prior installation.  This process is similar to the process used when installing tool
-        shed repositories (i.e., the order_components_for_installation() method in ~/lib/tool_shed/galaxy_install/repository_util), but does not handle
-        managing tool panel sections and other components since repository dependency definitions contained in tool shed repositories with migrated
-        tools must never define a relationship to a repository dependency that contains a tool.
+        Some repositories may have repository dependencies that are required to be installed before the dependent
+        repository.  This method will inspect the list of repositories about to be installed and make sure to order
+        them appropriately.  For each repository about to be installed, if required repositories are not contained
+        in the list of repositories about to be installed, then they are not considered.  Repository dependency
+        definitions that contain circular dependencies should not result in an infinite loop, but obviously prior
+        installation will not be handled for one or more of the repositories that require prior installation.  This
+        process is similar to the process used when installing tool shed repositories (i.e., the
+        order_components_for_installation() method in ~/lib/tool_shed/galaxy_install/repository_util), but does not
+        handle managing tool panel sections and other components since repository dependency definitions contained
+        in tool shed repositories with migrated tools must never define a relationship to a repository dependency
+        that contains a tool.
         """
         ordered_tool_shed_repositories = []
         ordered_tsr_ids = []

lib/tool_shed/scripts/api/common.py

@@ -8,28 +8,18 @@ new_path = [ os.path.join( os.path.dirname( __file__ ), '..', '..', '..', '..',
 new_path.extend( sys.path[ 1: ] )
 sys.path = new_path
 
+import tool_shed.util.shed_util_common as suc
+
 from galaxy import eggs
 import pkg_resources
 
-pkg_resources.require( "pycrypto" )
-from Crypto.Cipher import Blowfish
-from Crypto.Util.randpool import RandomPool
-from Crypto.Util import number
-
-def encode_id( config_id_secret, obj_id ):
-    # Utility method to encode ID's
-    id_cipher = Blowfish.new( config_id_secret )
-    # Convert to string
-    s = str( obj_id )
-    # Pad to a multiple of 8 with leading "!"
-    s = ( "!" * ( 8 - len(s) % 8 ) ) + s
-    # Encrypt
-    return id_cipher.encrypt( s ).encode( 'hex' )
-
 def delete( api_key, url, data, return_formatted=True ):
-    # Sends an API DELETE request and acts as a generic formatter for the JSON response - 'data' will become the JSON payload read by Galaxy.
+    """
+    Sends an API DELETE request and acts as a generic formatter for the JSON response.  The
+    'data' will become the JSON payload read by the Tool Shed.
+    """
     try:
-        url = make_url( api_key, url )
+        url = make_url( url, api_key=api_key, args=None )
         req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = json.dumps( data ))
         req.get_method = lambda: 'DELETE'
         r = json.loads( urllib2.urlopen( req ).read() )
@@ -47,12 +37,13 @@ def delete( api_key, url, data, return_formatted=True ):
     print r
 
 def display( url, api_key=None, return_formatted=True ):
-    # Sends an API GET request and acts as a generic formatter for the JSON response.
+    """Sends an API GET request and acts as a generic formatter for the JSON response."""
     try:
         r = get( url, api_key=api_key )
     except urllib2.HTTPError, e:
         print e
-        print e.read( 1024 ) # Only return the first 1K of errors.
+        # Only return the first 1K of errors.
+        print e.read( 1024 )
         sys.exit( 1 )
     if type( r ) == unicode:
         print 'error: %s' % r
@@ -84,16 +75,94 @@ def display( url, api_key=None, return_formatted=True ):
         print 'response is unknown type: %s' % type( r )
 
 def get( url, api_key=None ):
-    # Do the actual GET.
-    url = make_url( url, api_key=api_key )
+    """Do the GET."""
+    url = make_url( url, api_key=api_key, args=None )
     try:
         return json.loads( urllib2.urlopen( url ).read() )
     except ValueError, e:
         print "URL did not return JSON data"
-        sys.exit(1)
+        sys.exit( 1 )
+
+def get_api_url( base, parts=[], params=None ):
+    """Compose and return a URL for the Tool Shed API."""
+    if 'api' in parts and parts.index( 'api' ) != 0:
+        parts.pop( parts.index( 'api' ) )
+        parts.insert( 0, 'api' )
+    elif 'api' not in parts:
+        parts.insert( 0, 'api' )
+    url = suc.url_join( base, *parts )
+    if params is not None:
+        try:
+            query_string = urllib.urlencode( params )
+        except Exception, e:
+            # The value of params must be a string.
+            query_string = params
+        url += '?%s' % query_string
+    return url
+
+def get_latest_downloadable_changeset_revision_via_api( url, name, owner ):
+    """
+    Return the latest downloadable changeset revision for the repository defined by the received
+    name and owner.
+    """
+    error_message = ''
+    parts = [ 'api', 'repositories', 'get_ordered_installable_revisions' ]
+    params = dict( name=name, owner=owner )
+    api_url = get_api_url( base=url, parts=parts, params=params )
+    changeset_revisions, error_message = json_from_url( api_url )
+    if changeset_revisions is None or error_message:
+        return None, error_message
+    if len( changeset_revisions ) >= 1:
+        return changeset_revisions[ -1 ], error_message
+    return suc.INITIAL_CHANGELOG_HASH, error_message
+
+def get_repository_dict( url, repository_dict ):
+    """
+    Send a request to the Tool Shed to get additional information about the repository defined
+    by the received repository_dict.  Add the information to the repository_dict and return it.
+    """
+    error_message = ''
+    if not isinstance( repository_dict, dict ):
+        error_message = 'Invalid repository_dict received: %s' % str( repository_dict )
+        return None, error_message
+    repository_id = repository_dict.get( 'repository_id', None )
+    if repository_id is None:
+        error_message = 'Invalid repository_dict does not contain a repository_id entry: %s' % str( repository_dict )
+        return None, error_message
+    parts = [ 'api', 'repositories', repository_id ]
+    api_url = get_api_url( base=url, parts=parts )
+    extended_dict, error_message = json_from_url( api_url )
+    if extended_dict is None or error_message:
+        return None, error_message
+    name = extended_dict.get( 'name', None )
+    owner = extended_dict.get( 'owner', None )
+    if name is not None and owner is not None:
+        name = str( name )
+        owner = str( owner )
+        latest_changeset_revision, error_message = get_latest_downloadable_changeset_revision_via_api( url, name, owner )
+        if latest_changeset_revision is None or error_message:
+            return None, error_message
+        extended_dict[ 'latest_revision' ] = str( latest_changeset_revision )
+        return extended_dict, error_message
+    else:
+        error_message = 'Invalid extended_dict does not contain name or woner entries: %s' % str( extended_dict )
+        return None, error_message
+
+def json_from_url( url ):
+    """Send a request to the Tool Shed via the Tool Shed API and handle the response."""
+    error_message = ''
+    url_handle = urllib.urlopen( url )
+    url_contents = url_handle.read()
+    try:
+        parsed_json = simplejson.loads( url_contents )
+    except Exception, e:
+        error_message = str( url_contents )
+        print 'Error parsing JSON data in json_from_url(): ', str( e )
+        return None, error_message
+    return parsed_json, error_message
 
 def make_url( url, api_key=None, args=None ):
-    # Adds the API Key to the URL if it's not already there.
+    """Adds the API Key to the URL if it's not already there."""
     if args is None:
         args = []
     argsep = '&'
@@ -105,20 +174,23 @@ def make_url( url, api_key=None, args=None ):
     return url + argsep + '&'.join( [ '='.join( t ) for t in args ] )
 
 def post( url, data, api_key=None ):
-    # Do the actual POST.
-    url = make_url( url, api_key=api_key )
+    """Do the POST."""
+    url = make_url( url, api_key=api_key, args=None )
     req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = json.dumps( data ) )
     return json.loads( urllib2.urlopen( req ).read() )
 
 def put( url, data, api_key=None ):
-    # Do the actual PUT.
-    url = make_url( url, api_key=api_key )
+    """Do the PUT."""
+    url = make_url( url, api_key=api_key, args=None )
     req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = json.dumps( data ))
     req.get_method = lambda: 'PUT'
     return json.loads( urllib2.urlopen( req ).read() )
 
 def submit( url, data, api_key=None, return_formatted=True ):
-    # Sends an API POST request and acts as a generic formatter for the JSON response - 'data' will become the JSON payload read by Galaxy.
+    """
+    Sends an API POST request and acts as a generic formatter for the JSON response.  The
+    'data' will become the JSON payload read by the Tool Shed.
+    """
     try:
         r = post( url, data, api_key=api_key )
     except urllib2.HTTPError, e:
@@ -133,7 +205,8 @@ def submit( url, data, api_key=None, return_formatted=True ):
     print 'Response'
     print '--------'
     if type( r ) == list:
-        # Currently the only implemented responses are lists of dicts, because submission creates some number of collection elements.
+        # Currently the only implemented responses are lists of dicts, because submission creates
+        # some number of collection elements.
         for i in r:
             if type( i ) == dict:
                 if 'url' in i:
@@ -150,9 +223,12 @@ def submit( url, data, api_key=None, return_formatted=True ):
         print r
 
 def update( api_key, url, data, return_formatted=True ):
-    # Sends an API PUT request and acts as a generic formatter for the JSON response - 'data' will become the JSON payload read by Galaxy.
+    """
+    Sends an API PUT request and acts as a generic formatter for the JSON response.  The
+    'data' will become the JSON payload read by the Tool Shed.
+    """
     try:
-        r = put( api_key, url, data )
+        r = put( url, data, api_key=api_key )
     except urllib2.HTTPError, e:
         if return_formatted:
             print e

lib/tool_shed/scripts/api/get_filtered_repository_revisions.py

@@ -26,69 +26,19 @@ import sys
 import urllib
 
 sys.path.insert( 0, os.path.dirname( __file__ ) )
+
 from common import get
+from common import get_api_url
+from common import get_repository_dict
+from common import json_from_url
+
+from galaxy.util import asbool
 from galaxy.util.json import from_json_string
 import tool_shed.util.shed_util_common as suc
 
-def get_api_url( base, parts=[], params=None ):
-    if 'api' in parts and parts.index( 'api' ) != 0:
-        parts.pop( parts.index( 'api' ) )
-        parts.insert( 0, 'api' )
-    elif 'api' not in parts:
-        parts.insert( 0, 'api' )
-    url = suc.url_join( base, *parts )
-    if params:
-        url += '?%s' % params
-    return url
-
-def get_latest_downloadable_changeset_revision( url, name, owner ):
-    error_message = ''
-    parts = [ 'api', 'repositories', 'get_ordered_installable_revisions' ]
-    params = urllib.urlencode( dict( name=name, owner=owner ) )
-    api_url = get_api_url( base=url, parts=parts, params=params )
-    changeset_revisions, error_message = json_from_url( api_url )
-    if error_message:
-        return None, error_message
-    if changeset_revisions:
-        return changeset_revisions[ -1 ], error_message
-    else:
-        return suc.INITIAL_CHANGELOG_HASH, error_message
-
-def get_repository_dict( url, repository_dict ):
-    error_message = ''
-    parts = [ 'api', 'repositories', repository_dict[ 'repository_id' ] ]
-    api_url = get_api_url( base=url, parts=parts )
-    extended_dict, error_message = json_from_url( api_url )
-    if error_message:
-        return None, error_message
-    name = str( extended_dict[ 'name' ] )
-    owner = str( extended_dict[ 'owner' ] )
-    latest_changeset_revision, error_message = get_latest_downloadable_changeset_revision( url, name, owner )
-    if error_message:
-        print error_message
-    extended_dict[ 'latest_revision' ] = str( latest_changeset_revision )
-    return extended_dict, error_message
-
-def json_from_url( url ):
-    error_message = ''
-    url_handle = urllib.urlopen( url )
-    url_contents = url_handle.read()
-    try:
-        parsed_json = from_json_string( url_contents )
-    except Exception, e:
-        error_message = str( url_contents )
-        return None, error_message
-    return parsed_json, error_message
-
-def string_as_bool( string ):
-    if str( string ).lower() in [ 'true' ]:
-        return True
-    else:
-        return False
-
 def main( options ):
     base_tool_shed_url = options.tool_shed_url.rstrip( '/' )
-    latest_revision_only = string_as_bool( options.latest_revision_only )
+    latest_revision_only = asbool( options.latest_revision_only )
     do_not_test = str( options.do_not_test )
     downloadable = str( options.downloadable )
     includes_tools = str( options.includes_tools )
@@ -108,30 +58,31 @@ def main( options ):
                                      tools_functionally_correct=tools_functionally_correct ) )
     api_url = get_api_url( base=base_tool_shed_url, parts=parts, params=params )
     baseline_repository_dicts, error_message = json_from_url( api_url )
-    if error_message:
+    if baseline_repository_dicts is None or error_message:
         print error_message
-    repository_dicts = []
-    for baseline_repository_dict in baseline_repository_dicts:
-        # We need to get some details from the tool shed API, such as repository name and owner, to pass on to the
-        # module that will generate the install methods.
-        repository_dict, error_message = get_repository_dict( base_tool_shed_url, baseline_repository_dict )
-        if error_message:
-            print 'Error getting additional details from the API: ', error_message
-            repository_dicts.append( baseline_repository_dict )
-        else:
-            # Don't test empty repositories.
-            changeset_revision = baseline_repository_dict[ 'changeset_revision' ]
-            if changeset_revision != suc.INITIAL_CHANGELOG_HASH:
-                # Merge the dictionary returned from /api/repository_revisions with the detailed repository_dict and
-                # append it to the list of repository_dicts to install and test.
-                if latest_revision_only:
-                    latest_revision = repository_dict[ 'latest_revision' ]
-                    if changeset_revision == latest_revision:
+    else:
+        repository_dicts = []
+        for baseline_repository_dict in baseline_repository_dicts:
+            # We need to get additional details from the tool shed API to pass on to the
+            # module that will generate the install methods.
+            repository_dict, error_message = get_repository_dict( base_tool_shed_url, baseline_repository_dict )
+            if error_message:
+                print 'Error getting additional details from the API: ', error_message
+                repository_dicts.append( baseline_repository_dict )
+            else:
+                # Don't test empty repositories.
+                changeset_revision = baseline_repository_dict.get( 'changeset_revision', suc.INITIAL_CHANGELOG_HASH )
+                if changeset_revision != suc.INITIAL_CHANGELOG_HASH:
+                    # Merge the dictionary returned from /api/repository_revisions with the detailed repository_dict and
+                    # append it to the list of repository_dicts to install and test.
+                    if latest_revision_only:
+                        latest_revision = repository_dict.get( 'latest_revision', suc.INITIAL_CHANGELOG_HASH )
+                        if changeset_revision == latest_revision:
+                            repository_dicts.append( dict( repository_dict.items() + baseline_repository_dict.items() ) )
+                    else:
                         repository_dicts.append( dict( repository_dict.items() + baseline_repository_dict.items() ) )
-                else:
-                    repository_dicts.append( dict( repository_dict.items() + baseline_repository_dict.items() ) )
-    print '\n\n', repository_dicts
-    print '\nThe url:\n\n', api_url, '\n\nreturned ', len( repository_dicts ), ' repository dictionaries...'
+        print '\n\n', repository_dicts
+        print '\nThe url:\n\n', api_url, '\n\nreturned ', len( repository_dicts ), ' repository dictionaries...'
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser( description='Get a filtered list of repository dictionaries.' )

lib/tool_shed/util/commit_util.py

@@ -313,6 +313,9 @@ def handle_repository_dependency_elem( trans, elem, unpopulate=False ):
             if lastest_installable_changeset_revision != suc.INITIAL_CHANGELOG_HASH:
                 elem.attrib[ 'changeset_revision' ] = lastest_installable_changeset_revision
                 revised = True
+            else:
+                error_message = 'Invalid latest installable changeset_revision %s ' % str( lastest_installable_changeset_revision )
+                error_message += 'retrieved for repository %s owned by %s.  ' % ( str( name ), str( owner ) )
         else:
             error_message = 'Unable to locate repository with name %s and owner %s.  ' % ( str( name ), str( owner ) )
     return revised, elem, error_message

lib/tool_shed/util/container_util.py

@@ -744,7 +744,8 @@ def build_repository_containers_for_galaxy( trans, repository, datatypes, invali
         lock.release()
     return containers_dict
 
-def build_repository_containers_for_tool_shed( trans, repository, changeset_revision, repository_dependencies, repository_metadata, exclude=None ):
+def build_repository_containers_for_tool_shed( trans, repository, changeset_revision, repository_dependencies, repository_metadata,
+                                               exclude=None ):
     """Return a dictionary of containers for the received repository's dependencies and contents for display in the tool shed."""
     if exclude is None:
         exclude = []
@@ -806,11 +807,12 @@ def build_repository_containers_for_tool_shed( trans, repository, changeset_revi
                     containers_dict[ 'readme_files' ] = readme_files_root_folder
             if 'repository_dependencies' not in exclude:
                 # Repository dependencies container.
-                folder_id, repository_dependencies_root_folder = build_repository_dependencies_folder( trans=trans,
-                                                                                                       folder_id=folder_id,
-                                                                                                       repository_dependencies=repository_dependencies,
-                                                                                                       label='Repository dependencies',
-                                                                                                       installed=False )
+                folder_id, repository_dependencies_root_folder = \
+                    build_repository_dependencies_folder( trans=trans,
+                                                          folder_id=folder_id,
+                                                          repository_dependencies=repository_dependencies,
+                                                          label='Repository dependencies',
+                                                          installed=False )
                 if repository_dependencies_root_folder:
                     containers_dict[ 'repository_dependencies' ] = repository_dependencies_root_folder
             # Tool dependencies container.
@@ -868,11 +870,13 @@ def build_repository_containers_for_tool_shed( trans, repository, changeset_revi
             if metadata:
                 if 'data_manager' not in exclude and 'data_manager' in metadata:
                     data_managers = metadata['data_manager'].get( 'data_managers', None )
-                    folder_id, data_managers_root_folder = build_data_managers_folder( trans, folder_id, data_managers, label="Data Managers" )
+                    folder_id, data_managers_root_folder = \
+                        build_data_managers_folder( trans, folder_id, data_managers, label="Data Managers" )
                     containers_dict[ 'valid_data_managers' ] = data_managers_root_folder
                     error_messages = metadata['data_manager'].get( 'error_messages', None )
                     data_managers = metadata['data_manager'].get( 'invalid_data_managers', None )
-                    folder_id, data_managers_root_folder = build_invalid_data_managers_folder( trans, folder_id, data_managers, error_messages, label="Invalid Data Managers" )
+                    folder_id, data_managers_root_folder = \
+                        build_invalid_data_managers_folder( trans, folder_id, data_managers, error_messages, label="Invalid Data Managers" )
                     containers_dict[ 'invalid_data_managers' ] = data_managers_root_folder
         except Exception, e:
             log.exception( "Exception in build_repository_containers_for_tool_shed: %s" % str( e ) )
@@ -890,15 +894,22 @@ def build_repository_dependencies_folder( trans, folder_id, repository_dependenc
         folder_id += 1
         # Create the Repository dependencies folder and add it to the root folder.
         repository_dependencies_folder_key = repository_dependencies[ 'root_key' ]
-        repository_dependencies_folder = Folder( id=folder_id, key=repository_dependencies_folder_key, label=label, parent=repository_dependencies_root_folder )
+        repository_dependencies_folder = Folder( id=folder_id,
+                                                 key=repository_dependencies_folder_key,
+                                                 label=label,
+                                                 parent=repository_dependencies_root_folder )
         del repository_dependencies[ 'root_key' ]
-        # The received repository_dependencies is a dictionary with keys: 'root_key', 'description', and one or more repository_dependency keys.
-        # We want the description value associated with the repository_dependencies_folder.
+        # The received repository_dependencies is a dictionary with keys: 'root_key', 'description', and one or more
+        # repository_dependency keys.  We want the description value associated with the repository_dependencies_folder.
         repository_dependencies_folder.description = repository_dependencies.get( 'description', None )
         repository_dependencies_root_folder.folders.append( repository_dependencies_folder )
         del repository_dependencies[ 'description' ]
         repository_dependencies_folder, folder_id, repository_dependency_id = \
-            populate_repository_dependencies_container( trans, repository_dependencies_folder, repository_dependencies, folder_id, repository_dependency_id )
+            populate_repository_dependencies_container( trans,
+                                                        repository_dependencies_folder,
+                                                        repository_dependencies,
+                                                        folder_id,
+                                                        repository_dependency_id )
         repository_dependencies_folder = prune_repository_dependencies( repository_dependencies_folder )
     else:
         repository_dependencies_root_folder = None
@@ -985,11 +996,13 @@ def build_tools_folder( trans, folder_id, tool_dicts, repository, changeset_revi
         tools_root_folder = None
     return folder_id, tools_root_folder
 
-def build_tool_dependencies_folder( trans, folder_id, tool_dependencies, label='Tool dependencies', missing=False, new_install=False, reinstalling=False ):
+def build_tool_dependencies_folder( trans, folder_id, tool_dependencies, label='Tool dependencies', missing=False,
+                                    new_install=False, reinstalling=False ):
     """Return a folder hierarchy containing tool dependencies."""
-    # When we're in Galaxy (not the tool shed) and the tool dependencies are not installed or are in an error state, they are considered missing.  The tool
-    # dependency status will be displayed only if a record exists for the tool dependency in the Galaxy database, but the tool dependency is not installed.
-    # The value for new_install will be True only if the associated repository in being installed for the first time.  This value is used in setting the
+    # When we're in Galaxy (not the tool shed) and the tool dependencies are not installed or are in an error state,
+    # they are considered missing.  The tool dependency status will be displayed only if a record exists for the tool
+    # dependency in the Galaxy database, but the tool dependency is not installed.  The value for new_install will be
+    # True only if the associated repository in being installed for the first time.  This value is used in setting the
     # container description.
     if tool_dependencies:
         tool_dependency_id = 0
@@ -1138,7 +1151,10 @@ def build_tool_test_results_folder( trans, folder_id, tool_test_results_dicts, l
             else:
                 containing_folder = tool_test_results_root_folder
             folder_id += 1
-            test_environment_folder = Folder( id=folder_id, key='test_environment', label='Automated test environment', parent=containing_folder )
+            test_environment_folder = Folder( id=folder_id,
+                                              key='test_environment',
+                                              label='Automated test environment',
+                                              parent=containing_folder )
             containing_folder.folders.append( test_environment_folder )
             try:
                 architecture = test_environment_dict.get( 'architecture', '' )
@@ -1184,7 +1200,10 @@ def build_tool_test_results_folder( trans, folder_id, tool_test_results_dicts, l
             passed_tests_dicts = tool_test_results_dict.get( 'passed_tests', [] )
             if len( passed_tests_dicts ) > 0:
                 folder_id += 1
-                passed_tests_folder = Folder( id=folder_id, key='passed_tests', label='Tests that passed successfully', parent=containing_folder )
+                passed_tests_folder = Folder( id=folder_id,
+                                              key='passed_tests',
+                                              label='Tests that passed successfully',
+                                              parent=containing_folder )
                 containing_folder.folders.append( passed_tests_folder )
                 passed_test_id = 0
                 for passed_tests_dict in passed_tests_dicts:
@@ -1395,7 +1414,8 @@ def build_tool_test_results_folder( trans, folder_id, tool_test_results_dicts, l
                                 rd_tool_shed = str( repository_dependency_successful_installation_dict.get( 'tool_shed', '' ) )
                                 rd_name = str( repository_dependency_successful_installation_dict.get( 'name', '' ) )
                                 rd_owner = str( repository_dependency_successful_installation_dict.get( 'owner', '' ) )
-                                rd_changeset_revision = str( repository_dependency_successful_installation_dict.get( 'changeset_revision', '' ) )
+                                rd_changeset_revision = \
+                                    str( repository_dependency_successful_installation_dict.get( 'changeset_revision', '' ) )
                             except Exception, e:
                                 rd_tool_shed = 'unknown'
                                 rd_name = 'unknown'
@@ -1495,7 +1515,8 @@ def can_display_tool_test_results( tool_test_results_dicts, exclude=None ):
         # only the "test_environment" entry, but we want at least 1 of "passed_tests", "failed_tests",
         # "installation_errors", "missing_test_components" "skipped_tests", "not_tested" or any other
         # entry that may be added in the future.
-        display_entries = [ 'failed_tests', 'installation_errors', 'missing_test_components', 'not_tested', 'passed_tests', 'skipped_tests' ]
+        display_entries = [ 'failed_tests', 'installation_errors', 'missing_test_components',
+                            'not_tested', 'passed_tests', 'skipped_tests' ]
         for k, v in tool_test_results_dict.items():
             if k in display_entries:
                 # We've discovered an entry that can be displayed, so see if it has a value since displaying
@@ -1506,8 +1527,9 @@ def can_display_tool_test_results( tool_test_results_dicts, exclude=None ):
 
 def cast_empty_repository_dependency_folders( folder, repository_dependency_id ):
     """
-    Change any empty folders contained within the repository dependencies container into a repository dependency since it has no repository dependencies
-    of it's own.  This method is not used (and may not be needed), but here it is just in case.
+    Change any empty folders contained within the repository dependencies container into a repository dependency
+    since it has no repository dependencies of it's own.  This method is not used (and may not be needed), but here
+    it is just in case.
     """
     if not folder.folders and not folder.repository_dependencies:
         repository_dependency_id += 1
@@ -1519,10 +1541,14 @@ def cast_empty_repository_dependency_folders( folder, repository_dependency_id )
         return cast_empty_repository_dependency_folders( sub_folder, repository_dependency_id )
     return folder, repository_dependency_id
 
-def generate_repository_dependencies_folder_label_from_key( repository_name, repository_owner, changeset_revision, prior_installation_required,
-                                                            only_if_compiling_contained_td, key ):
+def generate_repository_dependencies_folder_label_from_key( repository_name, repository_owner, changeset_revision,
+                                                            prior_installation_required, only_if_compiling_contained_td, key ):
     """Return a repository dependency label based on the repository dependency key."""
-    if key_is_current_repositorys_key( repository_name, repository_owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, key ):
+    if key_is_current_repositorys_key( repository_name,
+                                       repository_owner,
+                                       changeset_revision,
+                                       prior_installation_required,
+                                       only_if_compiling_contained_td, key ):
         label = 'Repository dependencies'
     else:
         if galaxy.util.asbool( prior_installation_required ):
@@ -1533,8 +1559,8 @@ def generate_repository_dependencies_folder_label_from_key( repository_name, rep
             ( repository_name, changeset_revision, repository_owner, prior_installation_required_str )
     return label
 
-def generate_repository_dependencies_key_for_repository( toolshed_base_url, repository_name, repository_owner, changeset_revision, prior_installation_required,
-                                                         only_if_compiling_contained_td ):
+def generate_repository_dependencies_key_for_repository( toolshed_base_url, repository_name, repository_owner, changeset_revision,
+                                                         prior_installation_required, only_if_compiling_contained_td ):
     """Assumes tool shed is current tool shed since repository dependencies across tool sheds is not yet supported."""
     return '%s%s%s%s%s%s%s%s%s%s%s' % ( str( toolshed_base_url ).rstrip( '/' ),
                                   STRSEP,
@@ -1551,13 +1577,6 @@ def generate_repository_dependencies_key_for_repository( toolshed_base_url, repo
 def generate_tool_dependencies_key( name, version, type ):
     return '%s%s%s%s%s' % ( str( name ), STRSEP, str( version ), STRSEP, str( type ) )
 
-def get_folder( folder, key ):
-    if folder.key == key:
-        return folder
-    for sub_folder in folder.folders:
-        return get_folder( sub_folder, key )
-    return None
-
 def get_components_from_repository_dependency_for_installed_repository( trans, repository_dependency ):
     """Parse a repository dependency and return components necessary for proper display in Galaxy on the Manage repository page."""
     # Default prior_installation_required and only_if_compiling_contained_td to False.
@@ -1610,7 +1629,15 @@ def get_components_from_key( key ):
         # For backward compatibility to the 12/20/12 Galaxy release we have to return the following, and callers must handle exceptions.
         return toolshed_base_url, repository_name, repository_owner, changeset_revision
 
-def handle_repository_dependencies_container_entry( trans, repository_dependencies_folder, rd_key, rd_value, folder_id, repository_dependency_id, folder_keys ):
+def get_folder( folder, key ):
+    if folder.key == key:
+        return folder
+    for sub_folder in folder.folders:
+        return get_folder( sub_folder, key )
+    return None
+
+def handle_repository_dependencies_container_entry( trans, repository_dependencies_folder, rd_key, rd_value, folder_id,
+                                                    repository_dependency_id, folder_keys ):
     repository_components_tuple = get_components_from_key( rd_key )
     components_list = suc.extract_components_from_tuple( repository_components_tuple )
     toolshed, repository_name, repository_owner, changeset_revision = components_list[ 0:4 ]
@@ -1713,11 +1740,21 @@ def key_is_current_repositorys_key( repository_name, repository_owner, changeset
         return True
     return False
 
-def populate_repository_dependencies_container( trans, repository_dependencies_folder, repository_dependencies, folder_id, repository_dependency_id ):
-    folder_keys = repository_dependencies.keys()
+def populate_repository_dependencies_container( trans, repository_dependencies_folder, repository_dependencies, folder_id,
+                                                repository_dependency_id ):
+    folder_keys = []
+    for key in repository_dependencies.keys():
+        if key not in folder_keys:
+            folder_keys.append( key )
     for key, value in repository_dependencies.items():
         repository_dependencies_folder, folder_id, repository_dependency_id = \
-            handle_repository_dependencies_container_entry( trans, repository_dependencies_folder, key, value, folder_id, repository_dependency_id, folder_keys )
+            handle_repository_dependencies_container_entry( trans,
+                                                            repository_dependencies_folder,
+                                                            key,
+                                                            value,
+                                                            folder_id,
+                                                            repository_dependency_id,
+                                                            folder_keys )
     return repository_dependencies_folder, folder_id, repository_dependency_id
 
 def print_folders( pad, folder ):
@@ -1731,19 +1768,22 @@ def print_folders( pad, folder ):
     for sub_folder in folder.folders:
         print_folders( pad+5, sub_folder )
 
+def prune_folder( folder, repository_dependency ):
+    listified_repository_dependency = repository_dependency.listify
+    if is_subfolder_of( folder, listified_repository_dependency ):
+        folder.repository_dependencies.remove( repository_dependency )
+
 def prune_repository_dependencies( folder ):
     """
-    Since the object used to generate a repository dependencies container is a dictionary and not an odict() (it must be json-serialize-able), the
-    order in which the dictionary is processed to create the container sometimes results in repository dependency entries in a folder that also
-    includes the repository dependency as a sub-folder (if the repository dependency has it's own repository dependency).  This method will remove
-    all repository dependencies from folder that are also sub-folders of folder.
+    Since the object used to generate a repository dependencies container is a dictionary and not an odict() (it must be
+    json-serialize-able), the order in which the dictionary is processed to create the container sometimes results in
+    repository dependency entries in a folder that also includes the repository dependency as a sub-folder (if the
+    repository dependency has it's own repository dependency).  This method will remove all repository dependencies from
+    folder that are also sub-folders of folder.
     """
     repository_dependencies = [ rd for rd in folder.repository_dependencies ]
     for repository_dependency in repository_dependencies:
-        listified_repository_dependency = repository_dependency.listify
-        if is_subfolder_of( folder, listified_repository_dependency ):
-            repository_dependencies.remove( repository_dependency )
-    folder.repository_dependencies = repository_dependencies
+        prune_folder( folder, repository_dependency )
     for sub_folder in folder.folders:
         return prune_repository_dependencies( sub_folder )
     return folder

lib/tool_shed/util/export_util.py

@@ -55,7 +55,8 @@ def clean_tool_shed_url( base_url ):
     base = base.rstrip( '/' )
     return base
 
-def export_repository( trans, tool_shed_url, repository_id, repository_name, changeset_revision, file_type, export_repository_dependencies, api=False ):
+def export_repository( trans, tool_shed_url, repository_id, repository_name, changeset_revision, file_type,
+                       export_repository_dependencies, api=False ):
     repository = suc.get_repository_in_tool_shed( trans, repository_id )
     repositories_archive_filename = generate_repository_archive_filename( tool_shed_url,
                                                                           str( repository.name ),
@@ -67,7 +68,8 @@ def export_repository( trans, tool_shed_url, repository_id, repository_name, cha
     if export_repository_dependencies:
         repo_info_dicts = get_repo_info_dicts( trans, tool_shed_url, repository_id, changeset_revision )
         repository_ids = get_repository_ids( trans, repo_info_dicts )
-        ordered_repository_ids, ordered_repositories, ordered_changeset_revisions = order_components_for_import( trans, repository_ids, repo_info_dicts )
+        ordered_repository_ids, ordered_repositories, ordered_changeset_revisions = \
+            order_components_for_import( trans, repository_id, repository_ids, repo_info_dicts )
     else:
         ordered_repository_ids = []
         ordered_repositories = []
@@ -193,8 +195,8 @@ def generate_export_elem( tool_shed_url, repository, changeset_revision, export_
 
 def get_components_from_repo_info_dict( trans, repo_info_dict ):
     """
-    Return the repository and the associated latest installable changeset_revision (including updates) for the repository defined by the received
-    repo_info_dict.
+    Return the repository and the associated latest installable changeset_revision (including updates) for the
+    repository defined by the received repo_info_dict.
     """
     for repository_name, repo_info_tup in repo_info_dict.items():
         # There should only be one entry in the received repo_info_dict.
@@ -208,8 +210,9 @@ def get_components_from_repo_info_dict( trans, repo_info_dict ):
 
 def get_repo_info_dict_for_import( encoded_repository_id, encoded_repository_ids, repo_info_dicts ):
     """
-    The received encoded_repository_ids and repo_info_dicts are lists that contain associated elements at each location in the list.  This method will return the element
-    from repo_info_dicts associated with the received encoded_repository_id by determining it's location in the received encoded_repository_ids list.
+    The received encoded_repository_ids and repo_info_dicts are lists that contain associated elements at each
+    location in the list.  This method will return the element from repo_info_dicts associated with the received
+    encoded_repository_id by determining it's location in the received encoded_repository_ids list.
     """
     for index, repository_id in enumerate( encoded_repository_ids ):
         if repository_id == encoded_repository_id:
@@ -218,6 +221,10 @@ def get_repo_info_dict_for_import( encoded_repository_id, encoded_repository_ids
     return None
 
 def get_repo_info_dicts( trans, tool_shed_url, repository_id, changeset_revision ):
+    """
+    Return a list of dictionaries defining repositories that are required by the repository associated with the
+    received repository_id.
+    """
     repository = suc.get_repository_in_tool_shed( trans, repository_id )
     repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision )
     # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend.
@@ -246,8 +253,9 @@ def get_repo_info_dicts( trans, tool_shed_url, repository_id, changeset_revision
 
 def get_repository_attributes_and_sub_elements( repository, archive_name ):
     """
-    Get the information about a repository to create and populate an XML tag set.  The generated attributes will be contained within the <repository>
-    tag, while the sub_elements will be tag sets contained within the <repository> tag set.
+    Get the information about a repository to create and populate an XML tag set.  The generated attributes will
+    be contained within the <repository> tag, while the sub_elements will be tag sets contained within the <repository>
+    tag set.
     """
     attributes = odict()
     sub_elements = odict()
@@ -268,6 +276,7 @@ def get_repository_attributes_and_sub_elements( repository, archive_name ):
     return attributes, sub_elements
 
 def get_repository_ids( trans, repo_info_dicts ):
+    """Return a list of repository ids associated with each dictionary in the received repo_info_dicts."""
     repository_ids = []
     for repo_info_dict in repo_info_dicts:
         for repository_name, repo_info_tup in repo_info_dict.items():
@@ -277,23 +286,36 @@ def get_repository_ids( trans, repo_info_dicts ):
             repository_ids.append( trans.security.encode_id( repository.id ) )
     return repository_ids
 
-def order_components_for_import( trans, repository_ids, repo_info_dicts ):
+def order_components_for_import( trans, primary_repository_id, repository_ids, repo_info_dicts ):
     """
-    Some repositories may have repository dependencies that must be imported and have metadata set on them before the dependent repository is imported.  This method
-    will inspect the list of repositories about to be exported and make sure to order them appropriately for proper import.  For each repository about to be exported,
-    if required repositories are not contained in the list of repositories about to be exported, then they are not considered.  Repository dependency definitions that
-    contain circular dependencies should not result in an infinite loop, but obviously ordering the list will not be handled for one or more of the repositories that
-    require prior import.
+    Some repositories may have repository dependencies that must be imported and have metadata set on
+    them before the dependent repository is imported.  This method will inspect the list of repositories
+    about to be exported and make sure to order them appropriately for proper import.  For each repository
+    about to be exported, if required repositories are not contained in the list of repositories about to
+    be exported, then they are not considered.  Repository dependency definitions that contain circular
+    dependencies should not result in an infinite loop, but obviously ordering the list will not be handled
+    for one or more of the repositories that require prior import.
     """
+    # The received primary_repository_id is the id of the repository being exported, with the received list
+    # of repository_ids being only the ids of all of its repository dependencies.  The primary repository will
+    # always be last in the returned lists.
     ordered_repository_ids = []
     ordered_repositories = []
     ordered_changeset_revisions = []
-    # Create a dictionary whose keys are the received repository_ids and whose values are a list of repository_ids, each of which is contained in the received list of
-    # repository_ids and whose associated repository must be imported prior to the repository associated with the repository_id key.
+    # Create a dictionary whose keys are the received repository_ids and whose values are a list of
+    # repository_ids, each of which is contained in the received list of repository_ids and whose associated
+    # repository must be imported prior to the repository associated with the repository_id key.
     prior_import_required_dict = suc.get_prior_import_or_install_required_dict( trans, repository_ids, repo_info_dicts )
     processed_repository_ids = []
+    # Process the list of repository dependencies defined for the repository associated with the received
+    # primary_repository_id.
     while len( processed_repository_ids ) != len( prior_import_required_dict.keys() ):
         repository_id = suc.get_next_prior_import_or_install_required_dict_entry( prior_import_required_dict, processed_repository_ids )
+        if repository_id == primary_repository_id:
+            # Append the primary_repository_id without processing it since it must be returned last in the order.
+            # It will be processed below after all dependencies are processed.
+            processed_repository_ids.append( primary_repository_id )
+            continue
         processed_repository_ids.append( repository_id )
         if repository_id not in ordered_repository_ids:
             prior_import_required_ids = prior_import_required_dict[ repository_id ]
@@ -312,4 +334,11 @@ def order_components_for_import( trans, repository_ids, repo_info_dicts ):
                 ordered_repository_ids.append( repository_id )
                 ordered_repositories.append( repository )
                 ordered_changeset_revisions.append( changeset_revision )
+    # Process the repository associated with the received primary_repository_id last.
+    repo_info_dict = get_repo_info_dict_for_import( primary_repository_id, repository_ids, repo_info_dicts )
+    repository, changeset_revision = get_components_from_repo_info_dict( trans, repo_info_dict )
+    if repository and changeset_revision:
+        ordered_repository_ids.append( repository_id )
+        ordered_repositories.append( repository )
+        ordered_changeset_revisions.append( changeset_revision )
     return ordered_repository_ids, ordered_repositories, ordered_changeset_revisions

lib/tool_shed/util/repository_maintenance_util.py

@@ -154,6 +154,9 @@ def create_repository_and_import_archive( trans, repository_archive_dict, import
                 # Populate the new repository with the contents of exported repository archive.
                 results_dict = import_util.import_repository_archive( trans, repository, repository_archive_dict )
                 ok = results_dict.get( 'ok', False )
+                error_message = results_dict.get( 'error_message', '' )
+                if error_message:
+                    results_message += error_message
                 import_results_tups.append( ( ok, ( str( name ), str( username ) ), results_message ) )
         else:
             # The repository either already exists in this Tool Shed or the current user is not authorized to create it.

lib/tool_shed/util/shed_util_common.py

@@ -745,10 +745,11 @@ def get_next_downloadable_changeset_revision( repository, repo, after_changeset_
 
 def get_next_prior_import_or_install_required_dict_entry( prior_required_dict, processed_tsr_ids ):
     """
-    This method is used in the Tool Shed when exporting a repository and it's dependencies, and in Galaxy when a repository and it's dependencies
-    are being installed.  The order in which the prior_required_dict is processed is critical in order to ensure that the ultimate repository import
-    or installation order is correctly defined.  This method determines the next key / value pair from the received prior_required_dict that should
-    be processed.
+    This method is used in the Tool Shed when exporting a repository and it's dependencies, and in Galaxy
+    when a repository and it's dependencies are being installed.  The order in which the prior_required_dict
+    is processed is critical in order to ensure that the ultimate repository import or installation order is
+    correctly defined.  This method determines the next key / value pair from the received prior_required_dict
+    that should be processed.
     """
     # Return the first key / value pair that is not yet processed and whose value is an empty list.
     for key, value in prior_required_dict.items():
@@ -756,7 +757,8 @@ def get_next_prior_import_or_install_required_dict_entry( prior_required_dict, p
             continue
         if not value:
             return key
-    # Return the first key / value pair that is not yet processed and whose ids in value are all included in processed_tsr_ids.
+    # Return the first key / value pair that is not yet processed and whose ids in value are all included
+    # in processed_tsr_ids.
     for key, value in prior_required_dict.items():
         if key in processed_tsr_ids:
             continue
@@ -767,7 +769,8 @@ def get_next_prior_import_or_install_required_dict_entry( prior_required_dict, p
                 break
         if all_contained:
             return key
-    # Return the first key / value pair that is not yet processed.  Hopefully this is all that is necessary at this point.
+    # Return the first key / value pair that is not yet processed.  Hopefully this is all that is necessary
+    # at this point.
     for key, value in prior_required_dict.items():
         if key in processed_tsr_ids:
             continue
@@ -862,9 +865,10 @@ def get_prior_import_or_install_required_dict( trans, tsr_ids, repo_info_dicts )
 
 def get_query_for_setting_metadata_on_repositories( trans, my_writable=False, order=True ):
     """
-    Return a query containing repositories for resetting metadata.  This method is called from both the Tool Shed and Galaxy.  The
-    my_writable parameter is ignored unless called from the Tool Shed, and the order parameter is used for displaying the list of
-    repositories ordered alphabetically for display on a page.  When called from wither the Tool Shed or Galaxy API, order is False.
+    Return a query containing repositories for resetting metadata.  This method is called from both the
+    Tool Shed and Galaxy.  The my_writable parameter is ignored unless called from the Tool Shed, and the
+    order parameter is used for displaying the list of repositories ordered alphabetically for display on
+    a page.  When called from either the Tool Shed or Galaxy API, order is False.
     """
     if trans.webapp.name == 'tool_shed':
         # When called from the Tool Shed API, the metadata is reset on all repositories of type tool_dependency_definition in addition
@@ -873,7 +877,8 @@ def get_query_for_setting_metadata_on_repositories( trans, my_writable=False, or
             username = trans.user.username
             clause_list = []
             for repository in trans.sa_session.query( trans.model.Repository ) \
-                                              .filter( trans.model.Repository.table.c.deleted == False ):
+                                              .filter( and_( trans.model.Repository.table.c.deleted == False,
+                                                             trans.model.Repository.table.c.deprecated == False ) ):
                 # Always reset metadata on all repositories of type tool_dependency_definition.
                 if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION:
                     clause_list.append( trans.model.Repository.table.c.id == repository.id )
@@ -900,12 +905,14 @@ def get_query_for_setting_metadata_on_repositories( trans, my_writable=False, or
         else:
             if order:
                 return trans.sa_session.query( trans.model.Repository ) \
-                                       .filter( trans.model.Repository.table.c.deleted == False ) \
+                                       .filter( and_( trans.model.Repository.table.c.deleted == False,
+                                                      trans.model.Repository.table.c.deprecated == False ) ) \
                                        .order_by( trans.model.Repository.table.c.name,
                                                   trans.model.Repository.table.c.user_id )
             else:
                 return trans.sa_session.query( trans.model.Repository ) \
-                                       .filter( trans.model.Repository.table.c.deleted == False )
+                                       .filter( and_( trans.model.Repository.table.c.deleted == False,
+                                                      trans.model.Repository.table.c.deprecated == False ) )
     else:
         # We're in Galaxy.
         if order:

scripts/cleanup_datasets/pgcleanup.py

@@ -105,7 +105,7 @@ class Cleanup(object):
         url = make_url(self.config.database_connection)
 
         log.info('Connecting to database with URL: %s' % url)
-        args = url.translate_connect_args()
+        args = url.translate_connect_args( username='user' )
         args.update(url.query)
 
         assert url.get_dialect().name == 'postgresql', 'This script can only be used with PostgreSQL.'

scripts/functional_tests.py

@@ -313,6 +313,7 @@ def main():
             file_path = os.path.join( galaxy_db_path, 'files' )
             new_file_path = tempfile.mkdtemp( prefix='new_files_path_', dir=tempdir )
             job_working_directory = tempfile.mkdtemp( prefix='job_working_directory_', dir=tempdir )
+            install_database_connection = os.environ.get( 'GALAXY_TEST_INSTALL_DBURI', None )
             if 'GALAXY_TEST_DBURI' in os.environ:
                 database_connection = os.environ['GALAXY_TEST_DBURI']
             else:
@@ -372,6 +373,8 @@ def main():
                        user_library_import_dir=user_library_import_dir,
                        master_api_key=master_api_key,
         )
+        if install_database_connection is not None:
+            kwargs[ 'install_database_connection' ] = install_database_connection
         if psu_production:
             kwargs[ 'global_conf' ] = None
         if not database_connection.startswith( 'sqlite://' ):

static/scripts/galaxy.library.js

@@ -90,7 +90,7 @@ define([
 // === VIEWS ====
 // MMMMMMMMMMMMMM
 
-// galaxy folder
+//main view for folder browsing
 var FolderContentView = Backbone.View.extend({
     // main element definition
     el : '#center',
@@ -116,7 +116,7 @@ var FolderContentView = Backbone.View.extend({
 // === TEMPLATES ====
 // MMMMMMMMMMMMMMMMMM
 
-    // set up
+    // main template for folder browsing
     templateFolder : function (){
         var tmpl_array = [];
 
@@ -162,7 +162,7 @@ var FolderContentView = Backbone.View.extend({
         tmpl_array.push('       <th>name</th>');
         tmpl_array.push('       <th>data type</th>');
         tmpl_array.push('       <th>size</th>');
-        tmpl_array.push('       <th>date</th>');
+        tmpl_array.push('       <th>date (UTC)</th>');
         tmpl_array.push('   </thead>');
         tmpl_array.push('   <tbody>');
         tmpl_array.push('       <td></td>');
@@ -184,7 +184,7 @@ var FolderContentView = Backbone.View.extend({
         tmpl_array.push('           <span class="muted">(empty folder)</span>');
         tmpl_array.push('           <% } %>');
         tmpl_array.push('           </td>');
-        tmpl_array.push('           <td>folder</td>'); // data type
+        tmpl_array.push('           <td>folder</td>');
         tmpl_array.push('           <td><%= _.escape(content_item.get("item_count")) %> item(s)</td>'); // size
         tmpl_array.push('           <% } else {  %>');
         tmpl_array.push('           <td style="text-align: center; "><input style="margin: 0;" type="checkbox"></td>');
@@ -228,7 +228,7 @@ var FolderContentView = Backbone.View.extend({
         tmpl_array.push('           <td><%= _.escape(size) %></td>');
         tmpl_array.push('       </tr>');
         tmpl_array.push('       <tr>');
-        tmpl_array.push('           <th scope="row">Date uploaded</th>');
+        tmpl_array.push('           <th scope="row">Date uploaded (UTC)</th>');
         tmpl_array.push('           <td><%= _.escape(item.get("date_uploaded")) %></td>');
         tmpl_array.push('       </tr>');
         tmpl_array.push('       <tr>');
@@ -240,7 +240,7 @@ var FolderContentView = Backbone.View.extend({
         tmpl_array.push('           <td scope="row"><%= _.escape(item.get("metadata_data_lines")) %></td>');
         tmpl_array.push('       </tr>');
         tmpl_array.push('       <th scope="row">Comment Lines</th>');
-        tmpl_array.push('           <% if (item.get("metadata_comment_lines") === "") { %>'); //folder
+        tmpl_array.push('           <% if (item.get("metadata_comment_lines") === "") { %>');
         tmpl_array.push('               <td scope="row"><%= _.escape(item.get("metadata_comment_lines")) %></td>');
         tmpl_array.push('           <% } else { %>');
         tmpl_array.push('               <td scope="row">unknown</td>');
@@ -296,19 +296,34 @@ var FolderContentView = Backbone.View.extend({
         return tmpl_array.join('');
         },
 
-      // convert size to nice string
-      size_to_string : function (size)
-      {
-            // identify unit
-            var unit = "";
-            if (size >= 100000000000)   { size = size / 100000000000; unit = "TB"; } else
-            if (size >= 100000000)      { size = size / 100000000; unit = "GB"; } else
-            if (size >= 100000)         { size = size / 100000; unit = "MB"; } else
-            if (size >= 100)            { size = size / 100; unit = "KB"; } else
-            { size = size * 10; unit = "b"; }
-            // return formatted string
-            return (Math.round(size) / 10) + unit;
-        },
+      templateProgressBar : function (){
+        var tmpl_array = [];
+
+        tmpl_array.push('<div class="import_text">');
+        tmpl_array.push('Importing selected datasets to history <b><%= _.escape(history_name) %></b>');
+        tmpl_array.push('</div>');
+        tmpl_array.push('<div class="progress">');
+        tmpl_array.push('   <div class="progress-bar progress-bar-import" role="progressbar" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100" style="width: 00%;">');
+        tmpl_array.push('       <span class="completion_span">0% Complete</span>');
+        tmpl_array.push('   </div>');
+        tmpl_array.push('</div>');
+        tmpl_array.push('');
+
+        return tmpl_array.join('');
+      },
+
+    templateNewFolderInModal: function(){
+        tmpl_array = [];
+
+        tmpl_array.push('<div id="new_folder_modal">');
+        tmpl_array.push('<form>');
+        tmpl_array.push('<input type="text" name="Name" value="" placeholder="Name">');
+        tmpl_array.push('<input type="text" name="Description" value="" placeholder="Description">');
+        tmpl_array.push('</form>');
+        tmpl_array.push('</div>');
+
+        return tmpl_array.join('');
+    },
 
 // MMMMMMMMMMMMMMM
 // === EVENTS ====
@@ -325,6 +340,10 @@ var FolderContentView = Backbone.View.extend({
             'click .btn_open_folder' : 'navigateToFolder'
         },
 
+// MMMMMMMMMMMMMMMMMM
+// === FUNCTIONS ====
+// MMMMMMMMMMMMMMMMMM
+
       //render the folder view
       render: function (options) {
         //hack to show scrollbars
@@ -360,10 +379,27 @@ var FolderContentView = Backbone.View.extend({
               // var template = _.template(that.templateFolder(), { path: folderContainer.full_path, items: folderContainer.attributes.folder.models, id: options.id });
               that.$el.html(template);
 
-            }
+            },
+              error: function(){
+                mod_toastr.error('An error occured :(');
+              }
           })
         },
 
+      // convert size to nice string
+      size_to_string : function (size)
+      {
+            // identify unit
+            var unit = "";
+            if (size >= 100000000000)   { size = size / 100000000000; unit = "TB"; } else
+            if (size >= 100000000)      { size = size / 100000000; unit = "GB"; } else
+            if (size >= 100000)         { size = size / 100000; unit = "MB"; } else
+            if (size >= 100)            { size = size / 100; unit = "KB"; } else
+            { size = size * 10; unit = "b"; }
+            // return formatted string
+            return (Math.round(size) / 10) + unit;
+        },
+
       // handles the click on 'open' and 'upper' folder icons
       navigateToFolder : function(event){
         var folder_id = $(event.target).attr('data-id');
@@ -398,9 +434,18 @@ var FolderContentView = Backbone.View.extend({
 // TODO can render here already
                 //fetch user histories for import purposes
                 histories.fetch({
-                    success: function (histories){self.renderModalAfterFetch(item, histories)}
+                    success: function (histories){
+                        self.renderModalAfterFetch(item, histories)
+                    },
+                      error: function(){
+                        mod_toastr.error('An error occured during fetching histories:(');
+                        self.renderModalAfterFetch(item)
+                    }
                 });
-            }
+            },
+              error: function(){
+                mod_toastr.error('An error occured during loading dataset details :(');
+              }
         });
     },
 
@@ -422,16 +467,17 @@ var FolderContentView = Backbone.View.extend({
             });
             
             $(".peek").html(item.get("peek"));
-            var history_footer_tmpl = _.template(this.templateHistorySelectInModal(), {histories : histories.models});
-            $(this.modal.elMain).find('.buttons').prepend(history_footer_tmpl);
-
-            // preset last selected history if we know it
-            if (self.lastSelectedHistory.length > 0) {
-                $(this.modal.elMain).find('#dataset_import_single').val(self.lastSelectedHistory);
-            }
 
+            // show the import-into-history footer only if the request for histories succeeded
+            if (typeof history.models !== undefined){
+                var history_footer_tmpl = _.template(this.templateHistorySelectInModal(), {histories : histories.models});
+                $(this.modal.elMain).find('.buttons').prepend(history_footer_tmpl);
+                // preset last selected history if we know it
+                if (self.lastSelectedHistory.length > 0) {
+                    $(this.modal.elMain).find('#dataset_import_single').val(self.lastSelectedHistory);
+                }
+            } 
             this.modal.bindEvents();
-            // show the prepared modal
             this.modal.show();
         },
 
@@ -555,7 +601,6 @@ var FolderContentView = Backbone.View.extend({
                 $('#toolbtn_bulk_import').hide();
                 $('#toolbtn_dl').hide();
             }
-
         },
 
         // show bulk import modal
@@ -577,8 +622,10 @@ var FolderContentView = Backbone.View.extend({
                             }
                         });
                         self.modal.bindEvents();
-                        // show the prepared modal
                         self.modal.show();
+                    },
+                    error: function(){
+                      mod_toastr.error('An error occured :(');
                     }
                 });
         },
@@ -616,7 +663,7 @@ var FolderContentView = Backbone.View.extend({
                 datasets_to_import.push(historyItem);
             };
 
-            // call the recursive function to call ajax one after each other
+            // call the recursive function to call ajax one after each other (request FIFO queue)
             this.chainCall(datasets_to_import);
         },
 
@@ -646,22 +693,7 @@ var FolderContentView = Backbone.View.extend({
             $('.completion_span').text(txt_representation);
         },
 
-      // progress bar
-      templateProgressBar : function (){
-        var tmpl_array = [];
 
-        tmpl_array.push('<div class="import_text">');
-        tmpl_array.push('Importing selected datasets to history <b><%= _.escape(history_name) %></b>');
-        tmpl_array.push('</div>');
-        tmpl_array.push('<div class="progress">');
-        tmpl_array.push('   <div class="progress-bar progress-bar-import" role="progressbar" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100" style="width: 00%;">');
-        tmpl_array.push('       <span class="completion_span">0% Complete</span>');
-        tmpl_array.push('   </div>');
-        tmpl_array.push('</div>');
-        tmpl_array.push('');
-
-        return tmpl_array.join('');
-      },
 
       // download selected datasets
       download : function(folder_id, format){
@@ -706,14 +738,13 @@ var FolderContentView = Backbone.View.extend({
             this.modal = new mod_modal.GalaxyModal({
                 destructible : true,
                 title   : 'Create New Folder',
-                body    : this.template_new_folder(),
+                body    : this.templateNewFolderInModal(),
                 buttons : {
                     'Create' : function() {self.create_new_folder_event()},
                     'Close'  : function() {self.modal.hideOrDestroy(); self.modal = null;}
                 }
             });
         this.modal.bindEvents();
-        // show prepared modal
         this.modal.show();
       },
 
@@ -726,7 +757,7 @@ var FolderContentView = Backbone.View.extend({
             url_items = Backbone.history.fragment.split('/');
             current_folder_id = url_items[url_items.length-1];
             folder.url = folder.urlRoot + '/' + current_folder_id ;
-            
+
             var self = this;
             folder.save(folderDetails, {
               success: function (folder) {
@@ -755,20 +786,6 @@ var FolderContentView = Backbone.View.extend({
     // validate new library info
     validate_new_folder: function(folderDetails){
         return folderDetails.name !== '';
-    },
-
-    // template for new library modal
-    template_new_folder: function(){
-        tmpl_array = [];
-
-        tmpl_array.push('<div id="new_folder_modal">');
-        tmpl_array.push('<form>');
-        tmpl_array.push('<input type="text" name="Name" value="" placeholder="Name">');
-        tmpl_array.push('<input type="text" name="Description" value="" placeholder="Description">');
-        tmpl_array.push('</form>');
-        tmpl_array.push('</div>');
-
-        return tmpl_array.join('');
     }
 
     });
@@ -785,11 +802,13 @@ var GalaxyLibraryview = Backbone.View.extend({
     initialize : function(){
     },
 
-    // template
-    template_library_list : function (){
+// MMMMMMMMMMMMMMMMMM
+// === TEMPLATES ====
+// MMMMMMMMMMMMMMMMMM
+
+    templateLibraryList: function(){
         tmpl_array = [];
         tmpl_array.push('<div id="library_container" style="width: 90%; margin: auto; margin-top: 2em; overflow: auto !important; ">');
-
         tmpl_array.push('');
         tmpl_array.push('<h3>New Data Libraries. This is work in progress. Report problems & ideas to <a href="mailto:marten@bx.psu.edu?Subject=DataLibraries_Feedback" target="_blank">Marten</a>.</h3>');
         tmpl_array.push('<a href="" id="create_new_library_btn" class="btn btn-primary file ">New Library</a>');
@@ -814,14 +833,27 @@ var GalaxyLibraryview = Backbone.View.extend({
         tmpl_array.push('       <% }); %>');
         tmpl_array.push('   </tbody>');
         tmpl_array.push('</table>');
-
         tmpl_array.push('</div>');
+
+        return tmpl_array.join('');
+    },
+
+    templateNewLibraryInModal: function(){
+        tmpl_array = [];
+
+        tmpl_array.push('<div id="new_library_modal">');
+        tmpl_array.push('   <form>');
+        tmpl_array.push('       <input type="text" name="Name" value="" placeholder="Name">');
+        tmpl_array.push('       <input type="text" name="Description" value="" placeholder="Description">');
+        tmpl_array.push('       <input type="text" name="Synopsis" value="" placeholder="Synopsis">');
+        tmpl_array.push('   </form>');
+        tmpl_array.push('</div>');
+
         return tmpl_array.join('');
     },
 
-    // render
     render: function () {
-        //hack to show scrollbars due to #center element inheritance
+        // modification of upper DOM element to show scrollbars due to the #center element inheritance
         $("#center").css('overflow','auto');
         
         var that = this;
@@ -829,7 +861,7 @@ var GalaxyLibraryview = Backbone.View.extend({
 
         libraries.fetch({
           success: function (libraries) {
-            var template = _.template(that.template_library_list(), { libraries : libraries.models });
+            var template = _.template(that.templateLibraryList(), { libraries : libraries.models });
             that.$el.html(template);
           },
           error: function(model, response){
@@ -863,14 +895,13 @@ var GalaxyLibraryview = Backbone.View.extend({
             this.modal = new mod_modal.GalaxyModal({
                 destructible : true,
                 title   : 'Create New Library',
-                body    : this.template_new_library(),
+                body    : this.templateNewLibraryInModal(),
                 buttons : {
                     'Create' : function() {self.create_new_library_event()},
                     'Close'  : function() {self.modal.hideOrDestroy();}
                 }
             });
         this.modal.bindEvents();
-        // show prepared modal
         this.modal.show();
     },
 
@@ -916,21 +947,6 @@ var GalaxyLibraryview = Backbone.View.extend({
     // validate new library info
     validate_new_library: function(libraryDetails){
         return libraryDetails.name !== '';
-    },
-
-    // template for new library modal
-    template_new_library: function(){
-        tmpl_array = [];
-
-        tmpl_array.push('<div id="new_library_modal">');
-        tmpl_array.push('<form>');
-        tmpl_array.push('<input type="text" name="Name" value="" placeholder="Name">');
-        tmpl_array.push('<input type="text" name="Description" value="" placeholder="Description">');
-        tmpl_array.push('<input type="text" name="Synopsis" value="" placeholder="Synopsis">');
-        tmpl_array.push('</form>');
-        tmpl_array.push('</div>');
-
-        return tmpl_array.join('');
     }
 });
 
@@ -938,6 +954,7 @@ var GalaxyLibraryview = Backbone.View.extend({
 var GalaxyLibrary = Backbone.View.extend({
     folderContentView : null,
     galaxyLibraryview : null,
+
     initialize : function(){
 
         folderContentView = new FolderContentView();

static/scripts/galaxy.modal.js

@@ -19,7 +19,9 @@ var GalaxyModal = Backbone.View.extend(
     
     // options
     options : {
-        destructible: false // by default the modal cannot be removed by theself.destroy() method
+        // by default the modal cannot be removed by the self.destroy() method 
+        // but only hidden through self.hide()
+        destructible: false 
     },
     
     // initialize
@@ -99,9 +101,9 @@ var GalaxyModal = Backbone.View.extend(
     hide: function(){
         // fade out
         this.$el.fadeOut('fast');
-        
         // set flag
         this.visible = false;
+        // unbind events
         this.unbindEvents();
     },    
 
@@ -109,8 +111,8 @@ var GalaxyModal = Backbone.View.extend(
     destroy: function(){
         // set flag
         this.visible = false;
+        // unbind events
         this.unbindEvents();
-
         // remove
         this.$el.remove();
     },

static/scripts/mvc/dataset/hda-base.js

@@ -215,7 +215,8 @@ var HDABaseView = Backbone.View.extend( LoggableMixin ).extend(
         // return either: a single download icon-button (if there are no meta files)
         if( _.isEmpty( meta_files ) ){
             return $([
-                '<a href="' + urls.download + '" title="' + _l( 'Download' ) + '" class="icon-btn">',
+                '<a href="' + urls.download + '" title="' + _l( 'Download' ) + '" ',
+                    'class="icon-btn dataset-download-btn">',
                     '<span class="fa fa-floppy-o"></span>',
                 '</a>'
             ].join( '' ) );
@@ -239,7 +240,8 @@ var HDABaseView = Backbone.View.extend( LoggableMixin ).extend(
                 '</div>',
 
                 '<div class="icon-btn-group">',
-                    '<a href="' + urls.download + '" title="' + _l( 'Download' ) + '" class="icon-btn">',
+                    '<a href="' + urls.download + '" title="' + _l( 'Download' ) + '" ',
+                        'class="icon-btn dataset-download-btn">',
                         '<span class="fa fa-floppy-o"></span>',
                     // join these w/o whitespace or there'll be a gap when rendered
                     '</a><a class="icon-btn popup" id="' + menuId + '">',
@@ -257,6 +259,7 @@ var HDABaseView = Backbone.View.extend( LoggableMixin ).extend(
         // gen. safe to show in all cases
         return faIconButton({
             title       : _l( 'View details' ),
+            classes     : 'dataset-params-btn',
             href        : this.urls.show_params,
             target      : this.linkTarget,
             faIcon      : 'fa-info-circle'

static/scripts/mvc/dataset/hda-edit.js

@@ -72,7 +72,7 @@ var HDAEditView = hdaBase.HDABaseView.extend( LoggableMixin ).extend(
         var purged = this.model.get( 'purged' ),
             deleted = this.model.get( 'deleted' ),
             editBtnData = {
-                title       : _l( 'Edit Attributes' ),
+                title       : _l( 'Edit attributes' ),
                 href        : this.urls.edit,
                 target      : this.linkTarget,
                 classes     : 'dataset-edit'
@@ -138,6 +138,7 @@ var HDAEditView = hdaBase.HDABaseView.extend( LoggableMixin ).extend(
         return faIconButton({
             title       : _l( 'View or report this error' ),
             href        : this.urls.report_error,
+            classes     : 'dataset-report-error-btn',
             target      : this.linkTarget,
             faIcon      : 'fa-bug'
         });
@@ -150,6 +151,7 @@ var HDAEditView = hdaBase.HDABaseView.extend( LoggableMixin ).extend(
         return faIconButton({
             title       : _l( 'Run this job again' ),
             href        : this.urls.rerun,
+            classes     : 'dataset-rerun-btn',
             target      : this.linkTarget,
             faIcon      : 'fa-refresh'
         });
@@ -189,6 +191,7 @@ var HDAEditView = hdaBase.HDABaseView.extend( LoggableMixin ).extend(
 
         var $icon = faIconButton({
             title       : _l( 'Visualize' ),
+            classes     : 'dataset-visualize-btn',
             faIcon      : 'fa-bar-chart-o'
         });
 
@@ -242,6 +245,7 @@ var HDAEditView = hdaBase.HDABaseView.extend( LoggableMixin ).extend(
         }
         var $icon = faIconButton({
             title       : _l( 'Visualize' ),
+            classes     : 'dataset-visualize-btn',
             faIcon      : 'fa-bar-chart-o'
         });
         $icon.addClass( 'visualize-icon' ); // needed?
@@ -364,8 +368,10 @@ var HDAEditView = hdaBase.HDABaseView.extend( LoggableMixin ).extend(
         // make the dbkey a link to editing
         if( this.model.get( 'metadata_dbkey' ) === '?'
         &&  !this.model.isDeletedOrPurged() ){
-            $body.find( '.dataset-dbkey .value' ).replaceWith(
-                $( '<a target="' + this.linkTarget + '">?</a>' ).attr( 'href', this.urls.edit ) );
+            var editableDbkey = $( '<a class="value">?</a>' )
+                .attr( 'href', this.urls.edit )
+                .attr( 'target', this.linkTarget );
+            $body.find( '.dataset-dbkey .value' ).replaceWith( editableDbkey );
         }
     },
 

templates/webapps/galaxy/root/index.mako

@@ -1,9 +1,7 @@
 <%inherit file="/webapps/galaxy/base_panels.mako"/>
 
 <%namespace file="/root/tool_menu.mako" import="*" />
-<%namespace file="/history/history_panel.mako" import="history_panel_javascripts" />
 <%namespace file="/history/history_panel.mako" import="current_history_panel" />
-<%namespace file="/history/history_panel.mako" import="history_panel" />
 
 <%def name="stylesheets()">
     ${parent.stylesheets()}

test-data/8.tabular

@@ -1,29 +0,0 @@
-AF099740	ENST00000296869
-AF377960	ENST00000160373
-AK131266	ENST00000350908
-BC005078	ENST00000319432
-BC007833	ENST00000349792
-NM_000054	ENST00000337474
-NM_000127	ENST00000333324
-NM_000425	ENST00000361699
-NM_000492	ENST00000003084
-NM_000588	ENST00000296870
-NM_001167	ENST00000245836
-NM_001666	ENST00000350060
-NM_001927	ENST00000273074
-NM_002651	ENST00000271657
-NM_002796	ENST00000290541
-NM_003269	ENST00000230083
-NM_003391	ENST00000265441
-NM_003687	ENST00000253754
-NM_004199	ENST00000166534
-NM_005997	ENST00000295315
-NM_006773	ENST00000263239
-NM_007214	ENST00000027474
-NM_014908	ENST00000308871
-NM_015354	ENST00000259310
-NM_020145	ENST00000291931
-NM_024536	ENST00000243776
-NM_145315	ENST00000258002
-NM_152827	ENST00000230085
-NM_174933	ENST00000308941

test-data/aggregate_binned_scores_in_intervals.bed

@@ -1 +0,0 @@
-chr7	60000002	60000010

test-data/library/3.bed

@@ -1,25 +0,0 @@
-chr1	147962006	147975713	NM_005997	0	-	147962192	147975670	0	6	574,145,177,115,153,160,	0,1543,7859,9048,9340,13547,
-chr1	147984101	148035079	BC007833	0	+	147984545	148033414	0	14	529,32,81,131,118,153,300,206,84,49,85,130,46,1668,	0,25695,28767,33118,33695,33998,35644,38005,39629,40577,41402,43885,48367,49310,
-chr1	148077485	148111797	NM_002651	0	-	148078400	148111728	0	12	1097,121,133,266,124,105,110,228,228,45,937,77,	0,2081,2472,6871,9907,10257,11604,14199,15637,18274,23636,34235,
-chr1	148185113	148187485	NM_002796	0	+	148185136	148187378	0	7	163,207,147,82,117,89,120,	0,416,877,1199,1674,1977,2252,
-chr2	118288484	118306183	NM_006773	0	+	118288583	118304530	0	14	184,285,144,136,101,200,115,140,162,153,114,57,178,1796,	0,2765,4970,6482,6971,7183,7468,9890,10261,10768,11590,14270,14610,15903,
-chr2	118389378	118390700	BC005078	0	-	118390395	118390500	0	1	1322,	0,
-chr2	220108603	220116964	NM_001927	0	+	220108689	220116217	0	9	664,61,96,162,126,221,44,83,789,	0,1718,1874,2118,2451,2963,5400,7286,7572,
-chr2	220229182	220233943	NM_024536	0	-	220229609	220233765	0	4	1687,180,574,492,	0,1990,2660,4269,
-chr5	131170738	131357870	AF099740	0	-	131311206	131357817	0	31	112,124,120,81,65,40,120,129,61,88,94,79,72,102,144,117,89,73,96,135,135,78,74,52,33,179,100,102,65,115,248,	0,11593,44117,47607,104668,109739,114675,126366,135488,137518,138009,140437,152389,153373,155388,159269,160793,162981,164403,165577,166119,167611,169501,178260,179675,180901,181658,182260,182953,183706,186884,
-chr5	131424245	131426795	NM_000588	0	+	131424298	131426383	0	5	215,42,90,42,535,	0,313,1658,1872,2015,
-chr5	131556201	131590458	NM_004199	0	-	131556601	131582218	0	15	471,97,69,66,54,100,71,177,194,240,138,152,97,100,170,	0,2316,2802,5596,6269,11138,11472,15098,16528,17674,21306,24587,25142,25935,34087,
-chr5	131621285	131637046	NM_003687	0	+	131621326	131635821	0	7	134,152,82,179,164,118,1430,	0,4915,8770,13221,13609,14097,14331,
-chr6	108298214	108386086	NM_007214	0	-	108299600	108385906	0	21	1530,105,99,102,159,174,60,83,148,155,93,133,95,109,51,59,62,113,115,100,304,	0,2490,6246,10831,12670,23164,23520,27331,31052,32526,34311,36130,36365,38609,41028,42398,43048,51479,54500,59097,87568,
-chr6	108593954	108616704	NM_003269	0	+	108594662	108615360	0	9	733,146,88,236,147,97,150,106,1507,	0,5400,8778,10445,12037,14265,14749,15488,21243,
-chr6	108639410	108689143	NM_152827	0	-	108640045	108688818	0	3	741,125,487,	0,2984,49246,
-chr6	108722790	108950942	NM_145315	0	+	108722976	108950321	0	13	325,224,52,102,131,100,59,83,71,101,141,114,750,	0,28931,52094,60760,61796,71339,107102,152319,181970,182297,215317,224802,227402,
-chr7	113320332	113924911	AK131266	0	+	113862563	113893433	0	20	285,91,178,90,58,75,138,51,201,178,214,105,88,84,77,102,122,70,164,1124,	0,201692,340175,448290,451999,484480,542213,543265,543478,545201,556083,558358,565876,567599,573029,573245,575738,577123,577946,603455,
-chr7	116511232	116557294	NM_003391	0	-	116512159	116556994	0	5	1157,265,278,227,383,	0,20384,37843,43339,45679,
-chr7	116713967	116902666	NM_000492	0	+	116714099	116901113	0	27	185,111,109,216,90,164,126,247,93,183,192,95,87,724,129,38,251,80,151,228,101,249,156,90,173,106,1754,	0,24290,29071,50936,54313,55285,56585,60137,62053,68678,79501,107776,110390,111971,114967,122863,123569,126711,130556,131618,134650,147559,162475,172879,184725,185496,186945,
-chr7	116944658	117107512	AF377960	0	-	116945541	116979926	0	23	1129,102,133,64,186,206,179,188,153,100,87,80,96,276,118,255,151,100,204,1654,225,108,173,	0,7364,8850,10413,13893,14398,17435,24259,24615,35177,35359,45901,47221,49781,56405,66857,69787,72208,73597,80474,100111,150555,162681,
-chr8	118880786	119193239	NM_000127	0	-	118881131	119192466	0	11	531,172,161,90,96,119,133,120,108,94,1735,	0,5355,7850,13505,19068,20309,23098,30863,36077,37741,310718,
-chr9	128763240	128783870	NM_174933	0	+	128764156	128783586	0	12	261,118,74,159,76,48,56,63,129,117,127,370,	0,522,875,5630,12374,12603,15040,15175,18961,19191,20037,20260,
-chr9	128787362	128789566	NM_014908	0	-	128787519	128789136	0	1	2204,	0,
-chr9	128789530	128848928	NM_015354	0	+	128789552	128848511	0	44	54,55,74,85,81,45,93,120,212,115,201,90,66,120,127,153,127,88,77,115,121,67,129,140,107,207,170,70,68,196,78,86,146,182,201,93,159,138,75,228,132,74,130,594,	0,1491,5075,8652,9254,10312,11104,11317,20808,21702,23060,25462,31564,32908,33566,34851,35204,35595,35776,37202,38860,39111,39891,40349,42422,45499,45827,46675,47158,47621,50453,50840,51474,51926,53831,54186,55119,55619,57449,57605,57947,58352,58541,58804,
-chr9	128849867	128870133	NM_020145	0	-	128850516	128869987	0	11	757,241,101,90,24,63,93,134,129,142,209,	0,1071,1736,2085,2635,4201,6376,6736,13056,14247,20057,

test-data/library/4.bed

@@ -1 +0,0 @@
-chr22	30128507	31828507	uc003bnx.1_cds_2_0_chr22_29227_f	0	+

test-data/library/5.bed

@@ -1,134 +0,0 @@
-chr7	115444712	115444739	CCDS5763.1_cds_0_0_chr7_115444713_f	0	+
-chr7	115468538	115468624	CCDS5763.1_cds_1_0_chr7_115468539_f	0	+
-chr7	115483024	115483277	CCDS5763.1_cds_2_0_chr7_115483025_f	0	+
-chr7	115484165	115484501	CCDS5763.1_cds_3_0_chr7_115484166_f	0	+
-chr7	115485764	115485980	CCDS5763.1_cds_4_0_chr7_115485765_f	0	+
-chr7	115486322	115486481	CCDS5763.1_cds_5_0_chr7_115486323_f	0	+
-chr7	115491298	115491487	CCDS5763.1_cds_6_0_chr7_115491299_f	0	+
-chr7	115468538	115468624	CCDS5764.1_cds_0_0_chr7_115468539_f	0	+
-chr7	115483024	115483277	CCDS5764.1_cds_1_0_chr7_115483025_f	0	+
-chr7	115484165	115484501	CCDS5764.1_cds_2_0_chr7_115484166_f	0	+
-chr7	115485764	115485980	CCDS5764.1_cds_3_0_chr7_115485765_f	0	+
-chr7	115486322	115486481	CCDS5764.1_cds_4_0_chr7_115486323_f	0	+
-chr7	115491298	115491487	CCDS5764.1_cds_5_0_chr7_115491299_f	0	+
-chr7	115733786	115733936	CCDS5766.1_cds_0_0_chr7_115733787_f	0	+
-chr7	115734264	115734452	CCDS5766.1_cds_1_0_chr7_115734265_f	0	+
-chr7	115739975	115740126	CCDS5766.1_cds_2_0_chr7_115739976_f	0	+
-chr7	115733786	115733936	CCDS5765.1_cds_0_0_chr7_115733787_f	0	+
-chr7	115739975	115740164	CCDS5765.1_cds_1_0_chr7_115739976_f	0	+
-chr7	115759067	115759097	CCDS5767.1_cds_0_0_chr7_115759068_f	0	+
-chr7	115760529	115760694	CCDS5767.1_cds_1_0_chr7_115760530_f	0	+
-chr7	115792950	115793292	CCDS5767.1_cds_2_0_chr7_115792951_f	0	+
-chr7	116096616	116096655	CCDS5768.1_cds_0_0_chr7_116096617_f	0	+
-chr7	116122131	116122195	CCDS5768.1_cds_1_0_chr7_116122132_f	0	+
-chr7	116126998	116127050	CCDS5768.1_cds_2_0_chr7_116126999_f	0	+
-chr7	116132776	116132840	CCDS5768.1_cds_3_0_chr7_116132777_f	0	+
-chr7	116138181	116138388	CCDS5768.1_cds_4_0_chr7_116138182_f	0	+
-chr7	116140267	116140347	CCDS5768.1_cds_5_0_chr7_116140268_f	0	+
-chr7	116144237	116144316	CCDS5768.1_cds_6_0_chr7_116144238_f	0	+
-chr7	116146073	116146145	CCDS5768.1_cds_7_0_chr7_116146074_f	0	+
-chr7	116150064	116150127	CCDS5768.1_cds_8_0_chr7_116150065_f	0	+
-chr7	116151731	116151872	CCDS5768.1_cds_9_0_chr7_116151732_f	0	+
-chr7	116187545	116187696	CCDS5770.1_cds_0_0_chr7_116187546_f	0	+
-chr7	116333766	116333849	CCDS5770.1_cds_1_0_chr7_116333767_f	0	+
-chr7	116353565	116353725	CCDS5770.1_cds_2_0_chr7_116353566_f	0	+
-chr7	116363797	116363852	CCDS5770.1_cds_3_0_chr7_116363798_f	0	+
-chr7	116364495	116364611	CCDS5770.1_cds_4_0_chr7_116364496_f	0	+
-chr7	116365889	116365965	CCDS5770.1_cds_5_0_chr7_116365890_f	0	+
-chr7	116368128	116368197	CCDS5770.1_cds_6_0_chr7_116368129_f	0	+
-chr7	116370085	116370240	CCDS5770.1_cds_7_0_chr7_116370086_f	0	+
-chr7	116372439	116372537	CCDS5770.1_cds_8_0_chr7_116372440_f	0	+
-chr7	116404866	116404981	CCDS5770.1_cds_9_0_chr7_116404867_f	0	+
-chr7	116423325	116423398	CCDS5770.1_cds_10_0_chr7_116423326_f	0	+
-chr7	116424838	116424941	CCDS5770.1_cds_11_0_chr7_116424839_f	0	+
-chr7	116443791	116443942	CCDS5770.1_cds_12_0_chr7_116443792_f	0	+
-chr7	116453088	116453181	CCDS5770.1_cds_13_0_chr7_116453089_f	0	+
-chr7	116455927	116456067	CCDS5770.1_cds_14_0_chr7_116455928_f	0	+
-chr7	116456865	116456985	CCDS5770.1_cds_15_0_chr7_116456866_f	0	+
-chr7	116187545	116187696	CCDS5769.1_cds_0_0_chr7_116187546_f	0	+
-chr7	116333766	116333849	CCDS5769.1_cds_1_0_chr7_116333767_f	0	+
-chr7	116353565	116353725	CCDS5769.1_cds_2_0_chr7_116353566_f	0	+
-chr7	116363797	116363852	CCDS5769.1_cds_3_0_chr7_116363798_f	0	+
-chr7	116364495	116364611	CCDS5769.1_cds_4_0_chr7_116364496_f	0	+
-chr7	116365889	116365965	CCDS5769.1_cds_5_0_chr7_116365890_f	0	+
-chr7	116370085	116370240	CCDS5769.1_cds_6_0_chr7_116370086_f	0	+
-chr7	116372439	116372537	CCDS5769.1_cds_7_0_chr7_116372440_f	0	+
-chr7	116404866	116404981	CCDS5769.1_cds_8_0_chr7_116404867_f	0	+
-chr7	116423325	116423398	CCDS5769.1_cds_9_0_chr7_116423326_f	0	+
-chr7	116424838	116424941	CCDS5769.1_cds_10_0_chr7_116424839_f	0	+
-chr7	116443791	116443942	CCDS5769.1_cds_11_0_chr7_116443792_f	0	+
-chr7	116453088	116453181	CCDS5769.1_cds_12_0_chr7_116453089_f	0	+
-chr7	116455927	116456067	CCDS5769.1_cds_13_0_chr7_116455928_f	0	+
-chr7	116463766	116463862	CCDS5769.1_cds_14_0_chr7_116463767_f	0	+
-chr7	116512159	116512389	CCDS5771.1_cds_0_0_chr7_116512160_r	0	-
-chr7	116531616	116531881	CCDS5771.1_cds_1_0_chr7_116531617_r	0	-
-chr7	116549075	116549353	CCDS5771.1_cds_2_0_chr7_116549076_r	0	-
-chr7	116554571	116554798	CCDS5771.1_cds_3_0_chr7_116554572_r	0	-
-chr7	116556911	116556994	CCDS5771.1_cds_4_0_chr7_116556912_r	0	-
-chr7	116597600	116597753	CCDS5772.1_cds_0_0_chr7_116597601_r	0	-
-chr7	116601356	116601470	CCDS5772.1_cds_1_0_chr7_116601357_r	0	-
-chr7	116602616	116602722	CCDS5772.1_cds_2_0_chr7_116602617_r	0	-
-chr7	116613942	116614052	CCDS5772.1_cds_3_0_chr7_116613943_r	0	-
-chr7	116615015	116615072	CCDS5772.1_cds_4_0_chr7_116615016_r	0	-
-chr7	116616073	116616149	CCDS5772.1_cds_5_0_chr7_116616074_r	0	-
-chr7	116616990	116617115	CCDS5772.1_cds_6_0_chr7_116616991_r	0	-
-chr7	116618730	116618865	CCDS5772.1_cds_7_0_chr7_116618731_r	0	-
-chr7	116619702	116619814	CCDS5772.1_cds_8_0_chr7_116619703_r	0	-
-chr7	116654167	116654279	CCDS5772.1_cds_9_0_chr7_116654168_r	0	-
-chr7	116656241	116656364	CCDS5772.1_cds_10_0_chr7_116656242_r	0	-
-chr7	116660840	116660940	CCDS5772.1_cds_11_0_chr7_116660841_r	0	-
-chr7	116661360	116661465	CCDS5772.1_cds_12_0_chr7_116661361_r	0	-
-chr7	116714099	116714152	CCDS5773.1_cds_0_0_chr7_116714100_f	0	+
-chr7	116738257	116738368	CCDS5773.1_cds_1_0_chr7_116738258_f	0	+
-chr7	116743038	116743147	CCDS5773.1_cds_2_0_chr7_116743039_f	0	+
-chr7	116764903	116765119	CCDS5773.1_cds_3_0_chr7_116764904_f	0	+
-chr7	116768280	116768370	CCDS5773.1_cds_4_0_chr7_116768281_f	0	+
-chr7	116769252	116769416	CCDS5773.1_cds_5_0_chr7_116769253_f	0	+
-chr7	116770552	116770678	CCDS5773.1_cds_6_0_chr7_116770553_f	0	+
-chr7	116774104	116774351	CCDS5773.1_cds_7_0_chr7_116774105_f	0	+
-chr7	116776020	116776113	CCDS5773.1_cds_8_0_chr7_116776021_f	0	+
-chr7	116782645	116782828	CCDS5773.1_cds_9_0_chr7_116782646_f	0	+
-chr7	116793468	116793660	CCDS5773.1_cds_10_0_chr7_116793469_f	0	+
-chr7	116821743	116821838	CCDS5773.1_cds_11_0_chr7_116821744_f	0	+
-chr7	116824357	116824444	CCDS5773.1_cds_12_0_chr7_116824358_f	0	+
-chr7	116825938	116826662	CCDS5773.1_cds_13_0_chr7_116825939_f	0	+
-chr7	116828934	116829063	CCDS5773.1_cds_14_0_chr7_116828935_f	0	+
-chr7	116836830	116836868	CCDS5773.1_cds_15_0_chr7_116836831_f	0	+
-chr7	116837536	116837787	CCDS5773.1_cds_16_0_chr7_116837537_f	0	+
-chr7	116840678	116840758	CCDS5773.1_cds_17_0_chr7_116840679_f	0	+
-chr7	116844523	116844674	CCDS5773.1_cds_18_0_chr7_116844524_f	0	+
-chr7	116845585	116845813	CCDS5773.1_cds_19_0_chr7_116845586_f	0	+
-chr7	116848617	116848718	CCDS5773.1_cds_20_0_chr7_116848618_f	0	+
-chr7	116861526	116861775	CCDS5773.1_cds_21_0_chr7_116861527_f	0	+
-chr7	116876442	116876598	CCDS5773.1_cds_22_0_chr7_116876443_f	0	+
-chr7	116886846	116886936	CCDS5773.1_cds_23_0_chr7_116886847_f	0	+
-chr7	116898692	116898865	CCDS5773.1_cds_24_0_chr7_116898693_f	0	+
-chr7	116899463	116899569	CCDS5773.1_cds_25_0_chr7_116899464_f	0	+
-chr7	116900912	116901113	CCDS5773.1_cds_26_0_chr7_116900913_f	0	+
-chr7	116945541	116945787	CCDS5774.1_cds_0_0_chr7_116945542_r	0	-
-chr7	116952022	116952124	CCDS5774.1_cds_1_0_chr7_116952023_r	0	-
-chr7	116953508	116953641	CCDS5774.1_cds_2_0_chr7_116953509_r	0	-
-chr7	116955071	116955135	CCDS5774.1_cds_3_0_chr7_116955072_r	0	-
-chr7	116958551	116958737	CCDS5774.1_cds_4_0_chr7_116958552_r	0	-
-chr7	116959056	116959262	CCDS5774.1_cds_5_0_chr7_116959057_r	0	-
-chr7	116962093	116962272	CCDS5774.1_cds_6_0_chr7_116962094_r	0	-
-chr7	116968917	116969105	CCDS5774.1_cds_7_0_chr7_116968918_r	0	-
-chr7	116969273	116969426	CCDS5774.1_cds_8_0_chr7_116969274_r	0	-
-chr7	116979835	116979935	CCDS5774.1_cds_9_0_chr7_116979836_r	0	-
-chr7	116980017	116980104	CCDS5774.1_cds_10_0_chr7_116980018_r	0	-
-chr7	116990559	116990639	CCDS5774.1_cds_11_0_chr7_116990560_r	0	-
-chr7	116991879	116991975	CCDS5774.1_cds_12_0_chr7_116991880_r	0	-
-chr7	116994439	116994715	CCDS5774.1_cds_13_0_chr7_116994440_r	0	-
-chr7	117001063	117001181	CCDS5774.1_cds_14_0_chr7_117001064_r	0	-
-chr7	117011515	117011770	CCDS5774.1_cds_15_0_chr7_117011516_r	0	-
-chr7	117014445	117014596	CCDS5774.1_cds_16_0_chr7_117014446_r	0	-
-chr7	117016866	117016966	CCDS5774.1_cds_17_0_chr7_117016867_r	0	-
-chr7	117018255	117018459	CCDS5774.1_cds_18_0_chr7_117018256_r	0	-
-chr7	117025132	117026786	CCDS5774.1_cds_19_0_chr7_117025133_r	0	-
-chr7	117044769	117044994	CCDS5774.1_cds_20_0_chr7_117044770_r	0	-
-chr7	117095213	117095321	CCDS5774.1_cds_21_0_chr7_117095214_r	0	-
-chr7	117107339	117107420	CCDS5774.1_cds_22_0_chr7_117107340_r	0	-
-chr5	131424298	131424460	CCDS4149.1_cds_0_0_chr5_131424299_f	0	+
-chr5	131424558	131424600	CCDS4149.1_cds_1_0_chr5_131424559_f	0	+
-chr5	131425903	131425993	CCDS4149.1_cds_2_0_chr5_131425904_f	0	+
-chr5	131426117	131426159	CCDS4149.1_cds_3_0_chr5_131426118_f	0	+

test-data/taxonomyGI.tabular

@@ -1,5 +0,0 @@
-33001686	9443
-23236241	9604
-12583	9606
-410771	40674
-2286205	63221

test-data/users/test3@bx.psu.edu/run1/2.fasta

@@ -1,11 +0,0 @@
->Sequence 561 BP; 135 A; 106 C; 98 G; 222 T; 0 other;
-gttcgatgcc taaaatacct tcttttgtcc ctacacagac cacagttttc ctaatggctt
-tacaccgact agaaattctt gtgcaagcac taattgaaag cggttggcct agagtgttac
-cggtttgtat agctgagcgc gtctcttgcc ctgatcaaag gttcattttc tctactttgg
-aagacgttgt ggaagaatac aacaagtacg agtctctccc ccctggtttg ctgattactg
-gatacagttg taataccctt cgcaacaccg cgtaactatc tatatgaatt attttccctt
-tattatatgt agtaggttcg tctttaatct tcctttagca agtcttttac tgttttcgac
-ctcaatgttc atgttcttag gttgttttgg ataatatgcg gtcagtttaa tcttcgttgt
-ttcttcttaa aatatttatt catggtttaa tttttggttt gtacttgttc aggggccagt
-tcattattta ctctgtttgt atacagcagt tcttttattt ttagtatgat tttaatttaa
-aacaattcta atggtcaaaa a

test/base/twilltestcase.py

@@ -335,6 +335,7 @@ class TwillTestCase( unittest.TestCase ):
             self.visit_page( "history?show_deleted=False" )
         else:
             self.visit_page( "history" )
+        json_data = {}
         try:
             tc.find( pattern, flags=( 'm' if multiline else '' ) )
             # twill stores the regex match in a special stack variable
@@ -541,7 +542,6 @@ class TwillTestCase( unittest.TestCase ):
         self.home()
         self.visit_page( "history/list" )
         self.check_page_for_string( 'Saved Histories' )
-        self.check_page_for_string( '<input type="checkbox" name="id" value=' )
         self.check_page_for_string( 'operation=Rename' )
         self.check_page_for_string( 'operation=Switch' )
         self.check_page_for_string( 'operation=Delete' )
@@ -553,7 +553,6 @@ class TwillTestCase( unittest.TestCase ):
         self.home()
         self.visit_page( "history/list?f-deleted=True" )
         self.check_page_for_string( 'Saved Histories' )
-        self.check_page_for_string( '<input type="checkbox" name="id" value=' )
         self.check_page_for_string( 'operation=Undelete' )
         for check_str in strings_displayed:
             self.check_page_for_string( check_str )

test/casperjs/anon-history-tests.js

@@ -54,27 +54,16 @@ var tooltipSelector     = spaceghost.data.selectors.tooltipBalloon,
     anonNameTooltip     = spaceghost.historypanel.data.text.anonymous.tooltips.name;
 
 var historyFrameInfo = {},
-    filepathToUpload = '../../test-data/1.txt',
+    filenameToUpload = '1.txt',
+    filepathToUpload = '../../test-data/' + filenameToUpload,
     testUploadInfo = {};
 
 
 // =================================================================== TESTS
-// ------------------------------------------------------------------- anonymous new, history
-// open galaxy - ensure not logged in
-spaceghost.thenOpen( spaceghost.baseUrl, function(){
-    var loggedInAs = spaceghost.user.loggedInAs();
-    this.debug( 'loggedInAs: ' + loggedInAs );
-    if( loggedInAs ){ this.logout(); }
-});
-
-// ------------------------------------------------------------------- check the empty history for well formedness
-spaceghost.historypanel.waitForHdas( function testPanelStructure(){
+// ------------------------------------------------------------------- check the anonymous new, history for form
+spaceghost.openHomePage().historypanel.waitForHdas( function testPanelStructure(){
     this.test.comment( 'history panel for anonymous user, new history' );
 
-    this.test.comment( "frame should have proper url and title: 'History'" );
-    this.test.assertMatch( this.getCurrentUrl(), /\/history/, 'Found history frame url' );
-    this.test.assertTitle( this.getTitle(), 'History', 'Found history frame title' );
-
     this.test.comment( "history name should exist, be visible, and have text " + unnamedName );
     this.test.assertExists( nameSelector, nameSelector + ' exists' );
     this.test.assertVisible( nameSelector, 'History name is visible' );
@@ -96,11 +85,6 @@ spaceghost.historypanel.waitForHdas( function testPanelStructure(){
     this.test.assertSelectorHasText( emptyMsgSelector, emptyMsgStr,
         'Message contains "' + emptyMsgStr + '"' );
 
-    this.test.comment( 'name should have a tooltip with info on anon-user name editing' );
-    this.historypanel.hoverOver( nameSelector );
-    this.test.assertExists( tooltipSelector, "Found tooltip after name hover" );
-    this.test.assertSelectorHasText( tooltipSelector, anonNameTooltip );
-
     this.test.comment( 'name should NOT be editable when clicked by anon-user' );
     this.assertDoesntHaveClass( nameSelector, editableTextClass, "Name field is not classed as editable text" );
     this.click( nameSelector );
@@ -115,10 +99,11 @@ spaceghost.then( function testAnonUpload(){
         this.debug( 'uploaded HDA info: ' + this.jsonStr( this.quickInfo( _uploadInfo.hdaElement ) ) );
         var hasHda = _uploadInfo.hdaElement,
             hasClass = _uploadInfo.hdaElement.attributes[ 'class' ],
-            hasOkClass = _uploadInfo.hdaElement.attributes[ 'class' ].indexOf( 'historyItem-ok' ) !== -1;
+            hasOkClass = _uploadInfo.hdaElement.attributes[ 'class' ].indexOf( 'state-ok' ) !== -1;
         this.test.assert( ( hasHda && hasClass && hasOkClass ), "Uploaded file: " + _uploadInfo.hdaElement.text );
         testUploadInfo = _uploadInfo;
     });
+
 });
 spaceghost.then( function testAnonUpload(){
     this.test.comment( "empty should be NO LONGER be displayed" );
@@ -128,15 +113,14 @@ spaceghost.then( function testAnonUpload(){
 // ------------------------------------------------------------------- anon user can run tool on file
 
 // ------------------------------------------------------------------- anon user registers/logs in -> same history
-spaceghost.user.loginOrRegisterUser( email, password );
-spaceghost.thenOpen( spaceghost.baseUrl, function(){
+spaceghost.user.loginOrRegisterUser( email, password ).openHomePage( function(){
     this.test.comment( 'anon-user should login and be associated with previous history' );
 
     var loggedInAs = spaceghost.user.loggedInAs();
     this.test.assert( loggedInAs === email, 'loggedInAs() matches email: "' + loggedInAs + '"' );
 
     this.historypanel.waitForHdas( function(){
-        var hdaInfo = this.historypanel.hdaElementInfoByTitle( testUploadInfo.hdaElement.text );
+        var hdaInfo = this.historypanel.hdaElementInfoByTitle( filenameToUpload );
         this.test.assert( hdaInfo !== null, "After logging in - found a matching hda by name and hid" );
         if( hdaInfo ){
             this.test.assert( testUploadInfo.hdaElement.attributes.id === hdaInfo.attributes.id,
@@ -146,8 +130,7 @@ spaceghost.thenOpen( spaceghost.baseUrl, function(){
 });
 
 // ------------------------------------------------------------------- logs out -> new history
-spaceghost.user.logout();
-spaceghost.thenOpen( spaceghost.baseUrl, function(){
+spaceghost.user.logout().openHomePage( function(){
     this.test.comment( 'logging out should create a new, anonymous history' );
 
     this.historypanel.waitForHdas( function(){

test/casperjs/api-history-tests.js

@@ -304,13 +304,15 @@ spaceghost.thenOpen( spaceghost.baseUrl ).then( function(){
         //this.debug( 'returned:\n' + this.jsonStr( returned ) );
         historyShow = this.api.histories.show( newFirstHistory.id );
         this.test.assert( historyShow.genome_build === unicodeBuild,
-            "Update accepted unicode genome_build: " + historyShow.name );
+            "Update accepted unicode genome_build: " + historyShow.genome_build );
     } catch( err ){
         //this.debug( this.jsonStr( err ) );
         if( ( err instanceof this.api.APIError )
         &&  ( err.status === 500 )
         &&  ( err.message.indexOf( '(ProgrammingError) You must not use 8-bit bytestrings' ) !== -1 ) ){
             this.skipTest( 'Unicode update failed. Are you using sqlite3 as the db?' );
+        } else {
+            throw err;
         }
     }
 

test/casperjs/hda-state-tests.js

@@ -56,23 +56,20 @@ var utils = require( 'utils' ),
     filepathToUpload = '../../test-data/1.txt',
     testUploadInfo = {},
     //TODO: get from the api module - that doesn't exist yet
-    summaryShouldBeArray = [ '10 lines', 'format: txt' ],
+    summaryShouldBeArray = [ '10 lines', 'format', 'txt' ],
     infoShouldBe = 'uploaded txt file',
     metadataFiles = null,
     peekShouldBeArray = [];
 
 // ------------------------------------------------------------------- set up
-// start a new user
+// start a new user and upload a file
 spaceghost.user.loginOrRegisterUser( email, password );
-
-// upload a file
 spaceghost.then( function upload(){
     spaceghost.tools.uploadFile( filepathToUpload, function uploadCallback( _uploadInfo ){
         testUploadInfo = _uploadInfo;
     });
 });
 
-
 // =================================================================== TEST HELPERS
 //NOTE: to be called with fn.call( spaceghost, ... )
 
@@ -104,13 +101,14 @@ function testIconButton( hdaDbId, containerSelector, buttonName, expectedButtonD
 
     if( expectedButtonData.hrefTpl ){
         var href = buttonElement.attributes.href,
-            hrefShouldBe = utils.format( expectedButtonData.hrefTpl, hdaDbId );
+            hrefShouldBe = ( expectedButtonData.hrefTpl.indexOf( '%s' ) !== -1 )?
+                ( utils.format( expectedButtonData.hrefTpl, hdaDbId ) ):( expectedButtonData.hrefTpl );
         this.assertTextContains( href, hrefShouldBe,
             buttonName + ' has proper href (' + hrefShouldBe + '): ' + href );
     }
 
     if( expectedButtonData.tooltip ){
-        this.historypanel.hoverOver( btnSelector );
+        this.hoverOver( btnSelector );
         var tooltipText = expectedButtonData.tooltip;
         this.test.assertVisible( tooltipSelector, buttonName + ' button tooltip is visible when hovering' );
         this.test.assertSelectorHasText( tooltipSelector, tooltipText,
@@ -159,7 +157,7 @@ function testDbkey( hdaSelector, dbkeySetTo ){
         this.test.assertSelectorHasText( dbkeySelector, dbkeySetTo,
             'dbkey is specified: ' + dbkey.text );
         this.test.assert( dbkey.nodeName === specifiedDbkeyNodeName,
-            'dbkey has proper nodeName (' + specifiedDbkeyNodeName + '):' + dbkey.nodeName );
+            'dbkey has proper nodeName (' + specifiedDbkeyNodeName + '): ' + dbkey.nodeName );
 
     // dbkey expected to be not set
     } else {
@@ -305,7 +303,7 @@ function testExpandedBody( hdaSelector, expectedSummaryTextArray, expectedInfoTe
 
 // =================================================================== TESTS
 // ------------------------------------------------------------------- ok state
-spaceghost.withHistoryPanel( function(){
+spaceghost.then( function(){
     this.test.comment( 'HDAs in the "ok" state should be well formed' );
 
     var uploadSelector = '#' + testUploadInfo.hdaElement.attributes.id;
@@ -329,13 +327,10 @@ spaceghost.withHistoryPanel( function(){
 
     this.test.comment( 'clicking the hda title should expand its body' );
     this.historypanel.thenExpandHda( uploadSelector, function(){
-        // ugh.
-        this.jumpToHistory( function(){
-            testExpandedBody.call( spaceghost, uploadSelector,
-                summaryShouldBeArray, infoShouldBe, false, metadataFiles );
-            //testExpandedBody.call( spaceghost, uploadSelector,
-            //    summaryShouldBeArray, infoShouldBe, false );
-        });
+        testExpandedBody.call( spaceghost, uploadSelector,
+            summaryShouldBeArray, infoShouldBe, false, metadataFiles );
+        //testExpandedBody.call( spaceghost, uploadSelector,
+        //    summaryShouldBeArray, infoShouldBe, false );
     });
 });
 // restore to collapsed
@@ -350,7 +345,7 @@ spaceghost.then( function(){
 });
 
 // ------------------------------------------------------------------- new state
-spaceghost.withHistoryPanel( function(){
+spaceghost.then( function(){
     // set state directly through model, wait for re-render
     //TODO: not ideal to test this
     this.evaluate( function(){
@@ -385,7 +380,7 @@ spaceghost.withHistoryPanel( function(){
                 'HDA body has text: ' + expectedBodyText );
         });
 
-        this.historypanel.then( function(){
+        this.then( function(){
             this.test.comment( 'a simulated error on a new dataset should appear in a message box' );
             // datasets that error on fetching their data appear as 'new', so do this here
             // more of a unit test, but ok
@@ -410,7 +405,7 @@ spaceghost.withHistoryPanel( function(){
     });
 });
 // restore state, collapse
-spaceghost.withHistoryPanel( function revertStateAndCollapse(){
+spaceghost.then( function revertStateAndCollapse(){
     var uploadSelector = '#' + testUploadInfo.hdaElement.attributes.id;
 
     this.historypanel.thenCollapseHda( uploadSelector, function(){

test/casperjs/history-options-tests.js

@@ -47,9 +47,8 @@ var filepathToUpload = '../../test-data/1.txt',
 
 // =================================================================== TESTS
 // ------------------------------------------------------------------- set up
-// start a new user
+// start a new user and upload a file
 spaceghost.user.loginOrRegisterUser( email, password );
-
 spaceghost.tools.uploadFile( filepathToUpload, function uploadCallback( _uploadInfo ){
     testUploadInfo = _uploadInfo;
 });
@@ -114,8 +113,15 @@ spaceghost.then( function(){
         this.test.assertDoesntExist( '#' + testUploadInfo.hdaElement.attributes.id,
             "Deleted HDA is not in the DOM (using history options -> " + includeDeletedOptionsLabel + ")" );
     });
+});
+
+spaceghost.then( function(){
     // undelete the uploaded file
-    this.historypanel.undeleteHda( '#' + testUploadInfo.hdaElement.attributes.id );
+    this.historyoptions.includeDeleted( function(){
+        this.historypanel.undeleteHda( '#' + testUploadInfo.hdaElement.attributes.id );
+        spaceghost.debug( 'undeleted' );
+    });
+
 });
 
 // ------------------------------------------------------------------- hidden hdas aren't shown
@@ -132,10 +138,8 @@ spaceghost.then( function(){
 
     this.historyoptions.collapseExpanded( function(){
         var uploadedSelector = '#' + testUploadInfo.hdaElement.attributes.id;
-        this.withHistoryPanel( function(){
-            this.test.assertNotVisible( uploadedSelector + ' ' + this.historypanel.data.selectors.hda.body,
-                "Body for uploaded file is not visible" );
-        });
+        this.test.assertNotVisible( uploadedSelector + ' ' + this.historypanel.data.selectors.hda.body,
+            "Body for uploaded file is not visible" );
     });
 });
 

test/casperjs/history-panel-tests.js

@@ -40,7 +40,7 @@ if( spaceghost.fixtureData.testUser ){
 // selectors and labels
 var tooltipSelector     = spaceghost.data.selectors.tooltipBalloon,
     editableTextClass   = spaceghost.data.selectors.editableText,
-    editableTextInput   = spaceghost.data.selectors.editableTextInput,
+    editableTextInput   = spaceghost.historypanel.data.selectors.history.nameEditableTextInput,
 
     nameSelector     = spaceghost.historypanel.data.selectors.history.name,
     subtitleSelector = spaceghost.historypanel.data.selectors.history.subtitle,
@@ -56,8 +56,7 @@ var tooltipSelector     = spaceghost.data.selectors.tooltipBalloon,
     nameTooltip      = spaceghost.historypanel.data.text.history.tooltips.name,
 
     refreshButtonSelector       = 'a#history-refresh-button',
-    refreshButtonIconSelector   = 'span.fa-icon-refresh',
-    refreshButtonHref           = '/history',
+    refreshButtonIconSelector   = 'span.fa-refresh',
 
     includeDeletedOptionsLabel = spaceghost.historyoptions.data.labels.options.includeDeleted;
 
@@ -74,11 +73,8 @@ var newHistoryName = "Test History",
 spaceghost.user.loginOrRegisterUser( email, password );
 
 // ------------------------------------------------------------------- check structure of empty history
-spaceghost.thenOpen( spaceghost.baseUrl ).historypanel.waitForHdas( function(){
+spaceghost.openHomePage().historypanel.waitForHdas( function(){
     this.test.comment( 'history panel with a new, empty history should be well formed' );
-    this.test.comment( "frame should have proper url and title: 'History'" );
-    this.test.assertMatch( this.getCurrentUrl(), /\/history/, 'Found history frame url' );
-    this.test.assertTitle( this.getTitle(), 'History', 'Found history frame title' );
 
     this.test.comment( "history name should exist, be visible, and have text " + unnamedName );
     this.test.assertExists( nameSelector, nameSelector + ' exists' );
@@ -103,11 +99,11 @@ spaceghost.thenOpen( spaceghost.baseUrl ).historypanel.waitForHdas( function(){
 });
 
 // ------------------------------------------------------------------- name editing
-spaceghost.withHistoryPanel( function(){
+spaceghost.then( function(){
     this.test.comment( 'history panel, editing the history name' );
 
     this.test.comment( 'name should have a tooltip with proper info on name editing' );
-    this.historypanel.hoverOver( nameSelector );
+    this.hoverOver( nameSelector );
     this.test.assertExists( tooltipSelector, "Found tooltip after name hover" );
     this.test.assertSelectorHasText( tooltipSelector, nameTooltip );
     // clear the tooltip
@@ -116,6 +112,7 @@ spaceghost.withHistoryPanel( function(){
     this.test.comment( 'name should be create an input when clicked' );
     this.assertHasClass( nameSelector, editableTextClass, "Name field classed for editable text" );
     this.click( nameSelector );
+    spaceghost.debug( editableTextInput );
     this.test.assertExists( editableTextInput, "Clicking on name creates an input" );
 
     this.test.comment( 'name should be editable by entering keys and pressing enter' );
@@ -129,7 +126,7 @@ spaceghost.withHistoryPanel( function(){
     });
 });
 
-spaceghost.withHistoryPanel( function(){
+spaceghost.then( function(){
     this.test.comment( 'name should revert if user clicks away while editing' );
 
     this.click( nameSelector );
@@ -142,7 +139,7 @@ spaceghost.withHistoryPanel( function(){
     });
 });
 
-spaceghost.withHistoryPanel( function(){
+spaceghost.then( function(){
     this.test.comment( 'name should revert if user hits ESC while editing' );
 
     this.click( nameSelector );
@@ -168,7 +165,7 @@ spaceghost.tools.uploadFile( filepathToUpload, function uploadCallback( _uploadI
     testUploadInfo = _uploadInfo;
 });
 
-spaceghost.withHistoryPanel( function checkPanelStructure(){
+spaceghost.then( function checkPanelStructure(){
     this.test.comment( 'checking structure of non-empty panel' );
 
     this.test.comment( "history name should exist, be visible, and have text " + unnamedName );
@@ -176,9 +173,9 @@ spaceghost.withHistoryPanel( function checkPanelStructure(){
     this.test.assertVisible( nameSelector, 'History name is visible' );
     this.test.assertSelectorHasText( nameSelector, newHistoryName, 'History name is ' + newHistoryName );
 
-    this.test.comment( "history subtitle should display size and size should be " + onetxtFilesize + " bytes" );
     var onetxtFilesize = require( 'fs' ).size( this.options.scriptDir + filepathToUpload ),
         expectedSubtitle = onetxtFilesize + ' bytes';
+    this.test.comment( "history subtitle should display size and size should be " + onetxtFilesize + " bytes" );
     this.test.assertExists( subtitleSelector, 'Found ' + subtitleSelector );
     this.test.assertVisible( subtitleSelector, 'History subtitle is visible' );
     this.test.assertSelectorHasText( subtitleSelector, expectedSubtitle,
@@ -196,7 +193,7 @@ spaceghost.withHistoryPanel( function checkPanelStructure(){
 // ------------------------------------------------------------------- tags
 // keeping this light here - better for it's own test file
 //TODO: check tooltips
-spaceghost.withHistoryPanel( function openTags(){
+spaceghost.then( function openTags(){
     this.test.comment( 'tag area should open when the history panel tag icon is clicked' );
 
     this.click( tagIconSelector );
@@ -204,7 +201,7 @@ spaceghost.withHistoryPanel( function openTags(){
         this.test.assertVisible( tagAreaSelector, 'Tag area is now displayed' );
     });
 });
-spaceghost.withHistoryPanel( function closeAnnotation(){
+spaceghost.then( function closeAnnotation(){
     this.test.comment( 'annotation area should close when the history panel tag icon is clicked again' );
 
     this.click( tagIconSelector );
@@ -216,7 +213,7 @@ spaceghost.withHistoryPanel( function closeAnnotation(){
 // ------------------------------------------------------------------- annotation
 // keeping this light here - better for it's own test file
 //TODO: check tooltips
-spaceghost.withHistoryPanel( function openAnnotation(){
+spaceghost.then( function openAnnotation(){
     this.test.comment( 'annotation area should open when the history panel annotation icon is clicked' );
 
     this.click( annoIconSelector );
@@ -224,7 +221,7 @@ spaceghost.withHistoryPanel( function openAnnotation(){
         this.test.assertVisible( annoAreaSelector, 'Annotation area is now displayed' );
     });
 });
-spaceghost.withHistoryPanel( function closeAnnotation(){
+spaceghost.then( function closeAnnotation(){
     this.test.comment( 'annotation area should close when the history panel tag icon is clicked again' );
 
     this.click( annoIconSelector );
@@ -240,12 +237,10 @@ spaceghost.then( function refreshButton(){
     this.test.assertExists(  refreshButtonSelector, "Found refresh button" );
     this.test.assertVisible( refreshButtonSelector, "Refresh button is visible" );
     this.test.assertVisible( refreshButtonSelector + ' ' + refreshButtonIconSelector, "Refresh icon is visible" );
-    this.test.assert( this.getElementAttribute( refreshButtonSelector, 'href' ) === refreshButtonHref,
-        "Refresh button has href: " + refreshButtonHref );
 
-    this.assertNavigationRequested( refreshButtonHref, "History refreshed when clicking refresh icon", function(){
-        this.click( refreshButtonSelector );
-    });
+    //this.assertNavigationRequested( refreshButtonHref, "History refreshed when clicking refresh icon", function(){
+    //    this.click( refreshButtonSelector );
+    //});
 });
 
 // ------------------------------------------------------------------- hdas can be expanded by clicking on the hda name
@@ -275,7 +270,7 @@ spaceghost.then( function(){
 });
 
 // ------------------------------------------------------------------- expanded hdas collapse by clicking name again
-spaceghost.withHistoryPanel( function(){
+spaceghost.then( function(){
     this.test.comment( 'Expanded hdas collapse by clicking name again' );
     var uploadedSelector = '#' + testUploadInfo.hdaElement.attributes.id;
 

test/casperjs/login-tests.js

@@ -1,5 +1,3 @@
-// have to handle errors here - or phantom/casper won't bail but _HANG_
-//TODO: global error handler?
 try {
     var utils = require( 'utils' ),
         xpath = require( 'casper' ).selectXPath,
@@ -25,12 +23,6 @@ try {
     phantom.exit( 1 );
 }
 
-// ===================================================================
-/* TODO:
-    move selectors and assertText strings into global object for easier editing
-
-
-*/
 // =================================================================== globals and helpers
 var email = spaceghost.user.getRandomEmail(),
     password = '123456';
@@ -39,22 +31,17 @@ if( spaceghost.fixtureData.testUser ){
     password = spaceghost.fixtureData.testUser.password;
 }
 
-var userEmailSelector = '//a[contains(text(),"Logged in as")]/span["id=#user-email"]';
+var userEmailSelector = '//a[contains(text(),"Logged in as")]';
 
 // =================================================================== TESTS
 // register a user (again...)
-spaceghost.thenOpen( spaceghost.baseUrl, function(){
-    this.test.comment( 'registering: ' + email );
-    spaceghost.user.registerUser( email, password );
-});
+spaceghost.openHomePage()
+    .user.registerUser( email, password )
+    .user.logout();
 
-// log them out - check for empty logged in text
-spaceghost.then( function(){
-    this.test.comment( 'logging out: ' + email );
-    spaceghost.user.logout();
-});
-spaceghost.then( function(){
-    this.test.assertSelectorDoesntHaveText( xpath( userEmailSelector ), /\w/ );
+spaceghost.openHomePage( function(){
+    this.test.comment( 'log out should be reflected in user menu' );
+    this.test.assertDoesntExist( xpath( userEmailSelector ) );
     this.test.assert( spaceghost.user.loggedInAs() === '', 'loggedInAs() is empty string' );
 });
 
@@ -63,16 +50,13 @@ spaceghost.then( function(){
     this.test.comment( 'logging back in: ' + email );
     spaceghost.user._submitLogin( email, password ); //No such user
 });
-spaceghost.thenOpen( spaceghost.baseUrl, function(){
+spaceghost.openHomePage( function(){
     this.test.assertSelectorHasText( xpath( userEmailSelector ), email );
     this.test.assert( spaceghost.user.loggedInAs() === email, 'loggedInAs() matches email' );
 });
 
 // finally log back out for next tests
-spaceghost.then( function(){
-    this.test.comment( 'logging out: ' + email );
-    spaceghost.user.logout();
-});
+spaceghost.user.logout();
 
 // ------------------------------------------------------------------- shouldn't work
 // can't log in: users that don't exist, bad emails, sql injection (hurhur)
@@ -118,10 +102,6 @@ spaceghost.then( function(){
         });
     });
 });
-
-spaceghost.then( function(){
-    this.user.logout();
-});
 /*
 */
 // ===================================================================

test/casperjs/modules/api.js

@@ -1,12 +1,4 @@
 // =================================================================== module object, exports
-/** Creates a new api module object.
- *  @param {SpaceGhost} spaceghost a spaceghost instance
- *  @exported
- */
-exports.create = function createAPI( spaceghost, apikey ){
-    return new API( spaceghost );
-};
-
 /** User object constructor.
  *  @param {SpaceGhost} spaceghost  a spaceghost instance
  *  @param {String} apikey          apikey for use when not using session authentication
@@ -28,14 +20,21 @@ var API = function API( spaceghost, apikey ){
 };
 exports.API = API;
 
+/** Creates a new api module object.
+ *  @param {SpaceGhost} spaceghost a spaceghost instance
+ *  @exported
+ */
+exports.create = function createAPI( spaceghost, apikey ){
+    return new API( spaceghost );
+};
+
+
 API.prototype.toString = function toString(){
     return ( this.spaceghost + '.API:'
         + (( this.apikey )?( this.apikey ):( '(session)' )) );
 };
 
 // ------------------------------------------------------------------- APIError
-APIError.prototype = new Error();
-APIError.prototype.constructor = Error;
 /** @class Thrown when Galaxy the API returns an error from a request */
 function APIError( msg, status ){
     Error.apply( this, arguments );
@@ -43,6 +42,8 @@ function APIError( msg, status ){
     this.message = msg;
     this.status = status;
 }
+APIError.prototype = new Error();
+APIError.prototype.constructor = Error;
 API.prototype.APIError = APIError;
 exports.APIError = APIError;
 

test/casperjs/modules/historyoptions.js

@@ -34,8 +34,11 @@ var xpath = require( 'casper' ).selectXPath;
  *  @returns {Any} the return value of fn
  */
 HistoryOptions.prototype.openMenu = function openMenu( fn ){
+
     return this.spaceghost.jumpToTop( function(){
-        this.click( this.historyoptions.data.selectors.button );
+        if( !spaceghost.visible( this.historyoptions.data.selectors.menu ) ){
+            this.click( this.historyoptions.data.selectors.button );
+        }
         return fn.call( this );
     });
 };
@@ -170,41 +173,32 @@ HistoryOptions.prototype.collapseExpanded = function collapseExpanded( then ){
  *  @param {Function} then  casper step to run when option is set
  */
 HistoryOptions.prototype.includeDeleted = function includeDeleted( then ){
-    return this.spaceghost.then( function(){
-        this.historyoptions.toggle( this.historyoptions.data.labels.options.includeDeleted, true );
-        this.historypanel.waitForHdas( then );
-    });
+    this.toggle( this.data.labels.options.includeDeleted, true );
+    this.spaceghost.historypanel.waitForHdas( then );
 };
 
 /** set 'Include Deleted Datasets' to off
  *  @param {Function} then  casper step to run when option is set
  */
 HistoryOptions.prototype.excludeDeleted = function excludeDeleted( then ){
-    return this.spaceghost.then( function(){
-        this.historyoptions.toggle( this.historyoptions.data.labels.options.includeDeleted, false );
-        //TODO:?? this puts in the history frame. Is that what we want?
-        this.historypanel.waitForHdas( then );
-    });
+    this.toggle( this.data.labels.options.includeDeleted, false );
+    this.spaceghost.historypanel.waitForHdas( then );
 };
 
 /** set 'Include Hidden Datasets' to on
  *  @param {Function} then  casper step to run when option is set
  */
 HistoryOptions.prototype.includeHidden = function includeHidden( then ){
-    return this.spaceghost.then( function(){
-        this.historyoptions.toggle( this.historyoptions.data.labels.options.includeHidden, true );
-        this.historypanel.waitForHdas( then );
-    });
+    this.toggle( this.data.labels.options.includeHidden, true );
+    this.spaceghost.historypanel.waitForHdas( then );
 };
 
 /** set 'Include Hidden Datasets' to off
  *  @param {Function} then  casper step to run when option is set
  */
 HistoryOptions.prototype.excludeHidden = function excludeHidden( then ){
-    return this.spaceghost.then( function(){
-        this.historyoptions.toggle( this.historyoptions.data.labels.options.includeHidden, false );
-        this.historypanel.waitForHdas( then );
-    });
+    this.toggle( this.data.labels.options.includeHidden, false );
+    this.spaceghost.historypanel.waitForHdas( then );
 };
 
 
@@ -213,14 +207,14 @@ HistoryOptions.prototype.excludeHidden = function excludeHidden( then ){
 HistoryOptions.prototype.data = {
     selectors : {
         button      : '#history-options-button',
-        buttonIcon  : '#history-options-button span.fa-icon-cog',
+        buttonIcon  : '#history-options-button span.fa-cog',
         menu        : '#history-options-button-menu',
         optionXpathByLabelFn : function optionXpathByLabelFn( label ){
             return xpath( '//ul[@id="history-options-button-menu"]/li/a[text()[contains(.,"' + label + '")]]' );
         },
         optionIsOnXpathByLabelFn : function optionIsOnXpathByLabelFn( label ){
             return xpath( '//ul[@id="history-options-button-menu"]/li/a[text()[contains(.,"' + label + '")]]'
-                        + '/span[@class="fa-icon-ok"]' );
+                        + '/span[@class="fa fa-check"]' );
         }
     },
     labels : {

test/casperjs/modules/historypanel.js

@@ -33,64 +33,38 @@ var xpath = require( 'casper' ).selectXPath;
 
 // =================================================================== API (external)
 // ------------------------------------------------------------------- frame control
-/** Is casper currently 'in' the history frame?
- *  @returns {Boolean}
- */
-HistoryPanel.prototype.inFrame = function inFrame(){
-    return ( ( this.spaceghost.page.frameName === this.spaceghost.data.selectors.frames.history )
-          && ( this.spaceghost.page.frameTitle === this.data.text.frameTitle ) );
-};
-
-/** Hover over an element in the history panel.
- *  @param {String} selector        a css or xpath selector for an historyItemWrapper
- */
-HistoryPanel.prototype.hoverOver = function hoverOver( selector ){
-    var spaceghost = this.spaceghost,
-        historyFrameInfo = spaceghost.jumpToTop( function(){
-            return spaceghost.getElementInfo( 'iframe[name="galaxy_history"]' );
-        });
-    //spaceghost.debug( 'historyFrameInfo:\n' + spaceghost.jsonStr( historyFrameInfo ) );
-
-    spaceghost.jumpToHistory( function(){
-        var elementInfo = spaceghost.getElementInfo( selector ),
-            newCoords = { x: ( historyFrameInfo.x + elementInfo.x ),
-                          y: ( historyFrameInfo.y + elementInfo.y ) };
-        spaceghost.page.sendEvent( 'mousemove', newCoords.x + 1, newCoords.y + 1 );
-    });
-    return spaceghost;
-};
+///** Hover over an element in the history panel.
+// *  @param {String} selector        a css or xpath selector for an historyItemWrapper
+// */
+//HistoryPanel.prototype.hoverOver = function hoverOver( selector ){
+//    var spaceghost = this.spaceghost,
+//        elementInfo = spaceghost.getElementInfo( selector );
+//    spaceghost.page.sendEvent( 'mousemove', elementInfo.x + 1, elementInfo.y + 1 );
+//    return spaceghost;
+//};
 
 // ------------------------------------------------------------------- hdas
-/** Parse the hid and name from an HDA title.
- *      NOTE: if more than one is found, will return the first found.
- *  @param {String} title   the title of the hda
- *  @returns {Object}       of the form { hid: <hid>, name: <name> }
- */
-HistoryPanel.prototype.hdaHidAndNameFromTitle = function hdaHidAndNameFromTitle( title ){
-    var sep = ': ', split = title.split( sep, 1 );
-    return {
-        name : (( split.length >= 2 )?( split[1] ):( split[0] )),
-        hid  : (( split.length >= 2 )?( parseInt( split[0], 10 ) ):( undefined ))
-    };
-};
+///** Parse the hid and name from an HDA title.
+// *      NOTE: if more than one is found, will return the first found.
+// *  @param {String} title   the title of the hda
+// *  @returns {Object}       of the form { hid: <hid>, name: <name> }
+// */
+//HistoryPanel.prototype.hdaHidAndNameFromTitle = function hdaHidAndNameFromTitle( title ){
+//    var sep = ': ', split = title.split( sep, 1 );
+//    return {
+//        name : (( split.length >= 2 )?( split[1] ):( split[0] )),
+//        hid  : (( split.length >= 2 )?( parseInt( split[0], 10 ) ):( undefined ))
+//    };
+//};
 
-/** Find the casper element info of the hda wrapper given the hda title and hid.
+/** Find the casper element info of the hda wrapper given the hda title.
  *      NOTE: if more than one is found, will return the first found.
  *  @param {String} title   the title of the hda
- *  @param {Int} hid        (optional) the hid of the hda to look for
  *  @returns {Object|null} ElementInfo of the historyItemWrapper found, null if not found
  */
-HistoryPanel.prototype.hdaElementInfoByTitle = function hdaElementInfoByTitle( title, hid ){
-    var spaceghost = this.spaceghost,
-        titleContains = ( hid !== undefined )?( hid + ': ' + title ):( title );
-    //TODO: this would fail if we named/uploaded a file that has quotations -
-    //  we can use underscore.un/escape to solve that
-
-    return spaceghost.jumpToHistory( function getWrapperInfo(){
-        var wrapperXpath = xpath( '//span[@class="historyItemTitle" ' +
-                                  'and contains(text(),"' + titleContains + '")]/../..' );
-        return spaceghost.elementInfoOrNull( wrapperXpath );
-    });
+HistoryPanel.prototype.hdaElementInfoByTitle = function hdaElementInfoByTitle( title ){
+    var wrapperXpath = xpath( '//span[@class="dataset-name" and contains(text(),"' + title + '")]/../../..' );
+    return this.spaceghost.elementInfoOrNull( wrapperXpath );
 };
 
 /** Get the state string of the given hda.
@@ -99,12 +73,10 @@ HistoryPanel.prototype.hdaElementInfoByTitle = function hdaElementInfoByTitle( t
  *  @returns {String|undefined}  class string of the historyItemWrapper found, undefined if not found or set
  */
 HistoryPanel.prototype.getHdaState = function getHdaState( hdaSelector ){
-    return this.spaceghost.jumpToHistory( function(){
-        var found = null,
-            hdaInfo = this.elementInfoOrNull( hdaSelector );
-        if( !hdaInfo ){ return undefined; }
-        return (( found = hdaInfo.attributes[ 'class' ].match( /historyItem\-(\w+)/ ) )?( found[1] ):( undefined ));
-    });
+    var found = null,
+        hdaInfo = this.spaceghost.elementInfoOrNull( hdaSelector );
+    if( !hdaInfo ){ return undefined; }
+    return (( found = hdaInfo.attributes[ 'class' ].match( /state\-(\w+)/ ) )?( found[1] ):( undefined ));
 };
 
 /** Get the encoded database/API id of the given hda.
@@ -119,20 +91,20 @@ HistoryPanel.prototype.getHdaEncodedId = function getHdaEncodedId( hdaSelector )
 };
 
 // ------------------------------------------------------------------- step functions
-/** Version of Casper#withFrame for the history iframe.
- *      Hopefully will allow easier test transition if/when frames are removed
- *      (i.e. -> just call the function).
- *      NOTE: is more than one Casper step.
- *  @param {Function} then  function called when in the history frame
- */
-HistoryPanel.prototype.then = function then( thenFn ){
-    if( this.inFrame() ){
-        thenFn.call( this.spaceghost );
-    } else {
-        this.spaceghost.withHistoryPanel( thenFn );
-    }
-};
-
+///** Version of Casper#withFrame for the history iframe.
+// *      Hopefully will allow easier test transition if/when frames are removed
+// *      (i.e. -> just call the function).
+// *      NOTE: is more than one Casper step.
+// *  @param {Function} then  function called when in the history frame
+// */
+//HistoryPanel.prototype.then = function then( thenFn ){
+//    if( this.inFrame() ){
+//        thenFn.call( this.spaceghost );
+//    } else {
+//        this.spaceghost.withHistoryPanel( thenFn );
+//    }
+//};
+//
 /** Moves into history iframe and waits until hdas are visible or empty message is.
  *      NOTE: is more than one Casper step.
  *  @see Casper@waitFor
@@ -140,13 +112,13 @@ HistoryPanel.prototype.then = function then( thenFn ){
 HistoryPanel.prototype.waitForHdas = function waitForHdas( then, timeout, maxWait ){
     //TODO:?? should this wait until the seletors are in AND they are opaque?
     var spaceghost = this.spaceghost;
-    this.then( function waitingForHdas(){
+    spaceghost.then( function waitingForHdas(){
         this.waitFor(
             function checkHpanel(){
-                // trying a subtitle opacity test for page.isloaded
                 var subtitleOpacity = this.evaluate( function( selector ){
                     return $( selector ).css( 'opacity' );
                 }, this.historypanel.data.selectors.history.subtitle );
+                // wait until the subtitle is faded in and either the hdas or the empty history msg is displayed
                 return ( subtitleOpacity !== 1
                        && ( ( this.visible( this.historypanel.data.selectors.hda.wrapper.itemClass ) )
                           ||( this.visible( this.historypanel.data.selectors.history.emptyMsg ) ) ) );
@@ -164,7 +136,7 @@ HistoryPanel.prototype.waitForHdas = function waitForHdas( then, timeout, maxWai
  */
 HistoryPanel.prototype._thenExpandOrCollapseHda = function _thenExpandOrCollapseHda( hdaSelector, then, desiredClosed ){
     // using a step here (instead of a jump) bc we need the wait for function
-    this.then( function checkingHda(){
+    this.spaceghost.then( function checkingHda(){
         this.info( (( desiredClosed )?( 'collapsing' ):( 'expanding' )) + ' hda: ' + hdaSelector );
 
         // click to open if the body isn't visible and call wait to account for opening animation
@@ -218,7 +190,7 @@ HistoryPanel.prototype.waitForHdaState = function waitForHdaState( hdaSelector,
     var hpanel = this,
         spaceghost = this.spaceghost;
 
-    this.then( function(){
+    this.spaceghost.then( function(){
         // get initial state, cache old timeout, set new timeout
         var prevState = hpanel.getHdaState( hdaSelector ),
             oldWaitTimeout = spaceghost.options.waitTimeout;
@@ -269,7 +241,7 @@ HistoryPanel.prototype.waitForHdaState = function waitForHdaState( hdaSelector,
  *  @param {Function} timeoutFn     function to be called if/when the deleted attempted times out (optional)
  */
 HistoryPanel.prototype.deleteHda = function deleteHda( hdaSelector, whenDeletedFn, timeoutFn ){
-    this.then( function deletingHda(){
+    this.spaceghost.then( function deletingHda(){
         var hdaId = this.getElementInfo( hdaSelector ).attributes.id,
             deleteIconSelector = '#' + hdaId + ' ' + this.historypanel.data.hdaTitleButtons['delete'].selector;
         this.click( deleteIconSelector );
@@ -299,7 +271,7 @@ HistoryPanel.prototype.deleteHda = function deleteHda( hdaSelector, whenDeletedF
  */
 HistoryPanel.prototype.undeleteHda = function undeleteHda( hdaSelector, whenUndeletedFn ){
     this.spaceghost.historyoptions.includeDeleted( function(){
-        this.click( hdaSelector + ' ' + this.historypanel.data.selectors.history.undeleteLink );
+        this.click( hdaSelector + ' ' + this.historypanel.data.selectors.hda.undeleteLink );
         this.historyoptions.excludeDeleted( function(){
             this.info( 'hda undeleted: ' + hdaSelector );
             if( utils.isFunction( whenUndeletedFn ) ){ whenUndeletedFn.call( this ); }
@@ -314,39 +286,39 @@ HistoryPanel.prototype.data = {
     hdaTitleButtons : {
         // mixing text and selectors here
         display : {
-            selector : '.icon-button.display',
+            selector : '.icon-btn.dataset-display',
             tooltip  : 'View data',
             hrefTpl  : '/datasets/%s/display',
             nodeName : 'a'
         },
         edit : {
-            selector : '.icon-button.edit',
-            tooltip  : 'Edit Attributes',
+            selector : '.icon-btn.dataset-edit',
+            tooltip  : 'Edit attributes',
             hrefTpl  : '/datasets/%s/edit',
             nodeName : 'a'
         },
         'delete' : {
-            selector : '.icon-button.delete',
+            selector : '.icon-btn.dataset-delete',
             tooltip  : 'Delete',
-            hrefTpl  : '/datasets/%s/delete_async',
+            hrefTpl  : 'javascript:void(0);',
             nodeName : 'a'
         }
     },
     hdaPrimaryActionButtons : {
         download : {
-            selector : '.icon-button.disk',
+            selector : '.icon-btn.dataset-download-btn',
             tooltip  : 'Download',
             hrefTpl  : '/datasets/%s/display?to_ext=',
             nodeName : 'a'
         },
         info : {
-            selector : '.icon-button.information',
+            selector : '.icon-btn.dataset-params-btn',
             tooltip  : 'View details',
             hrefTpl  : '/datasets/%s/show_params',
             nodeName : 'a'
         },
         rerun : {
-            selector : '.icon-button.arrow-circle',
+            selector : '.icon-btn.dataset-rerun-btn',
             tooltip  : 'Run this job again',
             hrefTpl  : '/tool_runner/rerun?id=%s',
             nodeName : 'a'
@@ -356,39 +328,42 @@ HistoryPanel.prototype.data = {
     },
     selectors : {
         history : {
-            name        : 'div#history-name',
-            subtitle    : 'div#history-subtitle-area',
-            tagIcon     : '#history-tag.icon-button',
-            tagArea     : '#history-tag-area',
-            annoIcon    : '#history-annotate.icon-button',
-            annoArea    : '#history-annotation-area',
-            // weak
-            emptyMsg    : '.infomessagesmall',
-            undeleteLink : '.historyItemUndelete',
-            hdaContainer : '.history-datasets-list'
+            title       : '.history-title',
+            name        : '.history-title .history-name',
+            nameEditableTextInput : '.history-name input',
+            subtitle    : '.history-subtitle',
+            tagIcon     : '.history-secondary-actions .history-tag-btn',
+            tagArea     : '.history-controls .tags-display',
+            annoIcon    : '.history-secondary-actions .history-annotate-btn',
+            annoArea    : '.history-controls .annotation-display',
+            emptyMsg    : '.empty-history-message',
+            hdaContainer: '.datasets-list'
         },
         hda : {
             wrapper : {
-                itemClass   : '.historyItem',
+                itemClass   : '.hda',
                 stateClasses : {
-                    prefix  : 'historyItem-',
-                    ok      : 'historyItem-ok',
-                    'new'   : 'historyItem-new'
+                    prefix  : 'state-',
+                    ok      : 'state-ok',
+                    'new'   : 'state-new'
                 }
             },
             errorMessage    : '.errormessagesmall',
 
-            title           : '.historyItemTitle',
-            titleButtonArea : '.historyItemButtons',
-            body            : '.historyItemBody',
-            summary         : '.hda-summary',
-            dbkey           : '.metadata-dbkey',
-            info            : '.hda-info',
+            title           : '.dataset-title',
+            titleButtonArea : '.dataset-primary-actions',
+            summary         : '.dataset-summary',
+            dbkey           : '.dataset-dbkey .value',
+            info            : '.dataset-info',
+            body            : '.dataset-body',
             
-            primaryActionButtons    : 'div[id^="primary-actions"]',
-            secondaryActionButtons  : 'div[id^="secondary-actions"]',
+            primaryActionButtons    : '.dataset-actions .left',
+            secondaryActionButtons  : '.dataset-actions .right',
 
-            peek            : 'pre.peek'
+            undeleteLink    : '.dataset-undelete',
+            purgeLink       : '.dataset-purge',
+
+            peek            : '.dataset-peek'
         }
     },
     labels : {
@@ -399,7 +374,7 @@ HistoryPanel.prototype.data = {
     },
     text : {
         windowTitle : 'History',
-        frameTitle : 'Galaxy History',
+        //frameTitle : 'Galaxy History',
         anonymous : {
             tooltips : {
                 name    : 'You must be logged in to edit your history name'

test/casperjs/modules/tools.js

@@ -86,10 +86,10 @@ Tools.prototype._uploadFile = function _uploadFile( filepath ){
         uploadInfo = {};
     uploadInfo[ spaceghost.data.selectors.tools.upload.fileInput ] = filepath;
 
-    // click the upload tool
-    spaceghost.thenOpen( spaceghost.baseUrl, function(){
-        // we can apprently click a tool label without expanding the tool container for it
-        this.clickLabel( spaceghost.data.labels.tools.upload.panelLabel );
+    spaceghost.openHomePage( function(){
+        // load the upload tool form
+        // (we can apprently click a tool label without expanding the tool container for it)
+        this.click( xpath( '//a[contains(text(),"Upload File")]' ) );
         this.jumpToMain( function(){
             this.waitForSelector( 'body' );
         });
@@ -108,7 +108,7 @@ Tools.prototype._uploadFile = function _uploadFile( filepath ){
 
         // wait for main panel, history reload
         ////NOTE!: assumes tool execution reloads the history panel
-        this.waitForMultipleNavigation( [ 'tool_runner/upload_async_message', 'history' ],
+        this.waitForMultipleNavigation( [ 'tool_runner/upload_async_message' ],
             function thenAfterUploadRefreshes(){
                 // debugging
                 this.jumpToMain( function(){

test/casperjs/modules/user.js

@@ -43,11 +43,11 @@ User.prototype._submitRegistration = function _submitRegistration( email, passwo
             confirm : ( confirm !== undefined )?( confirm ):( password )
         };
 
-    spaceghost.thenOpen( spaceghost.baseUrl, function(){
-        this.waitForMasthead( function() {
-            this.clickLabel( spaceghost.data.labels.masthead.menus.user );
-            this.clickLabel( spaceghost.data.labels.masthead.userMenu.register );
+    spaceghost.openHomePage( function(){
+        this.clickLabel( spaceghost.data.labels.masthead.menus.user );
+        this.clickLabel( spaceghost.data.labels.masthead.userMenu.register );
 
+        this.waitForNavigation( 'user/create', function beforeRegister(){
             this.withMainPanel( function mainBeforeRegister(){
                 spaceghost.debug( '(' + spaceghost.getCurrentUrl() + ') registering user:\n'
                     + spaceghost.jsonStr( userInfo ) );
@@ -56,11 +56,13 @@ User.prototype._submitRegistration = function _submitRegistration( email, passwo
                 this.click( xpath( spaceghost.data.selectors.registrationPage.submit_xpath ) );
             });
         });
-        //// debugging
-        //spaceghost.withFrame( spaceghost.data.selectors.frames.main, function mainAfterRegister(){
-        //    var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all );
-        //    spaceghost.debug( 'post registration message:\n' + spaceghost.jsonStr( messageInfo ) );
-        //});
+
+        this.waitForNavigation( 'user/create', function afterRegister(){
+        //    this.withMainPanel( function mainAfterRegister(){
+        //        var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all );
+        //        spaceghost.debug( 'post registration message:\n' + spaceghost.jsonStr( messageInfo ) );
+        //    });
+        });
     });
 };
 
@@ -79,25 +81,23 @@ User.prototype._submitLogin = function _submitLogin( email, password ){
             password: password
         };
 
-    spaceghost.thenOpen( spaceghost.baseUrl, function(){
-        spaceghost.waitForMasthead( function() {
-            spaceghost.clickLabel( spaceghost.data.labels.masthead.menus.user );
-            spaceghost.clickLabel( spaceghost.data.labels.masthead.userMenu.login );
+    spaceghost.openHomePage( function(){
+        this.clickLabel( spaceghost.data.labels.masthead.menus.user );
+        this.clickLabel( spaceghost.data.labels.masthead.userMenu.login );
 
-            spaceghost.withMainPanel( function mainBeforeLogin(){
+        this.waitForNavigation( 'user/login', function beforeLogin(){
+            this.withMainPanel( function mainBeforeLogin(){
                 spaceghost.debug( '(' + spaceghost.getCurrentUrl() + ') logging in user:\n'
                     + spaceghost.jsonStr( loginInfo ) );
                 spaceghost.fill( spaceghost.data.selectors.loginPage.form, loginInfo, false );
                 spaceghost.click( xpath( spaceghost.data.selectors.loginPage.submit_xpath ) );
             });
+        });
 
-            //// debugging
-            //spaceghost.withFrame( spaceghost.data.selectors.frames.main, function mainAfterLogin(){
-            //    //TODO: prob. could use a more generalized form of this for url breakdown/checking
-            //    if( spaceghost.getCurrentUrl().search( spaceghost.data.selectors.loginPage.url_regex ) != -1 ){
-            //        var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all );
-            //        spaceghost.debug( 'post login message:\n' + spaceghost.jsonStr( messageInfo ) );
-            //    }
+        this.waitForNavigation( 'user/login', function afterLogin(){
+            //this.withMainPanel( function mainAfterLogin(){
+            //    var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all );
+            //    spaceghost.debug( 'post login message:\n' + spaceghost.jsonStr( messageInfo ) );
             //});
         });
     });
@@ -123,6 +123,9 @@ User.prototype.registerUser = function registerUser( email, password, username )
             this.warning( 'Registration failed: ' + messageInfo.text );
             throw new spaceghost.GalaxyError( 'RegistrationError: ' + messageInfo.text );
         }
+        
+        this.clickLabel( 'Return to the home page.' );
+        this.waitForNavigation( '' );
     });
     return spaceghost;
 };
@@ -136,19 +139,21 @@ User.prototype.login = function login( email, password ){
     var spaceghost = this.spaceghost;
 
     this._submitLogin( email, password );
-    spaceghost.waitForMasthead( function() {
-        spaceghost.withMainPanel( function mainAfterLogin(){
-            if( spaceghost.getCurrentUrl().search( spaceghost.data.selectors.loginPage.url_regex ) !== -1 ){
-                var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all );
-                if( messageInfo && messageInfo.attributes[ 'class' ] === 'errormessage' ){
-                    this.warning( 'Login failed: ' + messageInfo.text );
-                    throw new spaceghost.GalaxyError( 'LoginError: ' + messageInfo.text );
-                }
-            }
-            if( spaceghost.user.loggedInAs() === email ){
-                spaceghost.info( 'logged in as ' + email );
-            }
-        });
+    //spaceghost.withMainPanel( function mainAfterLogin(){
+    //    if( spaceghost.getCurrentUrl().search( spaceghost.data.selectors.loginPage.url_regex ) !== -1 ){
+    //        var messageInfo = spaceghost.getElementInfo( spaceghost.data.selectors.messages.all );
+    //        if( messageInfo && messageInfo.attributes[ 'class' ] === 'errormessage' ){
+    //            this.warning( 'Login failed: ' + messageInfo.text );
+    //            throw new spaceghost.GalaxyError( 'LoginError: ' + messageInfo.text );
+    //        }
+    //    }
+    //});
+    this.spaceghost.then( function checkLogin(){
+        if( spaceghost.user.loggedInAs() !== email ){
+            throw new spaceghost.GalaxyError( 'LoginError' );
+        } else {
+            spaceghost.info( 'logged in as ' + email );
+        }
     });
     return spaceghost;
 };
@@ -157,17 +162,19 @@ User.prototype.login = function login( email, password ){
  *  @returns {String} email of currently logged in user or '' if no one logged in
  */
 User.prototype.loggedInAs = function loggedInAs(){
-    var spaceghost = this.spaceghost,
-        userEmail = '';
-    try {
-        var loggedInInfo = spaceghost.getElementInfo(
-            xpath( spaceghost.data.selectors.masthead.userMenu.userEmail_xpath ) );
-        userEmail = loggedInInfo.text;
-    } catch( err ){
-        spaceghost.error( err );
-    }
-    //console.debug( 'loggedInInfo:', spaceghost.jsonStr( loggedInInfo ) );
-    return userEmail;
+    return this.spaceghost.jumpToTop( function(){
+        var userEmail = '';
+        try {
+            var emailSelector = xpath( this.data.selectors.masthead.userMenu.userEmail_xpath ),
+                loggedInInfo = this.elementInfoOrNull( emailSelector );
+            if( loggedInInfo !== null ){
+                userEmail = loggedInInfo.text.replace( 'Logged in as ', '' );
+            }
+        } catch( err ){
+            this.warn( err );
+        }
+        return userEmail;
+    });
 };
 
 /** Log out the current user
@@ -175,13 +182,15 @@ User.prototype.loggedInAs = function loggedInAs(){
  */
 User.prototype.logout = function logout(){
     var spaceghost = this.spaceghost;
-    spaceghost.thenOpen( spaceghost.baseUrl, function(){
-        this.info( 'user logging out' );
-        //TODO: handle already logged out
-        spaceghost.waitForMasthead( function _logout() {
+    this.spaceghost.openHomePage( function(){
+        if( spaceghost.user.loggedInAs() ){
             spaceghost.clickLabel( spaceghost.data.labels.masthead.menus.user );
             spaceghost.clickLabel( spaceghost.data.labels.masthead.userMenu.logout );
-        });
+            spaceghost.waitForNavigation( 'user/logout', function _toLogoutPage() {
+                spaceghost.clickLabel( 'go to the home page' );
+                spaceghost.waitForNavigation( '' );
+            });
+        }
     });
     return spaceghost;
 };
@@ -196,10 +205,10 @@ User.prototype.loginOrRegisterUser = function loginOrRegisterUser( email, passwo
     var spaceghost = this.spaceghost;
     // attempt a login, if that fails - register
     spaceghost.tryStepsCatch( function tryToLogin(){
-        spaceghost.open( spaceghost.baseUrl ).user.login( email, password );
+        spaceghost.openHomePage().user.login( email, password );
 
     }, function failedLoginRegister(){
-        spaceghost.open( spaceghost.baseUrl ).user.registerUser( email, password, username );
+        spaceghost.openHomePage().user.registerUser( email, password, username );
     });
     return spaceghost;
 };
@@ -214,7 +223,8 @@ User.prototype.getAdminData = function getAdminData(){
     // check for the setting in sg and the universe_wsgi.ini file
     var adminData = this.spaceghost.options.adminUser,
         iniAdminEmails = this.spaceghost.getUniverseSetting( 'admin_users' );
-    iniAdminEmails = ( iniAdminEmails )?( iniAdminEmails.split( ',' ).map( function( email ) { return email.trim(); } ) ):( null );
+    iniAdminEmails = ( iniAdminEmails )?
+        ( iniAdminEmails.split( ',' ).map( function( email ) { return email.trim(); } ) ):( null );
 
     //TODO: seems like we only need the wsgi setting - that's the only thing we can't change
     if( adminData ){

test/casperjs/registration-tests.js

@@ -1,4 +1,3 @@
-// have to handle errors here - or phantom/casper won't bail but _HANG_
 try {
     var utils = require( 'utils' ),
         xpath = require( 'casper' ).selectXPath,
@@ -24,13 +23,6 @@ try {
     phantom.exit( 1 );
 }
 
-// ===================================================================
-/* TODO:
-    move selectors and assertText strings into global object for easier editing
-    pass email, etc. for first (successful) registration (for use with other tests)
-
-
-*/
 // =================================================================== globals and helpers
 var email = spaceghost.user.getRandomEmail(),
     password = '123456',
@@ -38,48 +30,27 @@ var email = spaceghost.user.getRandomEmail(),
     username = 'test' + Date.now();
 
 // =================================================================== TESTS
-spaceghost.thenOpen( spaceghost.baseUrl, function(){
+spaceghost.openHomePage( function(){
     this.test.comment( 'loading galaxy homepage' );
-    // can we load galaxy?
     this.test.assertTitle( 'Galaxy' );
-    // xpath selector use:
     this.test.assertExists( xpath( "//div[@id='masthead']" ), 'found masthead' );
 });
 
-// failing tests for...testing...the tests
-//spaceghost.thenOpen( spaceghost.baseUrl, function(){
-//    this.test.comment( 'loading galaxy homepage' );
-//    // can we load galaxy?
-//    this.test.assertTitle( 'Blorgo' );
-//    // xpath selector use:
-//    this.test.assertExists( xpath( "//div[@id='facebook']" ), 'found facebook' );
-//});
-
-
 // ------------------------------------------------------------------- register a new user
 spaceghost.then( function(){
     this.test.comment( 'registering user: ' + email );
     this.user._submitRegistration( email, password, username, confirm );
 });
-spaceghost.thenOpen( spaceghost.baseUrl, function(){
-    this.waitForMasthead( function() {
-        this.clickLabel( 'User' );
-        this.test.assertSelectorHasText( 'a #user-email', email, '#user-email === ' + email );
-    });
+spaceghost.openHomePage( function(){
+    this.clickLabel( 'User' );
+    var loggedInAs = this.fetchText( xpath( spaceghost.data.selectors.masthead.userMenu.userEmail_xpath ) );
+    this.test.assert( loggedInAs.indexOf( email ) !== -1, 'found proper email in user menu: ' + loggedInAs );
 });
 
 // ------------------------------------------------------------------- log out that user
-spaceghost.then( function(){
-    this.waitForMasthead( function() {
-        this.test.comment( 'logging out user: ' + email );
-        this.user.logout();
-    });
-});
-spaceghost.then( function(){
-    this.waitForMasthead( function() {
-        this.debug( 'email:' + this.getElementInfo( 'a #user-email' ).html );
-        this.test.assert( !this.getElementInfo( 'a #user-email' ).html, '#user-email is empty' );
-    });
+spaceghost.user.logout().openHomePage( function(){
+    var emailSelector = xpath( this.data.selectors.masthead.userMenu.userEmail_xpath );
+    this.test.assert( !this.elementInfoOrNull( emailSelector ), 'user email not found' );
 });
 
 // ------------------------------------------------------------------- bad user registrations
@@ -99,7 +70,7 @@ spaceghost.each( badEmails, function( self, badEmail ){
         this.user._submitRegistration( badEmail, password, username, confirm );
     });
     self.then(function(){
-        this.assertErrorMessage( 'Enter a real email address' );
+        this.assertErrorMessage( 'Please enter your valid email address' );
     });
 });
 
@@ -108,10 +79,10 @@ var badPasswords = [ '1234' ];
 spaceghost.each( badPasswords, function( self, badPassword ){
     self.then( function(){
         this.test.comment( 'attempting bad password: ' + badPassword );
-        this.user._submitRegistration( spaceghost.user.getRandomEmail(), badPassword, username, confirm );
+        this.user._submitRegistration( spaceghost.user.getRandomEmail(), badPassword, username, badPassword );
     });
     self.then(function(){
-        this.assertErrorMessage( 'Use a password of at least 6 characters' );
+        this.assertErrorMessage( 'Please use a password of at least 6 characters' );
     });
 });
 
@@ -123,7 +94,7 @@ spaceghost.each( badConfirms, function( self, badConfirm ){
         this.user._submitRegistration( spaceghost.user.getRandomEmail(), password, username, badConfirm );
     });
     self.then(function(){
-        this.assertErrorMessage( 'Passwords do not match' );
+        this.assertErrorMessage( 'Passwords don\'t match' );
     });
 });
 
@@ -149,7 +120,7 @@ spaceghost.each( badUsernames, function( self, badUsername ){
         this.user._submitRegistration( newEmail, password, badUsername, confirm );
     });
     self.then(function(){
-        this.assertErrorMessage( "Public name must contain only lower-case letters, numbers and '-'" );
+        this.assertErrorMessage( 'Public name must contain only lowercase letters, numbers and "-"' );
     });
 });
 
@@ -163,7 +134,6 @@ spaceghost.then(function(){
     this.assertErrorMessage( 'Public name is taken; please choose another' );
 });
 
-
 // ------------------------------------------------------------------- test the convenience fns
 // these versions are for conv. use in other tests, they should throw errors if used improperly
 spaceghost.then( function(){
@@ -175,11 +145,6 @@ spaceghost.then( function(){
     });
 });
 
-spaceghost.then( function(){
-    //??: necessary?
-    this.user.logout();
-});
-
 // ===================================================================
 spaceghost.run( function(){
     this.test.done();

test/casperjs/spaceghost.js

@@ -507,6 +507,38 @@ SpaceGhost.prototype.run = function run( onComplete, time ){
     Casper.prototype.run.call( this, new_onComplete, time );
 };
 
+// ------------------------------------------------------------------- home page
+/** Wait for the homepage/index/Analyze Data to load fully.
+ */
+SpaceGhost.prototype.openHomePage = function openHomePage( then, delay ){
+//TODO: delay doesn't seem to work
+    this.thenOpen( this.baseUrl, function _openHomePage(){
+        this.waitFor(
+            function waitForCheck(){
+                return this.homePageIsLoaded();
+            },
+            then,
+            function openHomePageTimeout(){
+                throw new GalaxyError( 'Homepage timed out' );
+            },
+            delay
+        );
+    });
+    return this;
+};
+
+/** Check for visibility of main home page elements: masthead, tool menu, history panel.
+ */
+SpaceGhost.prototype.homePageIsLoaded = function homePageIsLoaded(){
+    //this.debug( 'homePageIsLoaded: ' + [
+    //    this.visible( '#masthead' ),
+    //    this.visible( this.data.selectors.toolMenu.container ),
+    //    this.visible( '#current-history-panel' )].join( ', ' ) );
+    return ( this.visible( '#masthead' )
+          && this.visible( this.data.selectors.toolMenu.container )
+          && this.visible( '#current-history-panel' ) );
+};
+
 // ------------------------------------------------------------------- try step
 /** Install a function as an error handler temporarily, run a function with steps, then remove the handler.
  *      A rough stand-in for try catch with steps.
@@ -549,6 +581,7 @@ SpaceGhost.prototype.tryStepsCatch = function tryStepsCatch( stepsFn, catchFn ){
     });
 };
 
+
 // ------------------------------------------------------------------- misc
 /** Hover over an element.
  *      NOTE: not for use with iframes (main, tool, history) - they need to re-calc
@@ -559,7 +592,7 @@ SpaceGhost.prototype.tryStepsCatch = function tryStepsCatch( stepsFn, catchFn ){
 SpaceGhost.prototype.hoverOver = function hoverOver( selector, whenHovering ){
     var elementInfo = this.getElementInfo( selector );
     this.page.sendEvent( 'mousemove', elementInfo.x + 1, elementInfo.y + 1 );
-    whenHovering.call( this );
+    if( whenHovering ){ whenHovering.call( this ); }
     return this;
 };
 
@@ -626,15 +659,6 @@ SpaceGhost.prototype.waitForMultipleNavigation = function waitForMultipleNavigat
 
 
 // ------------------------------------------------------------------- iframes, damnable iframes
-/** Version of Casper#withFrame for the history iframe.
- *      Hopefully will allow easier test transition if/when frames are removed
- *      (i.e. -> just call the function).
- *  @param {Function} then  function called when in the history frame
- */
-SpaceGhost.prototype.withHistoryPanel = function withHistoryPanel( then ){
-    return this.withFrame( this.data.selectors.frames.history, then );
-};
-
 /** Version of Casper#withFrame for the main iframe.
  *  @param {Function} then  function called when in the frame
  */
@@ -671,15 +695,6 @@ SpaceGhost.prototype.jumpToFrame = function jumpToFrame( frame, fn ){
     return returned;
 };
 
-/** Jumps into history frame, exectutes fn, and jumps back to original frame.
- *  @param {Selector} frame the selector for the frame to jump to
- *  @param {Function} fn    function called when in the frame
- *  @returns {Any} the return value of fn
- */
-SpaceGhost.prototype.jumpToHistory = function jumpToHistory( fn ){
-    return this.jumpToFrame( this.data.selectors.frames.history, fn );
-};
-
 /** Jumps into main frame, exectutes fn, and jumps back to original frame.
  *  @param {Selector} frame the selector for the frame to jump to
  *  @param {Function} fn    function called when in the frame
@@ -793,7 +808,7 @@ SpaceGhost.prototype.assertVisibleWithText = function assertVisibleWithText( sel
  *  @param {CasperJS selector} messageSelector what element in which to search for the text
  *      (defaults to '.errormessage')
  */
-SpaceGhost.prototype.assertErrorMessage = function assertSelectorAndTextInFrame( message, frame, messageSelector ){
+SpaceGhost.prototype.assertErrorMessage = function assertErrorMessage( message, frame, messageSelector ){
     messageSelector = messageSelector || this.data.selectors.messages.error;
     frame = frame || this.data.selectors.frames.main;
     this.assertSelectorAndTextInFrame( messageSelector, message, frame );
@@ -871,6 +886,7 @@ SpaceGhost.prototype.assertDoesntHaveClass = function assertDoesntHaveClass( sel
     this.test.assert( classes.indexOf( className ) === -1, msg );
 };
 
+
 // =================================================================== CONVENIENCE
 /** Wraps casper.getElementInfo in try, returning null if element not found instead of erroring.
  *  @param {String} selector    css or xpath selector for the element to find
@@ -1105,7 +1121,7 @@ SpaceGhost.prototype.getUniverseSetting = function getUniverseSetting( iniKey ){
 
 SpaceGhost.prototype.waitForMasthead = function wait( then ) {
     return this.waitForText( this.data.labels.masthead.menus.user, then );
-}
+};
 
 
 // =================================================================== TEST DATA
@@ -1116,7 +1132,6 @@ SpaceGhost.prototype.data = {
         tooltipBalloon          : '.tooltip',
 
         editableText            : '.editable-text',
-        editableTextInput       : 'input#renaming-active',
 
         messages : {
             all         : '[class*="message"]',
@@ -1128,17 +1143,23 @@ SpaceGhost.prototype.data = {
         },
 
         frames : {
-            main    : 'galaxy_main',
-            history : 'galaxy_history'
+            main    : 'galaxy_main'
         },
 
         masthead : {
-            adminLink : '#masthead a[href="/admin/index"]',
-            userMenu : {
-                userEmail       : 'a #user-email',
-                userEmail_xpath : '//a[contains(text(),"Logged in as")]/span["id=#user-email"]'
+            id          : '#masthead',
+            adminLink   : '#masthead a[href="/admin/index"]',
+            userMenu    : {
+                userEmail_xpath : '//a[contains(text(),"Logged in as")]'
             }
         },
+        toolMenu : {
+            container   : '.toolMenuContainer'
+        },
+        historyPanel : {
+            current     : '#current-history-panel'
+        },
+
         loginPage : {
             form            : 'form#login',
             submit_xpath    : "//input[@value='Login']",
@@ -1146,7 +1167,8 @@ SpaceGhost.prototype.data = {
         },
         registrationPage : {
             form            : 'form#registration',
-            submit_xpath    : "//input[@value='Submit']"
+            submit_xpath    : "//input[@value='Submit']",
+            returnLink      : '//a[contains(text(),"Return to the home page")]'
         },
         tools : {
             general : {

test/casperjs/upload-tests.js

@@ -43,23 +43,18 @@ var testUploadInfo = {};
 
 // =================================================================== TESTS
 // ------------------------------------------------------------------- start a new user
-spaceghost.user.loginOrRegisterUser( email, password );
-//??: why is a reload needed here? If we don't, loggedInAs === '' ...
-spaceghost.thenOpen( spaceghost.baseUrl, function(){
+spaceghost.user.loginOrRegisterUser( email, password ).openHomePage( function(){
     var loggedInAs = spaceghost.user.loggedInAs();
     this.test.assert( loggedInAs === email, 'loggedInAs() matches email: "' + loggedInAs + '"' );
 });
 
-
 // ------------------------------------------------------------------- long form
-
 // upload a file...
 spaceghost.then( function(){
     this.test.comment( 'Test uploading a file' );
 
     var filename = '1.txt',
         filepath = this.options.scriptDir + '/../../test-data/' + filename;
-
     this.tools._uploadFile( filepath );
 
     // when an upload begins successfully main should reload with a infomessagelarge
@@ -83,6 +78,7 @@ spaceghost.historypanel.waitForHdas( function(){
         this.test.fail( 'Could not locate new hda: ' + testUploadInfo.name );
 
     } else {
+        spaceghost.debugElement( spaceghost.jsonStr( hdaInfo ) );
         this.historypanel.waitForHdaState( '#' + hdaInfo.attributes.id, 'ok',
             function whenInStateFn( newHdaInfo ){
                 //this.debug( 'newHdaInfo:\n' + this.jsonStr( newHdaInfo ) );

test/functional/test_history_functions.py

@@ -82,6 +82,7 @@ class TestHistory( TwillTestCase ):
                              .first()
         assert historyB is not None, "Problem retrieving historyB from database"
         assert historyA.id == historyB.id, "After the same user logged out and back in, their last used history was not associated with their new session"
+
     def test_005_deleting_histories( self ):
         """Testing deleting histories"""
         # Logged in as admin_user
@@ -144,6 +145,7 @@ class TestHistory( TwillTestCase ):
             raise AssertionError, "Default permissions were incorrectly deleted from the db for history id %d when it was deleted" % history2.id
         # Current history is empty
         self.history_options( user=True )
+
     def test_010_history_rename( self ):
         """Testing renaming a history"""
         # Logged in as admin_user
@@ -157,10 +159,12 @@ class TestHistory( TwillTestCase ):
             raise AssertionError, "History id %d deleted when it should not be" % latest_history.id
         self.rename_history( self.security.encode_id( history3.id ), history3.name, new_name=urllib.quote( 'history 3' ) )
         sa_session.refresh( history3 )
+
     def test_015_history_list( self ):
         """Testing viewing previously stored active histories"""
         # Logged in as admin_user
         self.view_stored_active_histories()
+
     def test_020_share_current_history( self ):
         """Testing sharing the current history which contains only public datasets"""
         # Logged in as admin_user
@@ -210,10 +214,13 @@ class TestHistory( TwillTestCase ):
         # Test sharing history3 with an invalid user
         self.share_current_history( 'jack@jill.com',
                                     strings_displayed_after_submit=[ 'jack@jill.com is not a valid Galaxy user.' ] )
+
     def test_025_delete_shared_current_history( self ):
         """Testing deleting the current history after it was shared"""
         # Logged in as admin_user
-        self.delete_current_history( strings_displayed=[ "History (%s) has been shared with others, unshare it before deleting it." % history3.name ] )
+        self.delete_current_history(
+            strings_displayed=[ "History (%s) has been shared with others, unshare it before deleting it." % history3.name ] )
+
     def test_030_clone_shared_history( self ):
         """Testing copying a shared history"""
         # logged in as admin user
@@ -656,7 +663,7 @@ class TestHistory( TwillTestCase ):
                 if hda[ 'id' ] == self.security.encode_id( hda_2_bed.id ):
                     return ( not hda[ 'accessible' ] )
             return False
-        self.check_history_json( r'\bhdaJson\s*=\s*(.*);', hda_2_bed_is_inaccessible )
+        self.check_history_json( r'\bhdaJSON\s*=\s*(.*);', hda_2_bed_is_inaccessible )
 
         # Admin users can view all datasets ( using the history/view feature ), so make sure 2.bed is accessible to the admin
         self.logout()
@@ -734,7 +741,7 @@ class TestHistory( TwillTestCase ):
         log.info( 'deleting last hda' )
         self.delete_history_item( str( latest_hda.id ) )
         # check the historyPanel settings.show_deleted for a null json value (no show_deleted in query string)
-        self.check_history_json( r'\bpage_show_deleted\s*=\s*(.*),', lambda x: x == None )
+        self.check_history_json( r'\bshow_deleted\s*:\s*(.*),', lambda x: x == None )
 
         # reload this history with the show_deleted flag set in the query string
         #   the deleted dataset should be there with the proper 'deleted' text
@@ -742,7 +749,7 @@ class TestHistory( TwillTestCase ):
         log.info( 'turning show_deleted on' )
         #self.visit_url( "%s/history/?show_deleted=True" % self.url )
         # check the historyPanel settings.show_deleted for a true json value
-        self.check_history_json( r'\bpage_show_deleted\s*=\s*(.*),', lambda x: x == True, show_deleted=True )
+        self.check_history_json( r'\bshow_deleted\s*:\s*(.*),', lambda x: x == True, show_deleted=True )
 
         # reload this history again with the show_deleted flag set TO FALSE in the query string
         #   make sure the 'history empty' message shows
@@ -750,7 +757,7 @@ class TestHistory( TwillTestCase ):
         log.info( 'turning show_deleted off' )
         #self.visit_url( "%s/history/?show_deleted=False" % self.url )
         # check the historyPanel settings.show_deleted for a false json value
-        self.check_history_json( r'\bpage_show_deleted\s*=\s*(.*),', lambda x: x == False, show_deleted=False )
+        self.check_history_json( r'\bshow_deleted\s*:\s*(.*),', lambda x: x == False, show_deleted=False )
 
         # delete this history
         self.delete_history( self.security.encode_id( latest_history.id ) )

test/install_and_test_tool_shed_repositories/base/util.py

@@ -3,10 +3,9 @@ import sys
 
 cwd = os.getcwd()
 sys.path.append( cwd )
-new_path = [ os.path.join( cwd, "scripts" ),
-             os.path.join( cwd, "lib" ),
+new_path = [ os.path.join( cwd, "lib" ),
              os.path.join( cwd, 'test' ),
-             os.path.join( cwd, 'scripts', 'api' ) ]
+             os.path.join( cwd, 'lib', 'tool_shed', 'scripts', 'api' ) ]
 new_path.extend( sys.path )
 sys.path = new_path
 
@@ -27,6 +26,10 @@ import urllib
 from datetime import datetime
 from datetime import timedelta
 
+from common import get_api_url
+from common import get_latest_downloadable_changeset_revision_via_api
+from common import get_repository_dict
+from common import json_from_url
 from common import update
 
 from galaxy.util import asbool
@@ -267,18 +270,6 @@ def display_tool_dependencies_by_name( tool_dependency_dicts ):
             version = str( tool_dependency_dict[ 'version' ] )
             print "# %s %s version %s" % ( type, name, version )
 
-def get_api_url( base, parts=[], params=None ):
-    if 'api' in parts and parts.index( 'api' ) != 0:
-        parts.pop( parts.index( 'api' ) )
-        parts.insert( 0, 'api' )
-    elif 'api' not in parts:
-        parts.insert( 0, 'api' )
-    url = suc.url_join( base, *parts )
-    if params is not None:
-        query_string = urllib.urlencode( params )
-        url += '?%s' % query_string
-    return url
-
 def get_database_version( app ):
     '''
     This method returns the value of the version column from the migrate_version table, using the provided app's SQLAlchemy session to determine
@@ -296,19 +287,6 @@ def get_database_version( app ):
         break
     return version
 
-def get_latest_downloadable_changeset_revision( url, name, owner ):
-    error_message = ''
-    parts = [ 'api', 'repositories', 'get_ordered_installable_revisions' ]
-    params = dict( name=name, owner=owner )
-    api_url = get_api_url( base=url, parts=parts, params=params )
-    changeset_revisions, error_message = json_from_url( api_url )
-    if error_message:
-        return None, error_message
-    if changeset_revisions:
-        return changeset_revisions[ -1 ], error_message
-    else:
-        return suc.INITIAL_CHANGELOG_HASH, error_message
-
 def get_missing_repository_dependencies( repository ):
     """
     Return the entire list of missing repository dependencies for the received repository.  The entire
@@ -467,21 +445,6 @@ def get_repository_dependencies_for_changeset_revision( tool_shed_url, encoded_r
         return None, error_message
     return repository_dependency_dicts, error_message
 
-def get_repository_dict( url, repository_dict ):
-    error_message = ''
-    parts = [ 'api', 'repositories', repository_dict[ 'repository_id' ] ]
-    api_url = get_api_url( base=url, parts=parts )
-    extended_dict, error_message = json_from_url( api_url )
-    if error_message:
-        return None, error_message
-    name = str( extended_dict[ 'name' ] )
-    owner = str( extended_dict[ 'owner' ] )
-    latest_changeset_revision, error_message = get_latest_downloadable_changeset_revision( url, name, owner )
-    if error_message:
-        return None, error_message
-    extended_dict[ 'latest_revision' ] = str( latest_changeset_revision )
-    return extended_dict, error_message
-
 def get_repository_dependencies_dicts( url, encoded_repository_metadata_id ):
     """
     Return a list if dictionaries that define the repository dependencies of the repository defined by the
@@ -702,23 +665,23 @@ def is_excluded( exclude_list_dicts, name, owner, changeset_revision, encoded_re
     return False, None
 
 def is_latest_downloadable_revision( url, repository_dict ):
-    name = str( repository_dict[ 'name' ] )
-    owner = str( repository_dict[ 'owner' ] )
-    changeset_revision = str( repository_dict[ 'changeset_revision' ] )
-    latest_revision = get_latest_downloadable_changeset_revision( url, name=name, owner=owner )
-    return changeset_revision == str( latest_revision )
-
-def json_from_url( url ):
+    """
+    Return True if the changeset_revision defined in the received repository_dict is the latest
+    installable revision for the repository.
+    """
     error_message = ''
-    url_handle = urllib.urlopen( url )
-    url_contents = url_handle.read()
-    try:
-        parsed_json = from_json_string( url_contents )
-    except Exception, e:
-        error_message = str( url_contents )
-        log.exception( 'Error parsing JSON data in json_from_url(): %s.' % str( e ) )
-        return None, error_message
-    return parsed_json, error_message
+    name = repository_dict.get( 'name', None )
+    owner = repository_dict.get( 'owner', None )
+    changeset_revision = repository_dict.get( 'changeset_revision', None )
+    if name is not None and owner is not None and changeset_revision is not None:
+        name = str( name )
+        owner = str( owner )
+        changeset_revision = str( changeset_revision )
+        latest_revision, error_message = get_latest_downloadable_changeset_revision_via_api( url, name=name, owner=owner )
+        if latest_revision is None or error_message:
+            return None, error_message
+    is_latest_downloadable = changeset_revision == str( latest_revision )
+    return is_latest_downloadable, error_message
 
 def parse_exclude_list( xml_filename ):
     """Return a list of repositories to exclude from testing."""
@@ -882,9 +845,19 @@ def populate_install_containers_for_repository_dependencies( app, repository, re
         log.debug( 'due to the following error getting repository_dependencies_dicts:\n%s' % str( error_message ) )
     else:
         for repository_dependencies_dict in repository_dependencies_dicts:
-            name = str( repository_dependencies_dict[ 'name' ] )
-            owner = str( repository_dependencies_dict[ 'owner' ] )
-            changeset_revision = str( repository_dependencies_dict[ 'changeset_revision' ] )
+            if not isinstance( repository_dependencies_dict, dict ):
+                log.debug( 'Skipping invalid repository_dependencies_dict: %s' % str( repository_dependencies_dict ) )
+                continue
+            name = repository_dependencies_dict.get( 'name', None )
+            owner = repository_dependencies_dict.get( 'owner', None )
+            changeset_revision = repository_dependencies_dict.get( 'changeset_revision', None )
+            if name is None or owner is None or changeset_revision is None:
+                log.debug( 'Skipping invalid repository_dependencies_dict due to missing name,owner or changeset_revision: %s' % \
+                    str( repository_dependencies_dict ) )
+                continue
+            name = str( name )
+            owner = str( owner )
+            changeset_revision = str( changeset_revision )
             log.debug( 'Checking installation containers for revision %s of repository dependency %s owned by %s' % \
                 ( changeset_revision, name, owner ) )
             required_repository_metadata_id = repository_dependencies_dict[ 'id' ]
@@ -898,22 +871,23 @@ def populate_install_containers_for_repository_dependencies( app, repository, re
                 log.debug( 'due to the following error getting tool_test_results:\n%s' % str( error_message ) )
             else:
                 # Check the required repository's time_last_tested value to see if its tool_test_results column
-                # has been updated within the past 12 hours.
-                twelve_hours_ago = datetime.utcnow() - timedelta( hours=12 )
+                # has been updated within the past 12 hours.  The RepositoryMetadata class's to_dict() method
+                # returns the value of time_last_tested in datetime.isoformat().
+                twenty_hours_ago = ( datetime.utcnow() - timedelta( hours=20 ) ).isoformat()
                 time_last_tested, error_message = get_time_last_tested( galaxy_tool_shed_url, repository_metadata_id )
-                if time_last_tested is not None and time_last_tested < twelve_hours_ago:
+                if time_last_tested is not None and time_last_tested < twenty_hours_ago:
                     log.debug( 'The install containers for version %s of repository dependency %s owned by %s have been ' % \
                         ( changeset_revision, name, owner ) )
-                    log.debug( 'populated within the past 12 hours (likely in this test run), so skipping this check.' )
+                    log.debug( 'populated within the past 20 hours (likely in this test run), so skipping this check.' )
                     continue
-                if time_last_tested is None:
+                elif time_last_tested is None:
                     log.debug( 'The time_last_tested column value is None for version %s of repository dependency %s owned by %s.' % \
                         ( changeset_revision, name, owner ) )
-                elif time_last_tested < twelve_hours_ago:
-                    log.debug( 'Version %s of repository dependency %s owned by %s was last tested less than 12 hours ago.' % \
+                elif time_last_tested < twenty_hours_ago:
+                    log.debug( 'Version %s of repository dependency %s owned by %s was last tested less than 20 hours ago.' % \
                         ( changeset_revision, name, owner ) )
                 else:
-                    log.debug( 'Version %s of repository dependency %s owned by %s was last tested more than 12 hours ago.' % \
+                    log.debug( 'Version %s of repository dependency %s owned by %s was last tested more than 20 hours ago.' % \
                         ( changeset_revision, name, owner ) )
                 # Inspect the tool_test_results_dict for the last test run to see if it has not yet been populated.
                 if len( tool_test_results_dicts ) == 0:
@@ -975,8 +949,8 @@ def populate_install_containers_for_repository_dependencies( app, repository, re
                 else:
                     log.debug( 'Cannot retrieve revision %s of required repository %s owned by %s from the database ' % \
                         ( changeset_revision, name, owner ) )
-                    log.debug( 'so tool_test_results cannot be saved.' )
-                    log.debug( 'The attributes used to retrieve the record are:\n' )
+                    log.debug( 'so tool_test_results cannot be saved at this time.' )
+                    log.debug( 'The attributes used to retrieve the record are:' )
                     log.debug( 'tool_shed: %s name: %s owner: %s changeset_revision: %s' % \
                         ( cleaned_tool_shed_url, name, owner, changeset_revision ) )
 
@@ -1006,9 +980,15 @@ def save_test_results_for_changeset_revision( url, tool_test_results_dicts, tool
     if can_update_tool_shed:
         metadata_revision_id = repository_dict.get( 'id', None )
         if metadata_revision_id is not None:
-            name = str( repository_dict[ 'name' ] )
-            owner = str( repository_dict[ 'owner' ] )
-            changeset_revision = str( repository_dict[ 'changeset_revision' ] )
+            name = repository_dict.get( 'name', None )
+            owner = repository_dict.get( 'owner', None )
+            changeset_revision = repository_dict.get( 'changeset_revision', None )
+            if name is None or owner is None or changeset_revision is None:
+                log.debug( 'Entries for name, owner or changeset_revision missing from repository_dict %s' % str( repository_dict ) )
+                return {}
+            name = str( name )
+            owner = str( owner )
+            changeset_revision = str( changeset_revision )
             log.debug('\n=============================================================\n' )
             log.debug( 'Inserting the following into tool_test_results for revision %s of repository %s owned by %s:\n%s' % \
                 ( changeset_revision, name, owner, str( tool_test_results_dict ) ) )

test/install_and_test_tool_shed_repositories/repositories_with_tools/functional_tests.py

@@ -685,9 +685,16 @@ def test_repository_tools( app, repository, repository_dict, tool_test_results_d
         tool_test_results_dict[ 'failed_tests' ] = failed_test_dicts
         failed_repository_dict = repository_identifier_dict
         install_and_test_statistics_dict[ 'at_least_one_test_failed' ].append( failed_repository_dict )
-        set_do_not_test = \
-            not install_and_test_base_util.is_latest_downloadable_revision( install_and_test_base_util.galaxy_tool_shed_url,
-                                                                            repository_dict )
+        is_latest_downloadable_revision, error_message = \
+            install_and_test_base_util.is_latest_downloadable_revision( install_and_test_base_util.galaxy_tool_shed_url,
+                                                                        repository_dict )
+        if is_latest_downloadable_revision is None or error_message:
+            log.debug( 'Error attempting to determine if revision %s of repository %s owned by %s ' % \
+                ( changeset_revision, name, owner ) )
+            log.debug( 'is the latest downloadable revision: %s' % str( error_message ) )
+            set_do_not_test = False
+        else:
+            set_do_not_test = not is_latest_downloadable_revision
         params = dict( tools_functionally_correct=False,
                        test_install_error=False,
                        do_not_test=str( set_do_not_test ) )

test/tool_shed/base/twilltestcase.py

@@ -382,7 +382,17 @@ class ShedTwillTestCase( TwillTestCase ):
                     except:
                         pass
         return previously_created, username_taken, invalid_username
-        
+    
+    def deactivate_repository( self, installed_repository, strings_displayed=[], strings_not_displayed=[] ):
+        url = '/admin_toolshed/deactivate_or_uninstall_repository?id=%s' % self.security.encode_id( installed_repository.id )
+        self.visit_galaxy_url( url )
+        self.check_for_strings( strings_displayed, strings_not_displayed )
+        form = tc.browser.get_form( 'deactivate_or_uninstall_repository' )
+        kwd = self.set_form_value( form, {}, 'remove_from_disk', False )
+        tc.submit( 'deactivate_or_uninstall_repository_button' )
+        strings_displayed = [ 'The repository named', 'has been deactivated' ]
+        self.check_for_strings( strings_displayed, strings_not_displayed=[] )
+
     def delete_files_from_repository( self, repository, filenames=[], strings_displayed=[ 'were deleted from the repository' ], strings_not_displayed=[] ):
         files_to_delete = []
         basepath = self.get_repo_path( repository )
@@ -1203,18 +1213,14 @@ class ShedTwillTestCase( TwillTestCase ):
         self.visit_url( url )
         self.check_for_strings( strings_displayed, strings_not_displayed )
         
-    def uninstall_repository( self, installed_repository, remove_from_disk=True, is_required=False, strings_displayed=[], strings_not_displayed=[] ):
+    def uninstall_repository( self, installed_repository, strings_displayed=[], strings_not_displayed=[] ):
         url = '/admin_toolshed/deactivate_or_uninstall_repository?id=%s' % self.security.encode_id( installed_repository.id )
         self.visit_galaxy_url( url )
         self.check_for_strings( strings_displayed, strings_not_displayed )
         form = tc.browser.get_form( 'deactivate_or_uninstall_repository' )
-        kwd = self.set_form_value( form, {}, 'remove_from_disk', remove_from_disk )
+        kwd = self.set_form_value( form, {}, 'remove_from_disk', True )
         tc.submit( 'deactivate_or_uninstall_repository_button' )
-        strings_displayed = [ 'The repository named' ]
-        if remove_from_disk and not is_required:
-            strings_displayed.append( 'has been uninstalled' )
-        else:
-            strings_displayed.append( 'has been deactivated' )
+        strings_displayed = [ 'The repository named', 'has been uninstalled' ]
         self.check_for_strings( strings_displayed, strings_not_displayed=[] )
         
     def update_installed_repository( self, installed_repository, strings_displayed=[], strings_not_displayed=[] ):

test/tool_shed/functional/test_0490_export_import_repositories.py

@@ -65,7 +65,7 @@ class TestExportImportRepository( ShedTwillTestCase ):
         global capsule_filepath
         repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name )
         capsule_filepath = self.export_capsule( repository )
-        log.debug( os.path.exists( capsule_filepath ) )
+        assert os.path.exists( capsule_filepath ), 'Exported repository capsule file path %s not found.' % capsule_filepath
         
     def test_0015_verify_exported_capsule( self ):
         '''Verify the exported capsule contents.'''

test/tool_shed/functional/test_1020_install_repository_with_repository_dependencies.py

@@ -121,7 +121,7 @@ class ToolWithRepositoryDependencies( ShedTwillTestCase ):
         global base_datatypes_count
         installed_repository = self.test_db_util.get_installed_repository_by_name_owner( datatypes_repository_name, common.test_user_1_name )
         old_datatypes_count = int( self.get_datatypes_count() )
-        self.uninstall_repository( installed_repository, remove_from_disk=False )
+        self.deactivate_repository( installed_repository )
         current_datatypes = int( self.get_datatypes_count() )
         assert current_datatypes < old_datatypes_count, 'Uninstalling emboss did not remove datatypes.'
 

test/tool_shed/functional/test_1040_install_repository_basic_circular_dependencies.py

@@ -109,7 +109,7 @@ class TestInstallingCircularDependencies( ShedTwillTestCase ):
         installed_filtering_repository = self.test_db_util.get_installed_repository_by_name_owner( filtering_repository_name, common.test_user_1_name )
         self.display_installed_repository_manage_page( installed_freebayes_repository )
         self.display_installed_repository_manage_page( installed_filtering_repository )
-        self.uninstall_repository( installed_filtering_repository, remove_from_disk=False )
+        self.deactivate_repository( installed_filtering_repository )
         self.test_db_util.ga_refresh( installed_filtering_repository )
         self.check_galaxy_repository_tool_panel_section( installed_filtering_repository, '' )
         strings_displayed = [ 'Missing repository', 'filtering', 'freebayes_0040', 'user1', "Galaxy's freebayes tool for test 0040" ]
@@ -134,7 +134,7 @@ class TestInstallingCircularDependencies( ShedTwillTestCase ):
         installed_filtering_repository = self.test_db_util.get_installed_repository_by_name_owner( filtering_repository_name, common.test_user_1_name )
         self.display_installed_repository_manage_page( installed_freebayes_repository )
         self.display_installed_repository_manage_page( installed_filtering_repository )
-        self.uninstall_repository( installed_freebayes_repository, remove_from_disk=False )
+        self.deactivate_repository( installed_freebayes_repository )
         self.test_db_util.ga_refresh( installed_freebayes_repository )
         self.check_galaxy_repository_tool_panel_section( installed_freebayes_repository, '' )
         strings_displayed = [ 'Missing repository', 'freebayes', 'filtering_0040', 'user1', "Galaxy's filtering tool for test 0040" ]
@@ -149,7 +149,7 @@ class TestInstallingCircularDependencies( ShedTwillTestCase ):
         installed_freebayes_repository = self.test_db_util.get_installed_repository_by_name_owner( freebayes_repository_name, common.test_user_1_name )
         self.display_installed_repository_manage_page( installed_freebayes_repository )
         self.display_installed_repository_manage_page( installed_filtering_repository )
-        self.uninstall_repository( installed_filtering_repository, remove_from_disk=False )
+        self.deactivate_repository( installed_filtering_repository )
         self.test_db_util.ga_refresh( installed_filtering_repository )
         self.check_galaxy_repository_tool_panel_section( installed_filtering_repository, '' )
         strings_displayed = [ 'Missing repository', 'filtering', 'freebayes_0040', 'user1', "Galaxy's freebayes tool for test 0040" ]

test/tool_shed/functional/test_1050_circular_dependencies_4_levels.py

@@ -315,7 +315,7 @@ class TestInstallRepositoryCircularDependencies( ShedTwillTestCase ):
         '''Deactivate emboss_datatypes and verify that the datatypes count is reduced.'''
         original_datatypes = self.get_datatypes_count()
         repository = self.test_db_util.get_installed_repository_by_name_owner( emboss_datatypes_repository_name, common.test_user_1_name )
-        self.uninstall_repository( repository, remove_from_disk=False )
+        self.deactivate_repository( repository )
         assert original_datatypes > self.get_datatypes_count(), 'Deactivating emboss_datatypes did not remove datatypes.'
         # Now we have emboss, bismark, column_maker, and convert_chars installed, filtering and freebayes never installed, and emboss_datatypes deactivated.
         installed_repositories = [ ( column_repository_name, common.test_user_1_name ), 
@@ -330,7 +330,7 @@ class TestInstallRepositoryCircularDependencies( ShedTwillTestCase ):
     def test_0070_uninstall_emboss_repository( self ):
         '''Uninstall the emboss_5 repository.'''
         repository = self.test_db_util.get_installed_repository_by_name_owner( emboss_repository_name, common.test_user_1_name )
-        self.uninstall_repository( repository, remove_from_disk=True )
+        self.uninstall_repository( repository )
         strings_not_displayed = [ repository.name, repository.installed_changeset_revision ]
         self.display_galaxy_browse_repositories_page( strings_not_displayed=strings_not_displayed )
         self.test_db_util.ga_refresh( repository )

test/tool_shed/functional/test_1080_advanced_circular_dependency_installation.py

@@ -162,7 +162,7 @@ class TestRepositoryDependencies( ShedTwillTestCase ):
                                                                                             common.test_user_1_name )
         installed_column_repository = self.test_db_util.get_installed_repository_by_name_owner( column_repository_name, 
                                                                                             common.test_user_1_name )
-        self.uninstall_repository( installed_convert_repository, remove_from_disk=False )
+        self.deactivate_repository( installed_convert_repository )
         strings_displayed = [ 'column_maker_0080',
                               'Add column',
                               self.url.replace( 'http://', '' ), 
@@ -199,7 +199,7 @@ class TestRepositoryDependencies( ShedTwillTestCase ):
                                                                                             common.test_user_1_name )
         installed_column_repository = self.test_db_util.get_installed_repository_by_name_owner( column_repository_name, 
                                                                                             common.test_user_1_name )
-        self.uninstall_repository( installed_column_repository, remove_from_disk=False )
+        self.deactivate_repository( installed_column_repository )
         strings_displayed = [ 'convert_chars_0080',
                               'Convert delimiters',
                               self.url.replace( 'http://', '' ), 
@@ -217,7 +217,7 @@ class TestRepositoryDependencies( ShedTwillTestCase ):
                                                                                             common.test_user_1_name )
         installed_column_repository = self.test_db_util.get_installed_repository_by_name_owner( column_repository_name, 
                                                                                             common.test_user_1_name )
-        self.uninstall_repository( installed_convert_repository, remove_from_disk=False )
+        self.deactivate_repository( installed_convert_repository  )
         strings_not_displayed = [ 'column_maker_0080',
                                   installed_column_repository.installed_changeset_revision,
                                   'convert_chars_0080',
@@ -274,7 +274,7 @@ class TestRepositoryDependencies( ShedTwillTestCase ):
                                                                                             common.test_user_1_name )
         installed_column_repository = self.test_db_util.get_installed_repository_by_name_owner( column_repository_name, 
                                                                                             common.test_user_1_name )
-        self.uninstall_repository( installed_column_repository, remove_from_disk=True )
+        self.uninstall_repository( installed_column_repository )
         strings_displayed = [ 'convert_chars_0080',
                               'Convert delimiters',
                               self.url.replace( 'http://', '' ), 
@@ -320,7 +320,7 @@ class TestRepositoryDependencies( ShedTwillTestCase ):
                                                                                             common.test_user_1_name )
         installed_column_repository = self.test_db_util.get_installed_repository_by_name_owner( column_repository_name, 
                                                                                             common.test_user_1_name )
-        self.uninstall_repository( installed_convert_repository, remove_from_disk=False )
+        self.deactivate_repository( installed_convert_repository )
         strings_displayed = [ 'column_maker_0080',
                               'Add column',
                               self.url.replace( 'http://', '' ), 
@@ -340,7 +340,7 @@ class TestRepositoryDependencies( ShedTwillTestCase ):
                                                                                             common.test_user_1_name )
         installed_column_repository = self.test_db_util.get_installed_repository_by_name_owner( column_repository_name, 
                                                                                             common.test_user_1_name )
-        self.uninstall_repository( installed_column_repository, remove_from_disk=False )
+        self.deactivate_repository( installed_column_repository )
         strings_displayed = [ 'convert_chars_0080',
                               'Convert delimiters',
                               self.url.replace( 'http://', '' ), 
@@ -387,8 +387,8 @@ class TestRepositoryDependencies( ShedTwillTestCase ):
                                                                                             common.test_user_1_name )
         installed_column_repository = self.test_db_util.get_installed_repository_by_name_owner( column_repository_name, 
                                                                                             common.test_user_1_name )
-        self.uninstall_repository( installed_column_repository, remove_from_disk=False )
-        self.uninstall_repository( installed_convert_repository, remove_from_disk=False )
+        self.deactivate_repository( installed_column_repository )
+        self.deactivate_repository( installed_convert_repository )
         self.test_db_util.install_session.refresh( installed_column_repository )
         self.test_db_util.install_session.refresh( installed_convert_repository )
         self.check_galaxy_repository_tool_panel_section( installed_column_repository, '' )

test/tool_shed/functional/test_1085_repository_dependency_handling.py

@@ -124,14 +124,14 @@ class TestRepositoryDependencies( ShedTwillTestCase ):
         '''uninstall column_maker, verify same section'''
         installed_column_repository = self.test_db_util.get_installed_repository_by_name_owner( column_repository_name, 
                                                                                             common.test_user_1_name )
-        self.uninstall_repository( installed_column_repository, remove_from_disk=True )
+        self.uninstall_repository( installed_column_repository )
         self.test_db_util.ga_refresh( installed_column_repository )
         self.check_galaxy_repository_tool_panel_section( installed_column_repository, 'column_maker' )
  
     def test_0030_uninstall_convert_repository( self ):
         installed_convert_repository = self.test_db_util.get_installed_repository_by_name_owner( convert_repository_name, 
                                                                                             common.test_user_1_name )
-        self.uninstall_repository( installed_convert_repository, remove_from_disk=True )
+        self.uninstall_repository( installed_convert_repository )
         self.test_db_util.ga_refresh( installed_convert_repository )
         self.check_galaxy_repository_tool_panel_section( installed_convert_repository, 'column_maker' )
   
@@ -173,8 +173,8 @@ class TestRepositoryDependencies( ShedTwillTestCase ):
                                                                                             common.test_user_1_name )
         installed_column_repository = self.test_db_util.get_installed_repository_by_name_owner( column_repository_name, 
                                                                                             common.test_user_1_name )
-        self.uninstall_repository( installed_convert_repository, remove_from_disk=True )
-        self.uninstall_repository( installed_column_repository, remove_from_disk=True )
+        self.uninstall_repository( installed_convert_repository )
+        self.uninstall_repository( installed_column_repository )
         self.test_db_util.ga_refresh( installed_convert_repository )
         self.test_db_util.ga_refresh( installed_column_repository )
         self.check_galaxy_repository_tool_panel_section( installed_column_repository, 'new_column_maker' )

test/tool_shed/functional/test_1087_install_updated_repository_dependencies.py

@@ -89,7 +89,7 @@ class TestRepositoryDependencies( ShedTwillTestCase ):
                                  new_tool_panel_section_label='column_maker',
                                  strings_not_displayed=[ 'install_repository_dependencies' ] )
         installed_column_repository = self.test_db_util.get_installed_repository_by_name_owner( column_repository_name, common.test_user_1_name )
-        self.uninstall_repository( installed_column_repository, remove_from_disk=True )
+        self.uninstall_repository( installed_column_repository )
  
     def test_0020_upload_dependency_xml( self ):
         '''Upload a repository_dependencies.xml file to column_maker that specifies convert_chars.'''

test/tool_shed/functional/test_1090_install_tool_from_tool_search.py

@@ -251,7 +251,7 @@ class TestToolSearchAndInstall( ShedTwillTestCase ):
         self.display_galaxy_browse_repositories_page( strings_displayed=strings_displayed, strings_not_displayed=strings_not_displayed )
         self.verify_installed_repositories( installed_repositories )
         filtering_repository = self.test_db_util.get_installed_repository_by_name_owner( filtering_repository_name, common.test_user_1_name )
-        self.uninstall_repository( filtering_repository, remove_from_disk=False )
+        self.deactivate_repository( filtering_repository )
         strings_displayed = [ 'freebayes_0090' ]
         strings_not_displayed = [ 'filtering_0090', 'emboss_0090', 'emboss_datatypes_0090', 'bwa_color_0090', 'bwa_base_0090' ]
         self.display_galaxy_browse_repositories_page( strings_displayed=strings_displayed, strings_not_displayed=strings_not_displayed )
@@ -278,7 +278,7 @@ class TestToolSearchAndInstall( ShedTwillTestCase ):
         self.display_galaxy_browse_repositories_page( strings_displayed=strings_displayed, strings_not_displayed=strings_not_displayed )
         self.verify_installed_repositories( installed_repositories )
         datatypes_repository = self.test_db_util.get_installed_repository_by_name_owner( emboss_datatypes_repository_name, common.test_user_1_name )
-        self.uninstall_repository( datatypes_repository, remove_from_disk=True )
+        self.uninstall_repository( datatypes_repository )
         strings_displayed = [ 'freebayes_0090' ]
         strings_not_displayed = [ 'emboss_datatypes_0090', 'filtering_0090', 'emboss_0090', 'bwa_color_0090', 'bwa_base_0090' ]
         self.display_galaxy_browse_repositories_page( strings_displayed=strings_displayed, strings_not_displayed=strings_not_displayed )

test/tool_shed/functional/test_1100_install_repository_with_complex_dependencies.py

@@ -278,7 +278,7 @@ class TestInstallingComplexRepositoryDependencies( ShedTwillTestCase ):
         '''
         base_repository = self.test_db_util.get_installed_repository_by_name_owner( bwa_base_repository_name, common.test_user_1_name )
         tool_repository = self.test_db_util.get_installed_repository_by_name_owner( bwa_package_repository_name, common.test_user_1_name )
-        self.uninstall_repository( tool_repository, remove_from_disk=False )
+        self.deactivate_repository( tool_repository )
         env_sh_path = os.path.join( self.galaxy_tool_dependency_dir, 
                                     'bwa', 
                                     '0.5.9', 

test/tool_shed/functional/test_1130_datatype_converters.py

@@ -87,5 +87,5 @@ class TestDatatypeConverters( ShedTwillTestCase ):
         so we have to uninstall it first, then verify that it was not assigned a tool panel section.
         '''
         repository = self.test_db_util.get_installed_repository_by_name_owner( repository_name, common.test_user_1_name )
-        self.uninstall_repository( repository, remove_from_disk=True )
+        self.uninstall_repository( repository )
         self.verify_installed_repository_no_tool_panel_section( repository )

test/tool_shed/functional/test_1160_circular_prior_installation_required.py

@@ -217,19 +217,19 @@ class TestSimplePriorInstallation( ShedTwillTestCase ):
         assert filter_repository.update_time > convert_repository.update_time, 'Error: convert_chars_0160 shows a later update time than filtering_0160'
         assert filter_repository.update_time > column_repository.update_time, 'Error: column_maker_0160 shows a later update time than filtering_0160'
 
-    def test_0040_uninstall_all_repositories( self ):
+    def test_0040_deactivate_all_repositories( self ):
         '''Uninstall convert_chars_0160, column_maker_0160, and filtering_0160.'''
         filter_repository = self.test_db_util.get_installed_repository_by_name_owner( filter_repository_name, common.test_user_1_name )
         column_repository = self.test_db_util.get_installed_repository_by_name_owner( column_repository_name, common.test_user_1_name )
         convert_repository = self.test_db_util.get_installed_repository_by_name_owner( convert_repository_name, common.test_user_1_name )
-        self.uninstall_repository( filter_repository, remove_from_disk=False )
-        self.uninstall_repository( column_repository, remove_from_disk=False )
-        self.uninstall_repository( convert_repository, remove_from_disk=False )
+        self.deactivate_repository( filter_repository )
+        self.deactivate_repository( column_repository )
+        self.deactivate_repository( convert_repository )
         
-    def test_0045_reinstall_filter_repository( self ):
+    def test_0045_reactivate_filter_repository( self ):
         '''Reinstall the filtering_0160 repository.'''
         filter_repository = self.test_db_util.get_installed_repository_by_name_owner( filter_repository_name, common.test_user_1_name )
-        self.reinstall_repository( filter_repository )
+        self.reactivate_repository( filter_repository )
         strings_displayed = [ 'filtering_0160',
                               "Galaxy's filtering tool for test 0160",
                               'user1', 
@@ -242,6 +242,7 @@ class TestSimplePriorInstallation( ShedTwillTestCase ):
         
     def test_0050_verify_reinstallation_order( self ):
         '''Verify that convert_chars_0160 and column_maker_0160 were reinstalled before filtering_0160.'''
+        # Fixme: this test is not covering any important behavior since repositories were only deactivated and not uninstalled.
         filter_repository = self.test_db_util.get_installed_repository_by_name_owner( filter_repository_name, common.test_user_1_name )
         column_repository = self.test_db_util.get_installed_repository_by_name_owner( column_repository_name, common.test_user_1_name )
         convert_repository = self.test_db_util.get_installed_repository_by_name_owner( convert_repository_name, common.test_user_1_name )

test/tool_shed/functional/test_1200_uninstall_and_reinstall_basic_repository.py

@@ -1,7 +1,9 @@
 from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os
 
+
 class UninstallingAndReinstallingRepositories( ShedTwillTestCase ):
     '''Test uninstalling and reinstalling a basic repository.'''
+
     def test_0000_initiate_users( self ):
         """Create necessary user accounts."""
         self.logout()
@@ -19,6 +21,7 @@ class UninstallingAndReinstallingRepositories( ShedTwillTestCase ):
         galaxy_admin_user = self.test_db_util.get_galaxy_user( common.admin_email )
         assert galaxy_admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email
         galaxy_admin_user_private_role = self.test_db_util.get_galaxy_private_role( galaxy_admin_user )
+
     def test_0005_ensure_repositories_and_categories_exist( self ):
         '''Create the 0000 category and upload the filtering repository to the tool shed, if necessary.'''
         category = self.create_category( name='Test 0000 Basic Repository Features 1', description='Test 0000 Basic Repository Features 1' )
@@ -67,6 +70,7 @@ class UninstallingAndReinstallingRepositories( ShedTwillTestCase ):
                               commit_message='Uploaded readme for 2.2.0',
                               strings_displayed=[], 
                               strings_not_displayed=[] )
+
     def test_0010_install_filtering_repository( self ):
         '''Install the filtering repository into the Galaxy instance.'''
         self.galaxy_logout()
@@ -82,14 +86,16 @@ class UninstallingAndReinstallingRepositories( ShedTwillTestCase ):
                               self.url.replace( 'http://', '' ), 
                               installed_repository.installed_changeset_revision ]
         self.display_galaxy_browse_repositories_page( strings_displayed=strings_displayed )
+
     def test_0015_uninstall_filtering_repository( self ):
         '''Uninstall the filtering repository.'''
         installed_repository = self.test_db_util.get_installed_repository_by_name_owner( 'filtering_0000', common.test_user_1_name )
-        self.uninstall_repository( installed_repository, remove_from_disk=True )
+        self.uninstall_repository( installed_repository )
         strings_not_displayed = [ 'filtering_0000',
                                   "Galaxy's filtering tool for test 0000",
                                   installed_repository.installed_changeset_revision ]
         self.display_galaxy_browse_repositories_page( strings_not_displayed=strings_not_displayed )
+
     def test_0020_reinstall_filtering_repository( self ):
         '''Reinstall the filtering repository.'''
         installed_repository = self.test_db_util.get_installed_repository_by_name_owner( 'filtering_0000', common.test_user_1_name )
@@ -103,14 +109,16 @@ class UninstallingAndReinstallingRepositories( ShedTwillTestCase ):
         strings_displayed.extend( [ 'Installed tool shed repository', 'Valid tools', 'Filter1' ] )
         self.display_installed_repository_manage_page( installed_repository, strings_displayed=strings_displayed )
         self.verify_tool_metadata_for_installed_repository( installed_repository )
+
     def test_0025_deactivate_filtering_repository( self ):
         '''Deactivate the filtering repository without removing it from disk.'''
         installed_repository = self.test_db_util.get_installed_repository_by_name_owner( 'filtering_0000', common.test_user_1_name )
-        self.uninstall_repository( installed_repository, remove_from_disk=False )
+        self.deactivate_repository( installed_repository )
         strings_not_displayed = [ 'filtering_0000',
                                   "Galaxy's filtering tool for test 0000",
                                   installed_repository.installed_changeset_revision ]
         self.display_galaxy_browse_repositories_page( strings_not_displayed=strings_not_displayed )
+
     def test_0030_reactivate_filtering_repository( self ):
         '''Reactivate the filtering repository and verify that it now shows up in the list of installed repositories.'''
         installed_repository = self.test_db_util.get_installed_repository_by_name_owner( 'filtering_0000', common.test_user_1_name )

test/tool_shed/functional/test_1210_uninstall_reinstall_repository_with_tool_dependencies.py

@@ -1,7 +1,9 @@
 from tool_shed.base.twilltestcase import ShedTwillTestCase, common, os
 
+
 class UninstallingAndReinstallingRepositories( ShedTwillTestCase ):
     '''Test uninstalling and reinstalling a repository with tool dependencies.'''
+
     def test_0000_initiate_users( self ):
         """Create necessary user accounts."""
         self.galaxy_logout()
@@ -19,6 +21,7 @@ class UninstallingAndReinstallingRepositories( ShedTwillTestCase ):
         admin_user = self.test_db_util.get_user( common.admin_email )
         assert admin_user is not None, 'Problem retrieving user with email %s from the database' % common.admin_email
         admin_user_private_role = self.test_db_util.get_private_role( admin_user )
+
     def test_0005_ensure_repositories_and_categories_exist( self ):
         '''Create the 0010 category and upload the freebayes repository to the tool shed, if necessary.'''
         category = self.create_category( name='Test 0010 Repository With Tool Dependencies', description='Tests for a repository with tool dependencies.' )
@@ -84,6 +87,7 @@ class UninstallingAndReinstallingRepositories( ShedTwillTestCase ):
                               commit_message='Uploaded valid tool dependency XML.',
                               strings_displayed=[], 
                               strings_not_displayed=[] )
+
     def test_0010_install_freebayes_repository( self ):
         '''Install the freebayes repository into the Galaxy instance.'''
         self.galaxy_logout()
@@ -101,12 +105,14 @@ class UninstallingAndReinstallingRepositories( ShedTwillTestCase ):
                               self.url.replace( 'http://', '' ), 
                               installed_repository.installed_changeset_revision ]
         self.display_galaxy_browse_repositories_page( strings_displayed=strings_displayed )
+
     def test_0015_uninstall_freebayes_repository( self ):
         '''Uninstall the freebayes repository.'''
         installed_repository = self.test_db_util.get_installed_repository_by_name_owner( 'freebayes_0010', common.test_user_1_name )
-        self.uninstall_repository( installed_repository, remove_from_disk=True )
+        self.uninstall_repository( installed_repository )
         strings_not_displayed = [ installed_repository.name, installed_repository.installed_changeset_revision ]
         self.display_galaxy_browse_repositories_page( strings_not_displayed=strings_not_displayed )
+
     def test_0020_reinstall_freebayes_repository( self ):
         '''Reinstall the freebayes repository.'''
         installed_repository = self.test_db_util.get_installed_repository_by_name_owner( 'freebayes_0010', common.test_user_1_name )
@@ -120,12 +126,14 @@ class UninstallingAndReinstallingRepositories( ShedTwillTestCase ):
         strings_displayed.extend( [ 'Installed tool shed repository', 'Valid tools', 'FreeBayes' ] )
         self.display_installed_repository_manage_page( installed_repository, strings_displayed=strings_displayed )
         self.verify_tool_metadata_for_installed_repository( installed_repository )
+
     def test_0025_deactivate_freebayes_repository( self ):
         '''Deactivate the freebayes repository without removing it from disk.'''
         installed_repository = self.test_db_util.get_installed_repository_by_name_owner( 'freebayes_0010', common.test_user_1_name )
-        self.uninstall_repository( installed_repository, remove_from_disk=False )
+        self.deactivate_repository( installed_repository )
         strings_not_displayed = [ installed_repository.name, installed_repository.installed_changeset_revision ]
         self.display_galaxy_browse_repositories_page( strings_not_displayed=strings_not_displayed )
+
     def test_0030_reactivate_freebayes_repository( self ):
         '''Reactivate the freebayes repository and verify that it now shows up in the list of installed repositories.'''
         installed_repository = self.test_db_util.get_installed_repository_by_name_owner( 'freebayes_0010', common.test_user_1_name )

test/tool_shed/functional/test_1220_uninstall_reinstall_repository_with_repository_dependencies.py

@@ -108,7 +108,7 @@ class UninstallingAndReinstallingRepositories( ShedTwillTestCase ):
     def test_0015_uninstall_emboss_repository( self ):
         '''Uninstall the emboss repository.'''
         installed_repository = self.test_db_util.get_installed_repository_by_name_owner( emboss_repository_name, common.test_user_1_name )
-        self.uninstall_repository( installed_repository, remove_from_disk=True )
+        self.uninstall_repository( installed_repository )
         strings_not_displayed = [ installed_repository.name, installed_repository.installed_changeset_revision ]
         self.display_galaxy_browse_repositories_page( strings_not_displayed=strings_not_displayed )
 
@@ -129,7 +129,7 @@ class UninstallingAndReinstallingRepositories( ShedTwillTestCase ):
     def test_0025_deactivate_emboss_repository( self ):
         '''Deactivate the emboss repository without removing it from disk.'''
         installed_repository = self.test_db_util.get_installed_repository_by_name_owner( emboss_repository_name, common.test_user_1_name )
-        self.uninstall_repository( installed_repository, remove_from_disk=False )
+        self.deactivate_repository( installed_repository )
         strings_not_displayed = [ 'emboss_0020',
                                   installed_repository.installed_changeset_revision ]
         self.display_galaxy_browse_repositories_page( strings_not_displayed=strings_not_displayed )

test/tool_shed/functional/test_1230_uninstall_reinstall_repository_with_dependency_revisions.py

@@ -144,7 +144,7 @@ class UninstallingAndReinstallingRepositories( ShedTwillTestCase ):
     def test_0015_uninstall_emboss_repository( self ):
         '''Uninstall the emboss repository.'''
         installed_repository = self.test_db_util.get_installed_repository_by_name_owner( emboss_repository_name, common.test_user_1_name )
-        self.uninstall_repository( installed_repository, remove_from_disk=True )
+        self.uninstall_repository( installed_repository )
         strings_not_displayed = [ installed_repository.installed_changeset_revision ]
         self.display_galaxy_browse_repositories_page( strings_not_displayed=strings_not_displayed )
 
@@ -165,7 +165,7 @@ class UninstallingAndReinstallingRepositories( ShedTwillTestCase ):
     def test_0025_deactivate_emboss_repository( self ):
         '''Deactivate the emboss repository without removing it from disk.'''
         installed_repository = self.test_db_util.get_installed_repository_by_name_owner( emboss_repository_name, common.test_user_1_name )
-        self.uninstall_repository( installed_repository, remove_from_disk=False )
+        self.deactivate_repository( installed_repository )
         strings_not_displayed = [ installed_repository.installed_changeset_revision ]
         self.display_galaxy_browse_repositories_page( strings_not_displayed=strings_not_displayed )
  

test/tool_shed/functional/test_1430_repair_installed_repository.py

@@ -146,10 +146,9 @@ class TestRepairRepository( ShedTwillTestCase ):
         installed_repository = self.test_db_util.get_installed_repository_by_name_owner( 'filter_1430', common.test_user_1_name )
         strings_displayed = [ 'This repository cannot be uninstalled' ]
         strings_not_displayed = [ 'Check to uninstall' ]
-        self.uninstall_repository( installed_repository, 
-                                   remove_from_disk=False, 
-                                   strings_displayed=strings_displayed, 
-                                   strings_not_displayed=strings_not_displayed  )
+        self.deactivate_repository( installed_repository, 
+                                    strings_displayed=strings_displayed, 
+                                    strings_not_displayed=strings_not_displayed  )
         strings_not_displayed = [ 'filter_1430',
                                   "Galaxy's filter tool for test 1430",
                                   installed_repository.installed_changeset_revision ]

test/tool_shed/functional/test_1450_installing_datatypes_sniffers.py

@@ -119,7 +119,7 @@ class TestInstallDatatypesSniffers( ShedTwillTestCase ):
         global repository_datatypes_count
         global base_datatypes_count
         global base_sniffers_count
-        self.uninstall_repository( repository, remove_from_disk=False )
+        self.deactivate_repository( repository )
         new_datatypes_count = self.get_datatypes_count()
         assert new_datatypes_count == base_datatypes_count, 'Expected %d datatypes, got %d' % ( base_datatypes_count, new_datatypes_count )
         current_sniffers = self.get_sniffers_count()
@@ -157,7 +157,7 @@ class TestInstallDatatypesSniffers( ShedTwillTestCase ):
         repository = self.test_db_util.get_installed_repository_by_name_owner( repository_name, common.test_user_1_name )
         global repository_datatypes_count
         global base_datatypes_count
-        self.uninstall_repository( repository, remove_from_disk=True )
+        self.uninstall_repository( repository )
         new_datatypes_count = self.get_datatypes_count()
         assert new_datatypes_count == base_datatypes_count, 'Expected %d datatypes, got %d' % ( base_datatypes_count, new_datatypes_count )
         current_sniffers = self.get_sniffers_count()

test/unit/test_galaxy_mapping.py

@@ -180,6 +180,41 @@ class MappingTests( unittest.TestCase ):
         assert hist1.name == "History 2b"
         # gvk TODO need to ad test for GalaxySessions, but not yet sure what they should look like.
 
+    def test_history_contents( self ):
+        model = self.model
+        u = model.User( email="contents@foo.bar.baz", password="password" )
+        # gs = model.GalaxySession()
+        h1 = model.History( name="HistoryContentsHistory1", user=u)
+
+        self.persist( u, h1, expunge=False )
+
+        d1 = self.new_hda( h1, name="1" )
+        d2 = self.new_hda( h1, name="2", visible=False )
+        d3 = self.new_hda( h1, name="3", deleted=True )
+        d4 = self.new_hda( h1, name="4", visible=False, deleted=True )
+
+        self.session().flush()
+
+        def contents_iter_names(**kwds):
+            history = model.context.query( model.History ).filter(
+                model.History.name == "HistoryContentsHistory1"
+            ).first()
+            return list( map( lambda hda: hda.name, history.contents_iter( **kwds ) ) )
+
+        self.assertEquals(contents_iter_names(), [ "1", "2", "3", "4" ])
+        assert contents_iter_names( deleted=False ) == [ "1", "2" ]
+        assert contents_iter_names( visible=True ) == [ "1", "3" ]
+        assert contents_iter_names( visible=False ) == [ "2", "4" ]
+        assert contents_iter_names( deleted=True, visible=False ) == [ "4" ]
+
+        assert contents_iter_names( ids=[ d1.id, d2.id, d3.id, d4.id ] ) == [ "1", "2", "3", "4" ]
+        assert contents_iter_names( ids=[ d1.id, d2.id, d3.id, d4.id ], max_in_filter_length=1 ) == [ "1", "2", "3", "4" ]
+
+        assert contents_iter_names( ids=[ d1.id, d3.id ] ) == [ "1", "3" ]
+
+    def new_hda( self, history, **kwds ):
+        return history.add_dataset( self.model.HistoryDatasetAssociation( create_dataset=True, sa_session=self.model.session, **kwds ) )
+
     @classmethod
     def setUpClass(cls):
         # Start the database and connect the mapping
@@ -191,10 +226,20 @@ class MappingTests( unittest.TestCase ):
         return cls.model.session.query( type )
 
     @classmethod
-    def persist(cls, *args):
+    def persist(cls, *args, **kwargs):
+        session = cls.session()
+        flush = kwargs.get('flush', True)
         for arg in args:
-            cls.model.session.add( arg )
-        cls.expunge()
+            session.add( arg )
+            if flush:
+                session.flush()
+        if kwargs.get('expunge', not flush):
+            cls.expunge()
+        return arg  # Return last or only arg.
+
+    @classmethod
+    def session(cls):
+        return cls.model.session
 
     @classmethod
     def expunge(cls):

tools/genomespace/genomespace_file_browser.py

@@ -9,11 +9,16 @@ import urllib2
 import urlparse
 
 
+import galaxy.model # need to import model before sniff to resolve a circular import dependency
+from galaxy.datatypes import sniff
+from galaxy.datatypes.registry import Registry
+
 GENOMESPACE_API_VERSION_STRING = "v1.0"
 GENOMESPACE_SERVER_URL_PROPERTIES = "https://dm.genomespace.org/config/%s/serverurl.properties" % ( GENOMESPACE_API_VERSION_STRING )
 
 CHUNK_SIZE = 2**20 #1mb
 
+AUTO_GALAXY_EXT = "auto"
 DEFAULT_GALAXY_EXT = "data"
 
 #genomespace format identifier is the URL
@@ -40,6 +45,9 @@ GENOMESPACE_EXT_TO_GALAXY_EXT = {'rifles': 'rifles',
                                  'gmt': 'gmt', 
                                  'gct': 'gct'}
 
+GENOMESPACE_UNKNOWN_FORMAT_KEY = 'unknown'
+GENOMESPACE_FORMAT_IDENTIFIER_UNKNOWN = None
+
 VALID_CHARS = '.-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ '
 
 def chunk_write( source_stream, target_stream, source_method = "read", target_method="write" ):
@@ -68,7 +76,7 @@ def get_galaxy_ext_from_genomespace_format_url( url_opener, file_format_url ):
         ext = GENOMESPACE_EXT_TO_GALAXY_EXT.get( ext, None )
     if ext is None:
         #could check content type, etc here
-        ext = DEFAULT_GALAXY_EXT
+        ext = AUTO_GALAXY_EXT
     return ext
 
 def get_genomespace_site_urls():
@@ -91,6 +99,8 @@ def set_genomespace_format_identifiers( url_opener, dm_site ):
     genomespace_formats = json.loads( opened_gs_request.read() )
     for format in genomespace_formats:
         GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT[ format['url'] ] = format['name']
+    global GENOMESPACE_FORMAT_IDENTIFIER_UNKNOWN
+    GENOMESPACE_FORMAT_IDENTIFIER_UNKNOWN = dict( map( lambda x: ( x[1], x[0] ) , GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT.iteritems() ) ).get( GENOMESPACE_UNKNOWN_FORMAT_KEY, GENOMESPACE_FORMAT_IDENTIFIER_UNKNOWN )
 
 def download_from_genomespace_file_browser( json_parameter_file, genomespace_site ):
     json_params = json.loads( open( json_parameter_file, 'r' ).read() )
@@ -109,6 +119,11 @@ def download_from_genomespace_file_browser( json_parameter_file, genomespace_sit
     file_url_prefix = "fileUrl"
     file_type_prefix = "fileFormat"
     metadata_parameter_file = open( json_params['job_config']['TOOL_PROVIDED_JOB_METADATA_FILE'], 'wb' )
+    
+    #setup datatypes registry for sniffing
+    datatypes_registry = Registry()
+    datatypes_registry.load_datatypes( root_dir = json_params[ 'job_config' ][ 'GALAXY_ROOT_DIR' ], config = json_params[ 'job_config' ][ 'GALAXY_DATATYPES_CONF_FILE' ] )
+    
     file_numbers = []
     for name in datasource_params.keys():
         if name.startswith( file_url_prefix ):
@@ -144,28 +159,47 @@ def download_from_genomespace_file_browser( json_parameter_file, genomespace_sit
             filename = urllib.unquote_plus( parsed_url[2].split( '/' )[-1] )
         if not filename:
             filename = download_url
+        metadata_dict = None
+        original_filename = filename
         if output_filename is None:
-            original_filename = filename
             filename = ''.join( c in VALID_CHARS and c or '-' for c in filename )
             while filename in used_filenames:
                 filename = "-%s" % filename
             used_filenames.append( filename )
             output_filename = os.path.join( datasource_params['__new_file_path__'],  'primary_%i_%s_visible_%s' % ( hda_id, filename, galaxy_ext ) )
-            metadata_parameter_file.write( "%s\n" % json.dumps( dict( type = 'new_primary_dataset',
-                                     base_dataset_id = dataset_id,
-                                     ext = galaxy_ext,
-                                     filename = output_filename,
-                                     name = "GenomeSpace import on %s" % ( original_filename ) ) ) )
+            
+            metadata_dict = dict( type = 'new_primary_dataset',
+                                base_dataset_id = dataset_id,
+                                ext = galaxy_ext,
+                                filename = output_filename,
+                                name = "GenomeSpace import on %s" % ( original_filename ) )
         else:
             if dataset_id is not None:
-               metadata_parameter_file.write( "%s\n" % json.dumps( dict( type = 'dataset',
-                                     dataset_id = dataset_id,
-                                     ext = galaxy_ext,
-                                     name = "GenomeSpace import on %s" % ( filename ) ) ) )
+                metadata_dict = dict( type = 'dataset',
+                                dataset_id = dataset_id,
+                                ext = galaxy_ext,
+                                name = "GenomeSpace import on %s" % ( filename ) )
         output_file = open( output_filename, 'wb' )
         chunk_write( target_download_url, output_file )
         output_file.close()
+        
+        if ( galaxy_ext == AUTO_GALAXY_EXT or filetype_url == GENOMESPACE_FORMAT_IDENTIFIER_UNKNOWN ) and metadata_dict:
+            #try to sniff datatype
+            try:
+                galaxy_ext = sniff.handle_uploaded_dataset_file( output_filename, datatypes_registry )
+            except:
+                #sniff failed
+                galaxy_ext = original_filename.rsplit( '.', 1 )[-1]
+                if galaxy_ext not in datatypes_registry.datatypes_by_extension:
+                    galaxy_ext = DEFAULT_GALAXY_EXT
+            metadata_dict[ 'ext' ] = galaxy_ext
+        
         output_filename = None #only have one filename available
+        
+        #write out metadata info
+        if metadata_dict:
+            metadata_parameter_file.write( "%s\n" % json.dumps( metadata_dict ) )
+        
     metadata_parameter_file.close()
     return True