Merge branch 'dev' of git://github.com/galaxyproject/galaxy into toolshed_decouple

client/galaxy/scripts/layout/menu.js

@@ -274,7 +274,7 @@ var Tab = Backbone.View.extend({
         this.$toggle.html( this.model.get( 'title' ) || '' )
                     .removeClass().addClass( 'dropdown-toggle' )
                     .addClass( this.model.get( 'cls' ) )
-                    .addClass( this.model.get( 'icon' ) && 'fa fa-2x ' + this.model.get( 'icon' ) )
+                    .addClass( this.model.get( 'icon' ) && 'dropdown-icon fa ' + this.model.get( 'icon' ) )
                     .addClass( this.model.get( 'toggle' ) && 'toggle' )
                     .attr( 'target', this.model.get( 'target' ) )
                     .attr( 'href', this.model.get( 'url' ) )

client/galaxy/scripts/mvc/history/history-contents.js

@@ -2,9 +2,10 @@ define([
     "mvc/history/history-content-model",
     "mvc/history/hda-model",
     "mvc/history/hdca-model",
+    "mvc/dataset/states",
     "mvc/base-mvc",
     "utils/localization"
-], function( HISTORY_CONTENT, HDA_MODEL, HDCA_MODEL, BASE_MVC, _l ){
+], function( HISTORY_CONTENT, HDA_MODEL, HDCA_MODEL, STATES, BASE_MVC, _l ){
 
 'use strict';
 
@@ -102,15 +103,8 @@ var HistoryContents = Backbone.Collection
      *  @see HistoryDatasetAssociation#inReadyState
      */
     running : function(){
-        var idList = [];
-        this.each( function( item ){
-            var isRunning = !item.inReadyState();
-            if( isRunning ){
-//TODO: is this still correct since type_id
-                idList.push( item.get( 'id' ) );
-            }
-        });
-        return idList;
+        function filterFn( c ){ return !c.inReadyState(); }
+        return new HistoryContents( this.filter( filterFn ) );
     },
 
     /** Get the model with the given hid
@@ -156,12 +150,53 @@ var HistoryContents = Backbone.Collection
         return new HistoryContents( this.filter( filterFn ) );
     },
 
+    /** return a new contents collection of only hidden items */
+    visibleAndUndeleted : function(){
+        function filterFn( c ){ return c.get( 'visible' ) && !c.get( 'deleted' ); }
+        return new HistoryContents( this.filter( filterFn ) );
+    },
+
     /** return true if any contents don't have details */
     haveDetails : function(){
         return this.all( function( content ){ return content.hasDetails(); });
     },
 
     // ........................................................................ ajax
+    /** override to use newest (versioned) api */
+    fetch : function( options ){
+        options = options || {};
+        options.data = _.defaults( options.data || {}, {
+            v : 'dev'
+        });
+        return Backbone.Collection.prototype.fetch.call( this, options );
+    },
+
+    /** override to use newest (versioned) api */
+    fetchUpdated : function( since, options ){
+        options = options || {};
+        options.traditional = true;
+        // TODO: this is painful - simplify here or move q/qv to named/mappable params
+        options.data = [{ name: 'v', value: 'dev' }];
+        if( since ){
+            options.data = options.data.concat( this._filtersFromMap({
+                'update_time-ge' : since.toISOString(),
+            }));
+        }
+        options.merge = true;
+        options.remove = false;
+        return this.fetch( options );
+    },
+
+    _filtersFromMap : function( filterMap ){
+        var filters = [];
+        // TODO: this seems unnecessary
+        _.each( filterMap, function( val, key ){
+            filters.push({ name: 'q',  value: key });
+            filters.push({ name: 'qv', value: val });
+        });
+        return filters;
+    },
+
     /** fetch detailed model data for all contents in this collection */
     fetchAllDetails : function( options ){
         options = options || {};
@@ -293,7 +328,6 @@ var HistoryContents = Backbone.Collection
             });
     },
 
-
     /** In this override, copy the historyId to the clone */
     clone : function(){
         var clone = Backbone.Collection.prototype.clone.call( this );

client/galaxy/scripts/mvc/history/history-model.js

@@ -137,6 +137,18 @@ var History = Backbone.Model
         return _.reduce( _.values( this.get( 'state_details' ) ), function( memo, num ){ return memo + num; }, 0 );
     },
 
+    /** Return the number of running jobs assoc with this history (note: unknown === 0) */
+    numOfUnfinishedJobs : function(){
+        var unfinishedJobIds = this.get( 'non_ready_jobs' );
+        return unfinishedJobIds? unfinishedJobIds.length : 0;
+    },
+
+    /** Return the number of running hda/hdcas in this history (note: unknown === 0) */
+    numOfUnfinishedShownContents : function(){
+        var contents = this.contents.running().visibleAndUndeleted();
+        return contents? contents.length : 0;
+    },
+
     // ........................................................................ search
     /** What model fields to search with */
     searchAttributes : [
@@ -150,39 +162,56 @@ var History = Backbone.Model
     },
 
     // ........................................................................ updates
-    /** does the contents collection indicate they're still running and need to be updated later?
-     *      delay + update if needed
-     *  @param {Function} onReadyCallback   function to run when all contents are in the ready state
-     *  events: ready
-     */
-    checkForUpdates : function( onReadyCallback ){
-        //this.info( 'checkForUpdates' )
-
-        // get overall History state from collection, run updater if History has running/queued contents
-        //  boiling it down on the client to running/not
-        if( this.contents.running().length ){
-            this.setUpdateTimeout();
-
-        } else {
-            this.trigger( 'ready' );
-            if( _.isFunction( onReadyCallback ) ){
-                onReadyCallback.call( this );
-            }
-        }
-        return this;
+    _getSizeAndRunning : function(){
+        return this.fetch({ data : $.param({ keys : 'size,non_ready_jobs' }) });
     },
 
-    /** create a timeout (after UPDATE_DELAY or delay ms) to refetch the contents. Clear any prev. timeout */
-    setUpdateTimeout : function( delay ){
-        delay = delay || History.UPDATE_DELAY;
-        var history = this;
+    /**  */
+    refresh : function( options ){
+        options = options || {};
+        var self = this;
 
-        // prevent buildup of updater timeouts by clearing previous if any, then set new and cache id
-        this.clearUpdateTimeout();
-        this.updateTimeoutId = setTimeout( function(){
-            history.refresh();
-        }, delay );
-        return this.updateTimeoutId;
+        var lastUpdateTime = self.lastUpdateTime;
+        self.lastUpdateTime = new Date();
+        // note if there was no previous update time, all summary contents will be fetched
+        return self.contents.fetchUpdated( lastUpdateTime )
+            .done( _.bind( self.checkForUpdates, self ) );
+    },
+
+    /**  */
+    checkForUpdates : function( options ){
+        options = options || {};
+        var delay = History.UPDATE_DELAY;
+        var self = this;
+
+        function _delayThenUpdate(){
+            // prevent buildup of updater timeouts by clearing previous if any, then set new and cache id
+            self.clearUpdateTimeout();
+            self.updateTimeoutId = setTimeout( function(){
+                self.refresh( options );
+            }, delay );
+        }
+
+        // if there are still datasets in the non-ready state, recurse into this function with the new time
+        if( this.numOfUnfinishedShownContents() > 0 ){
+            _delayThenUpdate();
+
+        } else {
+            // no datasets are running, but currently runnning jobs may still produce new datasets
+            // see if the history has any running jobs and continue to update if so
+            // (also update the size for the user in either case)
+            self._getSizeAndRunning()
+                .done( function( historyData ){
+                    if( self.numOfUnfinishedJobs() > 0 ){
+                        _delayThenUpdate();
+
+                    } else {
+                        // otherwise, let listeners know that all updates have stopped
+                        self.trigger( 'ready' );
+                        // self.lastUpdateTime = null;
+                    }
+                });
+        }
     },
 
     /** clear the timeout and the cached timeout id */
@@ -193,32 +222,6 @@ var History = Backbone.Model
         }
     },
 
-    /* update the contents, getting full detailed model data for any whose id is in detailIds
-     *  set up to run this again in some interval of time
-     *  @param {String[]} detailIds list of content ids to get detailed model data for
-     *  @param {Object} options     std. backbone fetch options map
-     */
-    refresh : function( detailIds, options ){
-        //this.info( 'refresh:', detailIds, this.contents );
-        detailIds = detailIds || [];
-        options = options || {};
-        var history = this;
-
-        // add detailIds to options as CSV string
-        options.data = options.data || {};
-        if( detailIds.length ){
-            options.data.details = detailIds.join( ',' );
-        }
-        var xhr = this.contents.fetch( options );
-        xhr.done( function( models ){
-            history.checkForUpdates( function(){
-                // fetch the history inside onReadyCallback in order to recalc history size
-                this.fetch();
-            });
-        });
-        return xhr;
-    },
-
     // ........................................................................ ajax
     /** save this history, _Mark_ing it as deleted (just a flag) */
     _delete : function( options ){
@@ -330,14 +333,11 @@ History.getHistoryData = function getHistoryData( historyId, options ){
         if( _.isFunction( hdcaDetailIds ) ){
             hdcaDetailIds = hdcaDetailIds( historyData );
         }
-        var data = {};
+        var data = {
+            v : 'dev'
+        };
         if( detailIdsFn.length ) {
-            data.dataset_details = detailIdsFn.join( ',' );
-        }
-        if( hdcaDetailIds.length ) {
-            // for symmetry, not actually used by backend of consumed
-            // by frontend.
-            data.dataset_collection_details = hdcaDetailIds.join( ',' );
+            data.details = detailIdsFn.join( ',' );
         }
         return jQuery.ajax( Galaxy.root + 'api/histories/' + historyData.id + '/contents', { data: data });
     }

client/galaxy/scripts/mvc/history/history-view.js

@@ -131,8 +131,8 @@ var HistoryView = _super.extend(
     /** In this override, clear the update timer on the model */
     freeModel : function(){
         _super.prototype.freeModel.call( this );
-//TODO: move to History.free()
         if( this.model ){
+//TODO: move to History.free()
             this.model.clearUpdateTimeout();
         }
         return this;
@@ -221,9 +221,9 @@ var HistoryView = _super.extend(
     },
 
     /** convenience alias to the model. Updates the item list only (not the history) */
-    refreshContents : function( detailIds, options ){
+    refreshContents : function( options ){
         if( this.model ){
-            return this.model.refresh( detailIds, options );
+            return this.model.refresh( options );
         }
         // may have callbacks - so return an empty promise
         return $.when();

client/galaxy/scripts/mvc/history/multi-panel.js

@@ -453,7 +453,7 @@ var MultiPanelColumns = Backbone.View.extend( baseMVC.LoggableMixin ).extend({
     handleDeletedHistory : function handleDeletedHistory( history ){
         if( history.get( 'deleted' ) || history.get( 'purged' ) ){
             this.log( 'handleDeletedHistory', this.collection.includeDeleted, history );
-            var multipanel = this;
+            var multipanel = this,
                 column = multipanel.columnMap[ history.id ];
             if( !column ){ return; }
 
@@ -723,7 +723,7 @@ var MultiPanelColumns = Backbone.View.extend( baseMVC.LoggableMixin ).extend({
             var xhrData = {},
                 ids = _.values( column.panel.storage.get( 'expandedIds' ) ).join();
             if( ids ){
-                xhrData.dataset_details = ids;
+                xhrData.details = ids;
             }
             // this uses a 'named' queue so that duplicate requests are ignored
             this.hdaQueue.add({

client/galaxy/scripts/mvc/tool/tool-form.js

@@ -14,7 +14,7 @@ define(['utils/utils', 'mvc/ui/ui-misc', 'mvc/tool/tool-form-base', 'mvc/tool/to
                             icon     : 'fa-check',
                             tooltip  : 'Execute: ' + options.name + ' (' + options.version + ')',
                             title    : 'Execute',
-                            cls      : 'btn btn-primary',
+                            cls      : 'ui-button btn btn-primary',
                             floating : 'clear',
                             onclick  : function() {
                                 execute_btn.wait();

client/galaxy/style/less/base.less

@@ -430,6 +430,10 @@ div.unified-panel-body-background {
         .toggle {
             color           : gold;
         }
+        .dropdown-icon {
+            top             : 1px;
+            font-size       : 1.8em;
+        }
         .dropdown-note {
             font-weight     : bold;
             font-size       : 10px;

client/galaxy/style/less/ui.less

@@ -111,7 +111,7 @@
 // buttons
 .ui-button {
     i {
-        font-size: 1.2em;
+        font-size: 1.1em;
     }
 }
 .ui-button-icon {
@@ -579,6 +579,7 @@
     
     i {
         padding-right: @ui-margin-horizontal;
+        font-size: 1.1em;
     }
 }
 
@@ -670,7 +671,7 @@
     position: relative;
     .icon-dropdown {
         position: absolute;
-        top: 8px;
+        top: 7px;
         right: 8px;
         cursor: pointer;
     }

client/galaxy/style/less/upload.less

@@ -63,6 +63,7 @@
             color: @btn-default-border;
             i {
                 margin-right: 10px;
+                font-size: inherit;
             }
         }
         .upload-row {

contrib/galaxy.fedora-init

@@ -8,6 +8,8 @@
 # chkconfig: 2345 98 20
 # description: Galaxy http://galaxyproject.org/
 
+#--- loading functions
+. /etc/init.d/functions
 #--- config
 
 SERVICE_NAME="galaxy"
@@ -53,17 +55,17 @@ stop() {
 	echo "done."
 }
 
-status() {
-	echo -n "$SERVICE_NAME status: "
-	
-	while read pid; do
-		if [ "$(readlink -m /proc/$pid/cwd)" = "$(readlink -m $RUN_IN)" ]; then
-			echo "started"
-			return 0
-		fi
-	done < <(ps ax -o 'pid cmd' | grep -P '^\s*\d+ python ./scripts/paster.py serve' | awk '{print $1}')
-	echo "stopped"
-	return 3
+galaxy_status() {
+	if [[ $(grep '\[server:' $RUN_IN/config/galaxy.ini|awk -F'(:)|(])' '{ print $2 }') == 'main' ]]
+	then
+		echo -n "$SERVICE_NAME status: "
+		status -p $RUN_IN/paster.pid galaxy
+	else
+		for proc in $(grep '\[server:' $RUN_IN/config/galaxy.ini|awk -F'(:)|(])' '{ print $2 }')
+		do
+			status -p $RUN_IN/${proc}.pid ${proc}
+		done
+	fi
 }
 
 notsupported() {
@@ -90,7 +92,7 @@ case "$1" in
 		;;
 	status)
 		set +e
-		status
+		galaxy_status
 		exit $?
 		;;
 	'')

contrib/galaxy_reports.fedora-init

@@ -0,0 +1,95 @@
+#!/bin/bash
+#
+# Init file for Galaxy (http://galaxyproject.org/)
+#   Suitable for use on Fedora and derivatives (RedHat Enterprise Linux, Scientific Linux, CentOS)
+#
+# Contributed by Brad Chapman
+#
+# chkconfig: 2345 98 20
+# description: Galaxy http://galaxyproject.org/
+
+#--- loading functions
+. /etc/init.d/functions
+#--- config
+
+SERVICE_NAME="galaxy-reports"
+RUN_AS="galaxy"
+RUN_IN="/path/to/galaxy-dist"
+
+#--- main actions
+
+start() {
+	echo "Starting $SERVICE_NAME... "
+	cmd="cd $RUN_IN && sh run_reports.sh --daemon"
+	case "$(id -un)" in
+		$RUN_AS)
+			eval "$cmd"
+			;;
+		root)
+			su - $RUN_AS -c "$cmd"
+			;;
+		*)
+			echo "*** ERROR *** must be $RUN_AS or root in order to control this service" >&2
+			exit 1
+	esac
+	echo "...done."
+}
+
+stop() {
+	echo -n "Stopping $SERVICE_NAME... "
+	
+	cmd="cd $RUN_IN && sh run_reports.sh --stop-daemon"
+
+	case "$(id -un)" in
+		$RUN_AS)
+			eval "$cmd"
+			;;
+		root)
+			su - $RUN_AS -c "$cmd"
+			;;
+		*)
+			echo "*** ERROR *** must be $RUN_AS or root in order to control this service" >&2
+			exit 1
+	esac
+	
+	echo "done."
+}
+
+notsupported() {
+	echo "*** ERROR*** $SERVICE_NAME: operation [$1] not supported"
+}
+
+usage() {
+	echo "Usage: $SERVICE_NAME start|stop|restart|status"
+}
+
+
+#---
+
+case "$1" in
+	start)
+		start "$@"
+		;;
+	stop)
+		stop
+		;;
+	restart|reload)
+		stop
+		start
+		;;
+	status)
+		set +e
+		echo -n "$SERVICE_NAME status: "
+		status -p $RUN_IN/reports_webapp.pid $SERVICE_NAME
+		exit $?
+		;;
+	'')
+		usage >&2
+		exit 1
+		;;
+	*)
+		notsupported "$1" >&2
+		usage >&2
+		exit 1
+		;;
+esac

doc/source/admin/framework_dependencies.rst

@@ -125,10 +125,10 @@ virtualenv using the ``--no-create-venv`` option:
       repoze.lru, Routes, WebOb, WebHelpers, Mako, pytz, Babel, Beaker,
       Markdown, Cheetah, requests, requests-toolbelt, boto, bioblend, amqp,
       anyjson, kombu, pbr, sqlparse, decorator, Tempita, sqlalchemy-migrate,
-      Parsley, nose, SVGFig, ecdsa, paramiko, Fabric, Whoosh, pysam
+      Parsley, nose, svgwrite, ecdsa, paramiko, Fabric, Whoosh, pysam
     Successfully installed Babel-2.0 Beaker-1.7.0 Cheetah-2.4.4 Fabric-1.10.2
       Mako-1.0.2 Markdown-2.6.3 MarkupSafe-0.23 Parsley-1.3 Paste-2.0.2
-      PasteDeploy-1.5.2 PyYAML-3.11 Routes-2.2 SQLAlchemy-1.0.8 SVGFig-1.1.6
+      PasteDeploy-1.5.2 PyYAML-3.11 Routes-2.2 SQLAlchemy-1.0.8 svgwrite-1.1.6
       Tempita-0.5.3.dev0 WebHelpers-1.3 WebOb-1.4.1 Whoosh-2.4.1+gx1 amqp-1.4.8
       anyjson-0.3.3 bioblend-0.6.1 boto-2.38.0 bx-python-0.7.3 decorator-4.0.2
       docutils-0.12 ecdsa-0.13 kombu-3.0.30 mercurial-3.4.2 nose-1.3.7
@@ -178,10 +178,8 @@ Galaxy to start without attempting to fetch wheels:
     $ sh run.sh --no-create-venv --skip-wheels
 
 Including ``--index-url=https://wheels.galaxyproject.org/simple/`` is important
-- at least one current Galaxy dependency (SVGFig) is not available in PyPI but
-is available (in both source and wheel form) on `wheels.galaxyproject.org`_,
-and two (pysam, Whoosh) include modifications specific to Galaxy which are only
-available on `wheels.galaxyproject.org`_.
+as two dependencies (pysam, Whoosh) include modifications specific to Galaxy
+which are only available on `wheels.galaxyproject.org`_.
 
 .. _unpinned requirements file: https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/dependencies/requirements.txt
 

lib/galaxy/app.py

@@ -160,6 +160,11 @@ class UniverseApplication( object, config.ConfiguresGalaxyMixin ):
                     self.heartbeat.start()
                 if not config.process_is_uwsgi:
                     _start()
+        if self.config.sentry_dsn:
+            import raven
+            self.sentry_client = raven.Client(self.config.sentry_dsn)
+        else:
+            self.sentry_client = None
         # Transfer manager client
         if self.config.get_bool( 'enable_beta_job_managers', False ):
             from galaxy.jobs import transfer_manager

lib/galaxy/dependencies/conda-environment.txt

@@ -64,7 +64,7 @@ sqlparse
 six
 #Parsley
 nose
-#SVGFig
+svgwrite
 
 # Fabric and dependencies
 Fabric

lib/galaxy/dependencies/requirements.txt

@@ -46,7 +46,8 @@ sqlparse
 six
 Parsley
 nose
-SVGFig
+
+svgwrite
 
 # Fabric and dependencies
 Fabric

lib/galaxy/jobs/__init__.py

@@ -1029,6 +1029,7 @@ class JobWrapper( object ):
 
             self.sa_session.add( job )
             self.sa_session.flush()
+        self._report_error_to_sentry()
         # Perform email action even on failure.
         for pja in [pjaa.post_job_action for pjaa in job.post_job_actions if pjaa.post_job_action.action_type == "EmailAction"]:
             ActionBox.execute(self.app, self.sa_session, pja, job)
@@ -1397,6 +1398,8 @@ class JobWrapper( object ):
             self._collect_metrics( job )
         self.sa_session.flush()
         log.debug( 'job %d ended (finish() executed in %s)' % (self.job_id, finish_timer) )
+        if job.state == job.states.ERROR:
+            self._report_error_to_sentry()
         cleanup_job = self.cleanup_job
         delete_files = cleanup_job == 'always' or ( job.state == job.states.OK and cleanup_job == 'onsuccess' )
         self.cleanup( delete_files=delete_files )
@@ -1746,6 +1749,28 @@ class JobWrapper( object ):
             return self.tool.requires_setting_metadata
         return False
 
+    def _report_error_to_sentry( self ):
+        job = self.get_job()
+        tool = self.app.toolbox.get_tool(job.tool_id, tool_version=job.tool_version) or None
+        if self.app.sentry_client and job.state == job.states.ERROR:
+            self.app.sentry_client.capture(
+                'raven.events.Message',
+                message="Galaxy Job Error: %s  v.%s" % (job.tool_id, job.tool_version),
+                extra={
+                    'info' : job.info,
+                    'id' : job.id,
+                    'command_line' : job.command_line,
+                    'stderr' : job.stderr,
+                    'traceback': job.traceback,
+                    'exit_code': job.exit_code,
+                    'stdout': job.stdout,
+                    'handler': job.handler,
+                    'user': self.user,
+                    'tool_version': job.tool_version,
+                    'tool_xml': tool.config_file if tool else None
+                }
+            )
+
 
 class TaskWrapper(JobWrapper):
     """

lib/galaxy/managers/histories.py

@@ -5,13 +5,16 @@ Histories are containers for datasets or dataset collections
 created (or copied) by users over the course of an analysis.
 """
 
-from sqlalchemy import desc, asc
+from sqlalchemy import desc
+from sqlalchemy import asc
 
 from galaxy import model
 from galaxy import exceptions as glx_exceptions
 from galaxy.managers import sharable
 from galaxy.managers import deletable
 from galaxy.managers import hdas
+# from galaxy.managers import hdcas
+from galaxy.managers import history_contents
 from galaxy.managers import collections_util
 
 
@@ -34,6 +37,8 @@ class HistoryManager( sharable.SharableModelManager, deletable.PurgableManagerMi
     def __init__( self, app, *args, **kwargs ):
         super( HistoryManager, self ).__init__( app, *args, **kwargs )
         self.hda_manager = hdas.HDAManager( app )
+        self.contents_manager = history_contents.HistoryContentsManager( app )
+        self.contents_filters = history_contents.HistoryContentsFilters( app )
 
     def copy( self, history, user, **kwargs ):
         """
@@ -147,6 +152,19 @@ class HistoryManager( sharable.SharableModelManager, deletable.PurgableManagerMi
         raise glx_exceptions.RequestParameterInvalidException( 'Unkown order_by', order_by=order_by_string,
             available=[ 'create_time', 'update_time', 'name', 'size' ])
 
+    def non_ready_jobs( self, history ):
+        """Return the currently running job objects associated with this history.
+
+        Where running is defined as new, waiting, queued, running, resubmitted,
+        and upload.
+        """
+        # TODO: defer to jobModelManager (if there was one)
+        # TODO: genericize the params to allow other filters
+        jobs = ( self.session().query( model.Job )
+            .filter( model.Job.history == history )
+            .filter( model.Job.state.in_( model.Job.non_ready_states ) ) )
+        return jobs
+
 
 class HistorySerializer( sharable.SharableModelSerializer, deletable.PurgableSerializerMixin ):
     """
@@ -215,7 +233,9 @@ class HistorySerializer( sharable.SharableModelSerializer, deletable.PurgableSer
             'hdas'          : lambda i, k, **c: [ self.app.security.encode_id( hda.id ) for hda in i.datasets ],
             'state_details' : self.serialize_state_counts,
             'state_ids'     : self.serialize_state_ids,
-            'contents'      : self.serialize_contents
+            'contents'      : self.serialize_contents,
+            'non_ready_jobs': lambda i, k, **c: [ self.app.security.encode_id( job.id ) for job
+                                                  in self.manager.non_ready_jobs( i ) ],
         })
 
     # remove this

lib/galaxy/managers/history_contents.py

@@ -14,7 +14,6 @@ from galaxy.managers import base
 from galaxy.managers import deletable
 from galaxy.managers import containers
 from galaxy.managers import hdas
-from galaxy.managers import collections
 
 import logging
 log = logging.getLogger( __name__ )
@@ -31,7 +30,6 @@ class HistoryContentsManager( containers.ContainerManagerMixin ):
     contained_class_type_name = 'dataset'
 
     subcontainer_class = model.HistoryDatasetCollectionAssociation
-    # TODO:
     subcontainer_class_manager_class = None
     subcontainer_class_type_name = 'dataset_collection'
 
@@ -59,7 +57,6 @@ class HistoryContentsManager( containers.ContainerManagerMixin ):
     def __init__( self, app ):
         self.app = app
         self.contained_manager = self.contained_class_manager_class( app )
-        self.subcontainer_manager = collections.DatasetCollectionManager( app )
 
     # ---- interface
     def contained( self, container, filters=None, limit=None, offset=None, order_by=None, **kwargs ):
@@ -136,15 +133,14 @@ class HistoryContentsManager( containers.ContainerManagerMixin ):
     def _get_filter_for_contained( self, container, content_class ):
         return content_class.history == container
 
-    def _union_of_contents( self, container, **kwargs ):
+    def _union_of_contents( self, container, expand_models=True, **kwargs ):
         """
         Returns a limited and offset list of both types of contents, filtered
         and in some order.
         """
         contents_results = self._union_of_contents_query( container, **kwargs ).all()
-        # import pprint
-        # for result in contents_results:
-        #     pprint.pprint( result )
+        if not expand_models:
+            return contents_results
 
         # partition ids into a map of { component_class names -> list of ids } from the above union query
         id_map = dict( (( self.contained_class_type_name, [] ), ( self.subcontainer_class_type_name, [] )) )

lib/galaxy/model/__init__.py

@@ -368,6 +368,15 @@ class Job( object, JobLike, Dictifiable ):
     terminal_states = [ states.OK,
                         states.ERROR,
                         states.DELETED ]
+    #: job states where the job hasn't finished and the model may still change
+    non_ready_states = [
+        states.NEW,
+        states.RESUBMITTED,
+        states.UPLOAD,
+        states.WAITING,
+        states.QUEUED,
+        states.RUNNING,
+    ]
 
     # Please include an accessor (get/set pair) for any new columns/members.
     def __init__( self ):
@@ -1622,6 +1631,7 @@ class Dataset( StorableObject ):
     # failed_metadata is only valid as DatasetInstance state currently
 
     non_ready_states = (
+        states.NEW,
         states.UPLOAD,
         states.QUEUED,
         states.RUNNING,

lib/galaxy/model/custom_types.py

@@ -272,7 +272,7 @@ class MetadataType( JSONType ):
                     sz = total_size(v)
                     if sz > app.app.config.max_metadata_value_size:
                         del value[k]
-                        log.error('Refusing to bind metadata key %s due to size (%s)' % (k, sz))
+                        log.warning('Refusing to bind metadata key %s due to size (%s)' % (k, sz))
             value = json_encoder.encode(value)
         return value
 

lib/galaxy/tools/__init__.py

@@ -1603,68 +1603,6 @@ class Tool( object, Dictifiable ):
         # create parameter object
         params = galaxy.util.Params( kwd, sanitize=False )
 
-        # convert value to jsonifiable value
-        def jsonify(v):
-            # check if value is numeric
-            isnumber = False
-            try:
-                float(v)
-                isnumber = True
-            except Exception:
-                pass
-
-            # fix hda parsing
-            if isinstance(v, self.app.model.HistoryDatasetAssociation):
-                return {
-                    'id'  : trans.security.encode_id(v.id),
-                    'src' : 'hda'
-                }
-            elif isinstance(v, self.app.model.HistoryDatasetCollectionAssociation):
-                return {
-                    'id'  : trans.security.encode_id(v.id),
-                    'src' : 'hdca'
-                }
-            elif isinstance(v, self.app.model.LibraryDatasetDatasetAssociation):
-                return {
-                    'id'  : trans.security.encode_id(v.id),
-                    'name': v.name,
-                    'src' : 'ldda'
-                }
-            elif isinstance(v, bool):
-                if v is True:
-                    return 'true'
-                else:
-                    return 'false'
-            elif isinstance(v, string_types) or isnumber:
-                return v
-            elif isinstance(v, dict) and hasattr(v, '__class__'):
-                return v
-            else:
-                return None
-
-        # ensures that input dictionary is jsonifiable
-        def sanitize( dict, key='value' ):
-            # get current value
-            value = dict[key] if key in dict else None
-
-            # jsonify by type
-            if dict['type'] in ['data']:
-                if isinstance(value, list):
-                    value = [ jsonify(v) for v in value ]
-                else:
-                    value = [ jsonify(value) ]
-                if None in value:
-                    value = None
-                else:
-                    value = { 'values': value }
-            elif isinstance(value, list):
-                value = [ jsonify(v) for v in value ]
-            else:
-                value = jsonify(value)
-
-            # update and return
-            dict[key] = value
-
         # populates model from state
         def populate_model( inputs, state_inputs, group_inputs, other_values=None ):
             other_values = ExpressionContext( state_inputs, other_values )
@@ -1681,7 +1619,7 @@ class Tool( object, Dictifiable ):
                     tool_dict = input.to_dict( request_context )
                     if 'test_param' in tool_dict:
                         test_param = tool_dict[ 'test_param' ]
-                        test_param[ 'value' ] = jsonify( group_state.get( test_param[ 'name' ], input.test_param.get_initial_value( request_context, other_values ) ) )
+                        test_param[ 'value' ] = input.test_param.value_to_basic( group_state.get( test_param[ 'name' ], input.test_param.get_initial_value( request_context, other_values ) ), self.app )
                         test_param[ 'text_value' ] = input.test_param.value_to_display_text( test_param[ 'value' ], self.app )
                         for i in range( len( tool_dict['cases'] ) ):
                             current_state = {}
@@ -1694,29 +1632,14 @@ class Tool( object, Dictifiable ):
                 else:
                     try:
                         tool_dict = input.to_dict( request_context, other_values=other_values )
-                        tool_dict[ 'value' ] = state_inputs.get( input.name, input.get_initial_value( request_context, other_values ) )
+                        tool_dict[ 'value' ] = input.value_to_basic( state_inputs.get( input.name, input.get_initial_value( request_context, other_values ) ), self.app )
                         tool_dict[ 'text_value' ] = input.value_to_display_text( tool_dict[ 'value' ], self.app )
                     except Exception as e:
                         tool_dict = input.to_dict( request_context )
                         log.exception('tools::to_json() - Skipping parameter expansion \'%s\': %s.' % ( input.name, e ) )
                         pass
-                    sanitize( tool_dict, 'value' )
                 group_inputs[ input_index ] = tool_dict
 
-        # sanatizes tool state
-        def sanitize_state( state ):
-            keys = None
-            if isinstance( state, dict ):
-                keys = state
-            elif isinstance( state, list ):
-                keys = range( len( state ) )
-            if keys:
-                for k in keys:
-                    if isinstance( state[ k ], dict ) or isinstance( state[ k ], list ):
-                        sanitize_state( state[ k ] )
-                    else:
-                        state[ k ] = jsonify( state[ k ] )
-
         # expand incoming parameters (parameters might trigger multiple tool executions,
         # here we select the first execution only in order to resolve dynamic parameters)
         expanded_incomings, _ = expand_meta_parameters( trans, self, params.__dict__ )
@@ -1737,7 +1660,10 @@ class Tool( object, Dictifiable ):
         populate_model( self.inputs, state_inputs, tool_model[ 'inputs' ] )
 
         # sanitize tool state
-        sanitize_state( state_inputs )
+        def value_to_basic( input, value, parent, **kwargs ):
+            parent[ input.name ] = input.value_to_basic( value, self.app )
+
+        visit_input_values( self.inputs, state_inputs, value_to_basic )
 
         # create tool help
         tool_help = ''

lib/galaxy/tools/actions/__init__.py

@@ -568,7 +568,7 @@ class DefaultToolAction( object ):
                     first_reduction = False
                     incoming[ name ] = []
                 if reduced:
-                    incoming[ name ].append( "__collection_reduce__|%s" % dataset_collection.id )
+                    incoming[ name ].append( dataset_collection )
                 # Should verify security? We check security of individual
                 # datasets below?
                 # TODO: verify can have multiple with same name, don't want to loose tracability

lib/galaxy/tools/evaluation.py

@@ -58,7 +58,7 @@ class ToolEvaluator( object ):
         request_context = WorkRequestContext( app=self.app, user=job.history and job.history.user, history=job.history )
 
         def validate_inputs( input, value, context, **kwargs ):
-            value = input.from_html( value, request_context, context )
+            value = input.from_json( value, request_context, context )
             input.validate( value, request_context )
         visit_input_values( self.tool.inputs, incoming, validate_inputs )
 

lib/galaxy/tools/parameters/__init__.py

@@ -120,7 +120,7 @@ def check_param( trans, param, incoming_value, param_values ):
             if isinstance( value, dict ):
                 if value.get( '__class__' ) == 'RuntimeValue':
                     return [ value, None ]
-        value = param.from_html( value, trans, param_values )
+        value = param.from_json( value, trans, param_values )
         param.validate( value, trans )
     except ValueError, e:
         error = str( e )

lib/galaxy/tools/parameters/basic.py

@@ -98,7 +98,7 @@ class ToolParameter( object, Dictifiable ):
         """
         return self.get_html_field( trans, value, other_values ).get_html()
 
-    def from_html( self, value, trans=None, other_values={} ):
+    def from_json( self, value, trans=None, other_values={} ):
         """
         Convert a value from an HTML POST into the parameters preferred value
         format.
@@ -135,12 +135,12 @@ class ToolParameter( object, Dictifiable ):
         """
         return []
 
-    def to_string( self, value, app ):
+    def to_json( self, value, app ):
         """Convert a value to a string representation suitable for persisting"""
         return unicodify( value )
 
     def to_python( self, value, app ):
-        """Convert a value created with to_string back to an object representation"""
+        """Convert a value created with to_json back to an object representation"""
         return value
 
     def value_to_basic( self, value, app ):
@@ -149,12 +149,14 @@ class ToolParameter( object, Dictifiable ):
         elif isinstance( value, dict ):
             if value.get('__class__') == 'RuntimeValue':
                 return value
-        return self.to_string( value, app )
+        return self.to_json( value, app )
 
     def value_from_basic( self, value, app, ignore_errors=False ):
         # Handle Runtime values (valid for any parameter?)
-        if isinstance( value, dict ) and '__class__' in value and value['__class__'] == "RuntimeValue":
+        if isinstance( value, dict ) and value.get( '__class__' ) == 'RuntimeValue':
             return RuntimeValue()
+        elif isinstance( value, dict ) and value.get( '__class__' ) == 'UnvalidatedValue':
+            return value[ 'value' ]
         # Delegate to the 'to_python' method
         if ignore_errors:
             try:
@@ -251,7 +253,7 @@ class TextToolParameter( ToolParameter ):
         else:
             return form_builder.TextField( self.name, self.size, value )
 
-    def to_string( self, value, app ):
+    def to_json( self, value, app ):
         """Convert a value to a string representation suitable for persisting"""
         if value is None:
             rval = ''
@@ -284,9 +286,9 @@ class IntegerToolParameter( TextToolParameter ):
     blah
     >>> print p.get_html()
     <input type="text" name="blah" size="4" value="10">
-    >>> type( p.from_html( "10", trans ) )
+    >>> type( p.from_json( "10", trans ) )
     <type 'int'>
-    >>> type( p.from_html( "bleh", trans ) )
+    >>> type( p.from_json( "bleh", trans ) )
     Traceback (most recent call last):
         ...
     ValueError: An integer or workflow parameter e.g. ${name} is required
@@ -324,7 +326,7 @@ class IntegerToolParameter( TextToolParameter ):
             value = str( value )
         return super( IntegerToolParameter, self ).get_html_field( trans=trans, value=value, other_values=other_values )
 
-    def from_html( self, value, trans, other_values={} ):
+    def from_json( self, value, trans, other_values={} ):
         try:
             return int( value )
         except:
@@ -364,9 +366,9 @@ class FloatToolParameter( TextToolParameter ):
     blah
     >>> print p.get_html()
     <input type="text" name="blah" size="4" value="3.141592">
-    >>> type( p.from_html( "36.1", trans ) )
+    >>> type( p.from_json( "36.1", trans ) )
     <type 'float'>
-    >>> type( p.from_html( "bleh", trans ) )
+    >>> type( p.from_json( "bleh", trans ) )
     Traceback (most recent call last):
         ...
     ValueError: A real number or workflow parameter e.g. ${name} is required
@@ -404,7 +406,7 @@ class FloatToolParameter( TextToolParameter ):
             value = str( value )
         return super( FloatToolParameter, self ).get_html_field( trans=trans, value=value, other_values=other_values )
 
-    def from_html( self, value, trans, other_values={} ):
+    def from_json( self, value, trans, other_values={} ):
         try:
             return float( value )
         except:
@@ -443,11 +445,11 @@ class BooleanToolParameter( ToolParameter ):
     blah
     >>> print p.get_html()
     <input type="checkbox" id="blah" name="blah" value="__CHECKED__" checked="checked"><input type="hidden" name="blah" value="__NOTHING__">
-    >>> print p.from_html( ["__CHECKED__","__NOTHING__"] )
+    >>> print p.from_json( ["__CHECKED__","__NOTHING__"] )
     True
     >>> print p.to_param_dict_string( True )
     bulletproof vests
-    >>> print p.from_html( ["__NOTHING__"] )
+    >>> print p.from_json( ["__NOTHING__"] )
     False
     >>> print p.to_param_dict_string( False )
     cellophane chests
@@ -462,10 +464,10 @@ class BooleanToolParameter( ToolParameter ):
     def get_html_field( self, trans=None, value=None, other_values={} ):
         checked = self.checked
         if value is not None:
-            checked = self.from_html( value )
+            checked = self.from_json( value )
         return form_builder.CheckboxField( self.name, checked, refresh_on_change=self.refresh_on_change )
 
-    def from_html( self, value, trans=None, other_values={} ):
+    def from_json( self, value, trans=None, other_values={} ):
         if form_builder.CheckboxField.is_checked( value ):
             return True
         return self.to_python( value )
@@ -473,6 +475,12 @@ class BooleanToolParameter( ToolParameter ):
     def to_python( self, value, app=None ):
         return ( value in [ True, 'True', 'true' ] )
 
+    def to_json( self, value, app=None ):
+        if value is True:
+            return 'true'
+        else:
+            return 'false'
+
     def get_initial_value( self, trans, other_values ):
         return self.checked
 
@@ -518,7 +526,7 @@ class FileToolParameter( ToolParameter ):
     def get_html_field( self, trans=None, value=None, other_values={}  ):
         return form_builder.FileField( self.name, ajax=self.ajax, value=value )
 
-    def from_html( self, value, trans=None, other_values={} ):
+    def from_json( self, value, trans=None, other_values={} ):
         # Middleware or proxies may encode files in special ways (TODO: this
         # should be pluggable)
         if type( value ) == dict:
@@ -542,7 +550,7 @@ class FileToolParameter( ToolParameter ):
         """
         return "multipart/form-data"
 
-    def to_string( self, value, app ):
+    def to_json( self, value, app ):
         if value in [ None, '' ]:
             return None
         elif isinstance( value, string_types ):
@@ -609,10 +617,10 @@ class FTPFileToolParameter( ToolParameter ):
         else:
             return lst[ 0 ]
 
-    def from_html( self, value, trans=None, other_values={} ):
+    def from_json( self, value, trans=None, other_values={} ):
         return self.to_python( value, trans.app, validate=True )
 
-    def to_string( self, value, app ):
+    def to_json( self, value, app ):
         return self.to_python( value, app )
 
     def to_python( self, value, app, validate=False ):
@@ -645,9 +653,6 @@ class HiddenToolParameter( ToolParameter ):
     """
     Parameter that takes one of two values.
 
-    FIXME: This seems hacky, parameters should only describe things the user
-           might change. It is used for 'initializing' the UCSC proxy tool
-
     >>> p = HiddenToolParameter( None, XML( '<param name="blah" type="hidden" value="wax so rockin"/>' ) )
     >>> print p.name
     blah
@@ -706,7 +711,7 @@ class BaseURLToolParameter( HiddenToolParameter ):
     def get_html_field( self, trans=None, value=None, other_values={} ):
         return form_builder.HiddenField( self.name, self._get_value() )
 
-    def from_html( self, value=None, trans=None, other_values={} ):
+    def from_json( self, value=None, trans=None, other_values={} ):
         return self._get_value()
 
     def _get_value( self ):
@@ -886,7 +891,7 @@ class SelectToolParameter( ToolParameter ):
             field.add_option( text, optval, selected )
         return field
 
-    def from_html( self, value, trans, other_values={} ):
+    def from_json( self, value, trans, other_values={} ):
         legal_values = self.get_legal_values( trans, other_values )
         if len(list(legal_values)) == 0 and trans.workflow_building_mode:
             if self.multiple:
@@ -948,20 +953,9 @@ class SelectToolParameter( ToolParameter ):
             value = value_map( value )
         return value
 
-    def value_to_basic( self, value, app ):
-        if isinstance( value, RuntimeValue ):
-            # Need to handle runtime value's ourself since delegating to the
-            # parent method causes the value to be turned into a string, which
-            # breaks multiple selection
-            return { "__class__": "RuntimeValue" }
+    def to_json( self, value, app ):
         return value
 
-    def value_from_basic( self, value, app, ignore_errors=False ):
-        # Backward compatibility for unvalidated values already stored in databases
-        if isinstance( value, dict ) and value.get( "__class__", None ) == "UnvalidatedValue":
-            return value[ "value" ]
-        return super( SelectToolParameter, self ).value_from_basic( value, app, ignore_errors=ignore_errors )
-
     def get_initial_value( self, trans, other_values ):
         options = list( self.get_options( trans, other_values ) )
         if len(options) == 0 and trans.workflow_building_mode:
@@ -1155,7 +1149,7 @@ class ColumnListParameter( SelectToolParameter ):
         self.is_dynamic = True
         self.usecolnames = input_source.get_bool( "use_header_names", False )
 
-    def from_html( self, value, trans, other_values={} ):
+    def from_json( self, value, trans, other_values={} ):
         """
         Label convention prepends column number with a 'c', but tool uses the integer. This
         removes the 'c' when entered into a workflow.
@@ -1182,7 +1176,7 @@ class ColumnListParameter( SelectToolParameter ):
         if not value and not self.get_legal_values( trans, other_values ) and self.accept_default:
             value = self.default_value or '1'
             return [ value ] if self.multiple else value
-        return super( ColumnListParameter, self ).from_html( value, trans, other_values )
+        return super( ColumnListParameter, self ).from_json( value, trans, other_values )
 
     @staticmethod
     def _strip_c(column):
@@ -1479,7 +1473,7 @@ class DrillDownSelectToolParameter( SelectToolParameter ):
                 return form_builder.TextField( self.name, value=(value or "") )
         return form_builder.DrillDownField( self.name, self.multiple, self.display, self.refresh_on_change, options, value, refresh_on_change_values=self.refresh_on_change_values )
 
-    def from_html( self, value, trans, other_values={} ):
+    def from_json( self, value, trans, other_values={} ):
         legal_values = self.get_legal_values( trans, other_values )
         if len( list( legal_values ) ) == 0 and trans.workflow_building_mode:
             if self.multiple:
@@ -1675,6 +1669,60 @@ class BaseDataToolParameter( ToolParameter ):
             self.options_filter_attribute = options_elem.get(  'options_filter_attribute', None )
         self.is_dynamic = self.options is not None
 
+    def to_json( self, value, app ):
+        def single_to_json( value ):
+            src = None
+            if isinstance( value, galaxy.model.DatasetCollectionElement ):
+                src = 'dce'
+            elif isinstance( value, app.model.HistoryDatasetCollectionAssociation ):
+                src = 'hdca'
+            else:
+                src = 'hda'
+            return { 'id' : app.security.encode_id( value.id ), 'src' : src }
+
+        if value not in [ None, '', 'None' ]:
+            if isinstance( value, list ) and len( value ) > 0:
+                values = [ single_to_json( v ) for v in value ]
+            else:
+                values = [ single_to_json( value ) ]
+            return { 'values': values }
+        return None
+
+    def to_python( self, value, app ):
+        def single_to_python( value ):
+            if isinstance( value, dict ) and 'src' in value:
+                id = app.security.decode_id( value[ 'id' ] )
+                if value[ 'src' ] == 'dce':
+                    return app.model.context.query( app.model.DatasetCollectionElement ).get( id )
+                elif value[ 'src' ] == 'hdca':
+                    return app.model.context.query( app.model.HistoryDatasetCollectionAssociation ).get( id )
+                else:
+                    return app.model.context.query( app.model.HistoryDatasetAssociation ).get( id )
+
+        if isinstance( value, dict ) and 'values' in value:
+            if hasattr( self, 'multiple' ) and self.multiple is True:
+                return [ single_to_python( v ) for v in value[ 'values' ] ]
+            elif len( value[ 'values' ] ) > 0:
+                return single_to_python( value[ 'values' ][ 0 ] )
+
+        # Handle legacy string values potentially stored in databases
+        none_values = [ None, '', 'None' ]
+        if value in none_values:
+            return None
+        if isinstance( value, string_types ) and value.find( ',' ) > -1:
+            return [ app.model.context.query( app.model.HistoryDatasetAssociation ).get( int( v ) ) for v in value.split( ',' ) if v not in none_values ]
+        elif str( value ).startswith( "__collection_reduce__|" ):
+            decoded_id = str( value )[ len( "__collection_reduce__|" ): ]
+            if not decoded_id.isdigit():
+                decoded_id = app.security.decode_id( decoded_id )
+            return app.model.context.query( app.model.HistoryDatasetCollectionAssociation ).get( app.security.decode_id( decoded_id ) )
+        elif str( value ).startswith( "dce:" ):
+            return app.model.context.query( app.model.DatasetCollectionElement ).get( int( value[ len( "dce:" ): ] ) )
+        elif str( value ).startswith( "hdca:" ):
+            return app.model.context.query( app.model.HistoryDatasetCollectionAssociation ).get( int( value[ len( "hdca:" ): ] ) )
+        else:
+            return app.model.context.query( app.model.HistoryDatasetAssociation ).get( int( value ) )
+
 
 class DataToolParameter( BaseDataToolParameter ):
     # TODO, Nate: Make sure the following unit tests appropriately test the dataset security
@@ -1818,7 +1866,7 @@ class DataToolParameter( BaseDataToolParameter ):
             return most_recent_dataset[0]
         return ''
 
-    def from_html( self, value, trans, other_values={} ):
+    def from_json( self, value, trans, other_values={} ):
         if trans.workflow_building_mode:
             return None
         if not value and not self.optional:
@@ -1887,47 +1935,6 @@ class DataToolParameter( BaseDataToolParameter ):
             rval = values[ 0 ]
         return rval
 
-    def to_string( self, value, app ):
-        if value is None or isinstance( value, string_types ):
-            return value
-        elif isinstance( value, int ):
-            return str( value )
-        elif isinstance( value, RuntimeValue ):
-            return None
-        elif isinstance( value, list ) and len( value ) > 0 and isinstance( value[ 0 ], RuntimeValue ):
-            return None
-        elif isinstance( value, list ):
-            return ",".join( [ str( self.to_string( val, app ) ) for val in value ] )
-        elif isinstance( value, app.model.HistoryDatasetCollectionAssociation ):
-            return "__collection_reduce__|%d" % value.id
-        try:
-            return value.id
-        except:
-            return str( value )
-
-    def to_python( self, value, app ):
-        # Both of these values indicate that no dataset is selected.  However, 'None'
-        # indicates that the dataset is optional, while '' indicates that it is not.
-        none_values = [ None, '', 'None' ]
-
-        def single_to_python(value):
-            if value in none_values:
-                return value
-            elif str( value ).startswith( "__collection_reduce__|" ):
-                decoded_id = str( value )[ len( "__collection_reduce__|" ): ]
-                if not decoded_id.isdigit():
-                    log.info("to_python called encoded data, bad data previously persisted to Galaxy databse - workflow extraction and rerun of this dataset may break if id_secret changed.")
-                    decoded_id = app.security.decode_id(decoded_id)
-                return app.model.context.query( app.model.HistoryDatasetCollectionAssociation ).get( decoded_id )
-            else:
-                return app.model.context.query( app.model.HistoryDatasetAssociation ).get( int( value ) )
-
-        if isinstance(value, string_types) and value.find(",") > -1:
-            values = value.split(",")
-            return [v for v in map( single_to_python, values ) if v not in none_values]
-        else:
-            return single_to_python( value )
-
     def to_param_dict_string( self, value, other_values={} ):
         if value is None:
             return "None"
@@ -2135,7 +2142,7 @@ class DataCollectionToolParameter( BaseDataToolParameter ):
         self._ensure_selection( field )
         return field
 
-    def from_html( self, value, trans, other_values={} ):
+    def from_json( self, value, trans, other_values={} ):
         if trans.workflow_building_mode:
             return None
         if not value and not self.optional:
@@ -2173,38 +2180,6 @@ class DataCollectionToolParameter( BaseDataToolParameter ):
             # TODO: Handle error states, implement error states ...
         return rval
 
-    def to_string( self, value, app ):
-        if value is None or isinstance( value, string_types ):
-            return value
-        elif isinstance( value, RuntimeValue ):
-            return None
-        try:
-            if isinstance( value, galaxy.model.DatasetCollectionElement ):
-                return "dce:%s" % value.id
-            else:
-                return "hdca:%s" % value.id
-        except Exception:
-            # This is not good...
-            return str( value )
-
-    def to_python( self, value, app ):
-        # Both of these values indicate that no dataset is selected.  However, 'None'
-        # indicates that the dataset is optional, while '' indicates that it is not.
-        if value is None or value == '' or value == 'None':
-            return value
-
-        if not isinstance( value, string_types ):
-            raise ValueError( "Can not convert data collection parameter value to python object - %s" % value )
-
-        if value.startswith( "dce:" ):
-            dce = app.model.context.query( app.model.DatasetCollectionElement ).get( int( value[ len( "dce:" ): ] ) )
-            return dce
-        elif value.startswith( "hdca:" ):
-            hdca = app.model.context.query( app.model.HistoryDatasetCollectionAssociation ).get( int( value[ len( "hdca:" ): ] ) )
-            return hdca
-        else:
-            raise ValueError( "Can not convert data collection parameter value to python object - %s" % value )
-
     def value_to_display_text( self, value, app ):
         try:
             if isinstance( value, galaxy.model.HistoryDatasetCollectionAssociation ):
@@ -2294,7 +2269,7 @@ class LibraryDatasetToolParameter( ToolParameter ):
     def get_initial_value( self, trans, other_values ):
         return None
 
-    def from_html( self, value, trans, other_values={} ):
+    def from_json( self, value, trans, other_values={} ):
         return self.to_python( value, trans.app, other_values=other_values, validate=True )
 
     def to_param_dict_string( self, value, other_values={} ):
@@ -2307,7 +2282,7 @@ class LibraryDatasetToolParameter( ToolParameter ):
 
     # converts values to json representation:
     #   { id: LibraryDatasetDatasetAssociation.id, name: LibraryDatasetDatasetAssociation.name, src: 'lda' }
-    def to_string( self, value, app ):
+    def to_json( self, value, app ):
         if not isinstance( value, list ):
             value = [value]
         lst = []

lib/galaxy/tools/toolbox/base.py

@@ -558,20 +558,22 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
                     # Backward compatibility issue - the tag used to be named 'changeset_revision'.
                     installed_changeset_revision_elem = item.elem.find( "changeset_revision" )
                 installed_changeset_revision = installed_changeset_revision_elem.text
-                try:
-                    splitted_path = path.split('/')
-                    assert splitted_path[0] == tool_shed
-                    assert splitted_path[2] == repository_owner
-                    assert splitted_path[3] == repository_name
-                    if splitted_path[4] != installed_changeset_revision:
-                        # This can happen if the Tool Shed repository has been
-                        # updated to a new revision and the installed_changeset_revision
-                        # element in shed_tool_conf.xml file has been updated too
-                        log.debug("The installed_changeset_revision for tool %s is %s, using %s instead", path, installed_changeset_revision, splitted_path[4])
-                        installed_changeset_revision = splitted_path[4]
-                except Exception as e:
-                    log.debug("Error while loading tool %s : %s", path, e)
-                    pass
+                if "/repos/" in path:  # The only time "/repos/" should not be in path is during testing!
+                    try:
+                        tool_shed_path, reduced_path = path.split('/repos/', 1)
+                        splitted_path = reduced_path.split('/')
+                        assert tool_shed_path == tool_shed
+                        assert splitted_path[0] == repository_owner
+                        assert splitted_path[1] == repository_name
+                        if splitted_path[2] != installed_changeset_revision:
+                            # This can happen if the Tool Shed repository has been
+                            # updated to a new revision and the installed_changeset_revision
+                            # element in shed_tool_conf.xml file has been updated too
+                            log.debug("The installed_changeset_revision for tool %s is %s, using %s instead", path, installed_changeset_revision, splitted_path[2])
+                            installed_changeset_revision = splitted_path[2]
+                    except AssertionError:
+                        log.debug("Error while loading tool %s", path)
+                        pass
                 tool_shed_repository = self._get_tool_shed_repository( tool_shed,
                                                                        repository_name,
                                                                        repository_owner,

lib/galaxy/visualization/data_providers/genome.py

@@ -195,7 +195,6 @@ class GenomeDataProvider( BaseDataProvider ):
         except AttributeError:
             # FIXME: some data providers do not have a close function implemented.
             # Providers without a close function include:
-            #  pysam Tabixfile
             #  bx IntervalIndex
             pass
 
@@ -346,13 +345,16 @@ class TabixDataProvider( FilterableMixin, GenomeDataProvider ):
 
     def open_data_file( self ):
         return ctabix.Tabixfile(self.dependencies['bgzip'].file_name,
-                                index_filename=self.converted_dataset.file_name)
+                                index=self.converted_dataset.file_name)
 
     def get_iterator( self, data_file, chrom, start, end, **kwargs ):
-        start, end = int(start), int(end)
+        # chrom must be a string, start/end integers.
+        # in previous versions of pysam, unicode was accepted for chrom, but not in 8.4
+        chrom = str(chrom)
+        start = int(start)
+        end = int(end)
         if end >= (2 << 29):
             end = (2 << 29 - 1)  # Tabix-enforced maximum
-
         # Get iterator using either naming scheme.
         iterator = iter( [] )
         if chrom in data_file.contigs:
@@ -887,6 +889,9 @@ class BamDataProvider( GenomeDataProvider, FilterableMixin ):
         """
 
         # Fetch and return data.
+        chrom = str(chrom)
+        start = int(start)
+        end = int(end)
         try:
             data = data_file.fetch( start=start, end=end, reference=chrom )
         except ValueError:

lib/galaxy/webapps/galaxy/api/quotas.py

@@ -50,7 +50,7 @@ class QuotaAPIController( BaseAPIController, Admin, AdminActions, UsesQuotaMixin
         Displays information about a quota.
         """
         quota = self.get_quota( trans, id, deleted=util.string_as_bool( deleted ) )
-        return quota.to_dict( view='element', value_mapper={ 'id': trans.security.encode_id } )
+        return quota.to_dict( view='element', value_mapper={ 'id': trans.security.encode_id, 'total_disk_usage': float } )
 
     @web.expose_api
     @web.require_admin

lib/galaxy/webapps/galaxy/api/remote_files.py

@@ -96,10 +96,11 @@ class RemoteFilesAPIController( BaseAPIController ):
                 if user_ftp_dir is not None:
                     response = self.__load_all_filenames( user_ftp_dir )
                 else:
-                    raise exceptions.ConfigDoesNotAllowException( 'You do not have an FTP directory named as your login at this Galaxy instance.' )
+                    log.warning( 'You do not have an FTP directory named as your login at this Galaxy instance.' )
+                    return None
             except Exception, exception:
-                log.error( 'Could not get ftp files: %s', str( exception ), exc_info=True )
-                raise exceptions.InternalServerError( 'Could not get the files from your FTP folder.' )
+                log.warning( 'Could not get ftp files: %s', str( exception ), exc_info=True )
+                return None
         return response
 
     def __load_all_filenames( self, directory ):
@@ -117,7 +118,8 @@ class RemoteFilesAPIController( BaseAPIController ):
                                            size=statinfo.st_size,
                                            ctime=time.strftime( "%m/%d/%Y %I:%M:%S %p", time.localtime( statinfo.st_ctime ) ) ) )
         else:
-            raise exceptions.ConfigDoesNotAllowException( 'The given directory does not exist.' )
+            log.warning( "The directory \"%s\" does not exist." % directory )
+            return response
         # sort by path
         response = sorted(response, key=itemgetter("path"))
         return response

lib/galaxy/workflow/modules.py

@@ -1139,21 +1139,13 @@ class ToolModule( WorkflowModule ):
                 dict( ( conn.input_name, conn ) for conn in connections )
         else:
             input_connections_by_name = {}
+
         # Any connected input needs to have value RuntimeValue (these
         # are not persisted so we need to do it every time)
-
         def callback( input, prefixed_name, **kwargs ):
-            replacement = None
-            if isinstance( input, DataToolParameter ):
+            if isinstance( input, DataToolParameter ) or isinstance( input, DataCollectionToolParameter ):
                 if connections is None or prefixed_name in input_connections_by_name:
-                    if input.multiple:
-                        replacement = [] if not connections else [RuntimeValue() for conn in connections]
-                    else:
-                        replacement = RuntimeValue()
-            elif isinstance( input, DataCollectionToolParameter ):
-                if connections is None or prefixed_name in input_connections_by_name:
-                    replacement = RuntimeValue()
-            return replacement
+                    return RuntimeValue()
 
         visit_input_values( self.tool.inputs, self.state.inputs, callback )
 

lib/tool_shed/tool_shed_registry.py

@@ -49,7 +49,7 @@ class Registry( object ):
         url_sans_protocol = common_util.remove_protocol_from_tool_shed_url( url )
         for shed_name, shed_url in self.tool_sheds.items():
             shed_url_sans_protocol = common_util.remove_protocol_from_tool_shed_url( shed_url )
-            if shed_url_sans_protocol.find( url_sans_protocol ) >= 0:
+            if url_sans_protocol.startswith( shed_url_sans_protocol ):
                 return self.tool_sheds_auth[ shed_name ]
         log.debug( "Invalid url '%s' received by tool shed registry's password_manager_for_url method." % str( url ) )
         return None

run.sh

@@ -33,8 +33,14 @@ do
             stop_daemon_arg_set=1
             shift
             ;;
-        --daemon|restart)
-            paster_args="$paster_args $1"
+        --daemon|--restart|restart)
+            if [ $1 == "--restart" ]
+            then
+                paster_args="$paster_args restart"
+            else
+                paster_args="$paster_args $1"
+            fi
+
             daemon_or_restart_arg_set=1
             shift
             ;;

static/maps/mvc/tool/tool-form.js.map

@@ -1 +1 @@
-{"version":3,"file":"tool-form.js","sources":["../../../src/mvc/tool/tool-form.js"],"names":["define","Utils","Ui","ToolFormBase","ToolTemplate","View","extend","initialize","options","self","this","prototype","call","merge","customize","buttons","execute","execute_btn","Button","icon","tooltip","name","version","title","cls","floating","onclick","wait","portlet","disable","submit","unwait","enable","job_id","job_remap","inputs","label","type","display","ignore","value","help","callback","job_def","tool_id","id","tool_version","data","create","trigger","validate","Galaxy","emit","debug","request","url","root","success","response","$el","empty","append","parent","currHistoryPanel","refreshContents","error","err_data","error_messages","matchResponse","input_id","highlight","modal","show","body","err_msg","Close","hide","job_inputs","batch_n","batch_src","job_input_id","input_value","match","input_field","field_list","input_def","input_list","optional","batch","n","values","length","src"],"mappings":"AAGAA,QAAQ,cAAe,iBAAkB,0BAA2B,0BAChE,SAAUC,EAAOC,EAAIC,EAAcC,GACnC,GAAIC,GAAOF,EAAaG,QACpBC,WAAY,SAAUC,GAClB,GAAIC,GAAOC,IACXP,GAAaQ,UAAUJ,WAAWK,KAAMF,KAAMT,EAAMY,OAChDC,UAAkB,SAAUN,GAExBA,EAAQO,SACJC,QAAUC,YAAc,GAAIf,GAAGgB,QAC3BC,KAAW,WACXC,QAAW,YAAcZ,EAAQa,KAAO,KAAOb,EAAQc,QAAU,IACjEC,MAAW,UACXC,IAAW,kBACXC,SAAW,QACXC,QAAW,WACPT,YAAYU,OACZlB,EAAKmB,QAAQC,UACbpB,EAAKqB,OAAQtB,EAAS,WAClBS,YAAYc,SACZtB,EAAKmB,QAAQI,eAMxBxB,EAAQyB,QAAUzB,EAAQ0B,YAC3B1B,EAAQ2B,OAA6B,oBACjCC,MAAc,oCACdf,KAAc,qBACdgB,KAAc,SACdC,QAAc,QACdC,OAAc,aACdC,MAAc,aACdhC,UAAkB,MAAOA,EAAQyB,SAAY,KAAM,eACnDQ,KAAc,uLAI3BjC,KAOPsB,OAAQ,SAAUtB,EAASkC,GACvB,GAAIjC,GAAOC,KACPiC,GACAC,QAAkBpC,EAAQqC,GAC1BC,aAAkBtC,EAAQc,QAC1Ba,OAAkBzB,KAAKqC,KAAKC,SAGhC,OADAtC,MAAKuC,QAAS,SACRxC,EAAKyC,SAAUP,IAKrBQ,OAAOC,KAAKC,MAAO,sBAAuB,uBAAwBV,OAClE1C,GAAMqD,SACFjB,KAAU,OACVkB,IAAUJ,OAAOK,KAAO,YACxBT,KAAUJ,EACVc,QAAU,SAAUC,GAChBhB,GAAYA,IACZjC,EAAKkD,IAAIC,QAAQC,OAAQzD,EAAaqD,QAASC,IAC/CI,OAAOX,QAAUW,OAAOX,OAAOY,kBAAoBD,OAAOX,OAAOY,iBAAiBC,mBAEtFC,MAAU,SAAUP,GAGhB,GAFAhB,GAAYA,IACZS,OAAOC,KAAKC,MAAO,oBAAqB,qBAAsBK,GACzDA,GAAYA,EAASQ,SAAW,CACjC,GAAIC,GAAiB1D,EAAKsC,KAAKqB,cAAeV,EAASQ,SACvD,KAAK,GAAIG,KAAYF,GAAgB,CACjC1D,EAAK6D,UAAWD,EAAUF,EAAgBE,GAC1C,YAGJ5D,GAAK8D,MAAMC,MACPjD,MAAU,wBACVkD,KAAYf,GAAYA,EAASgB,SAAatE,EAAa6D,MAAOtB,GAClE5B,SACI4D,MAAU,WACNlE,EAAK8D,MAAMK,gBA7B/BzB,OAAOC,KAAKC,MAAO,sBAAuB,gDAC1CX,GAAYA,OAwCpBQ,SAAU,SAAUP,GAChB,GAAIkC,GAAclC,EAAQR,OACtB2C,EAAc,GACdC,EAAc,IAClB,KAAM,GAAIC,KAAgBH,GAAa,CACnC,GAAII,GAAcJ,EAAYG,GAC1BX,EAAc3D,KAAKqC,KAAKmC,MAAOF,GAC/BG,EAAczE,KAAK0E,WAAYf,GAC/BgB,EAAc3E,KAAK4E,WAAYjB,EACnC,IAAMA,GAAagB,GAAcF,EAAjC,CAIA,IAAME,EAAUE,UAA2B,MAAfN,EAExB,MADAvE,MAAK4D,UAAWD,IACT,CAEX,IAAKY,GAAeA,EAAYO,MAAQ,CACpC,GAAIC,GAAIR,EAAYS,OAAOC,OACvBC,EAAMH,EAAI,GAAKR,EAAYS,OAAQ,IAAOT,EAAYS,OAAQ,GAAIE,GACtE,IAAKA,EACD,GAAmB,OAAdb,EACDA,EAAYa,MACT,IAAKb,IAAca,EAEtB,MADAlF,MAAK4D,UAAWD,EAAU,mFACnB,CAGf,IAAiB,KAAZS,EACDA,EAAUW,MACP,IAAKX,IAAYW,EAEpB,MADA/E,MAAK4D,UAAWD,EAAU,gHAAkHoB,EAAI,wDAA0DX,EAAU,UAC7M,OAtBX3B,QAAOC,KAAKC,MAAM,wBAAyB,oCA0BnD,OAAO,IAIf,QACIhD,KAAMA"}
+{"version":3,"file":"tool-form.js","sources":["../../../src/mvc/tool/tool-form.js"],"names":["define","Utils","Ui","ToolFormBase","ToolTemplate","View","extend","initialize","options","self","this","prototype","call","merge","customize","buttons","execute","execute_btn","Button","icon","tooltip","name","version","title","cls","floating","onclick","wait","portlet","disable","submit","unwait","enable","job_id","job_remap","inputs","label","type","display","ignore","value","help","callback","job_def","tool_id","id","tool_version","data","create","trigger","validate","Galaxy","emit","debug","request","url","root","success","response","$el","empty","append","parent","currHistoryPanel","refreshContents","error","err_data","error_messages","matchResponse","input_id","highlight","modal","show","body","err_msg","Close","hide","job_inputs","batch_n","batch_src","job_input_id","input_value","match","input_field","field_list","input_def","input_list","optional","batch","n","values","length","src"],"mappings":"AAGAA,QAAQ,cAAe,iBAAkB,0BAA2B,0BAChE,SAAUC,EAAOC,EAAIC,EAAcC,GACnC,GAAIC,GAAOF,EAAaG,QACpBC,WAAY,SAAUC,GAClB,GAAIC,GAAOC,IACXP,GAAaQ,UAAUJ,WAAWK,KAAMF,KAAMT,EAAMY,OAChDC,UAAkB,SAAUN,GAExBA,EAAQO,SACJC,QAAUC,YAAc,GAAIf,GAAGgB,QAC3BC,KAAW,WACXC,QAAW,YAAcZ,EAAQa,KAAO,KAAOb,EAAQc,QAAU,IACjEC,MAAW,UACXC,IAAW,4BACXC,SAAW,QACXC,QAAW,WACPT,YAAYU,OACZlB,EAAKmB,QAAQC,UACbpB,EAAKqB,OAAQtB,EAAS,WAClBS,YAAYc,SACZtB,EAAKmB,QAAQI,eAMxBxB,EAAQyB,QAAUzB,EAAQ0B,YAC3B1B,EAAQ2B,OAA6B,oBACjCC,MAAc,oCACdf,KAAc,qBACdgB,KAAc,SACdC,QAAc,QACdC,OAAc,aACdC,MAAc,aACdhC,UAAkB,MAAOA,EAAQyB,SAAY,KAAM,eACnDQ,KAAc,uLAI3BjC,KAOPsB,OAAQ,SAAUtB,EAASkC,GACvB,GAAIjC,GAAOC,KACPiC,GACAC,QAAkBpC,EAAQqC,GAC1BC,aAAkBtC,EAAQc,QAC1Ba,OAAkBzB,KAAKqC,KAAKC,SAGhC,OADAtC,MAAKuC,QAAS,SACRxC,EAAKyC,SAAUP,IAKrBQ,OAAOC,KAAKC,MAAO,sBAAuB,uBAAwBV,OAClE1C,GAAMqD,SACFjB,KAAU,OACVkB,IAAUJ,OAAOK,KAAO,YACxBT,KAAUJ,EACVc,QAAU,SAAUC,GAChBhB,GAAYA,IACZjC,EAAKkD,IAAIC,QAAQC,OAAQzD,EAAaqD,QAASC,IAC/CI,OAAOX,QAAUW,OAAOX,OAAOY,kBAAoBD,OAAOX,OAAOY,iBAAiBC,mBAEtFC,MAAU,SAAUP,GAGhB,GAFAhB,GAAYA,IACZS,OAAOC,KAAKC,MAAO,oBAAqB,qBAAsBK,GACzDA,GAAYA,EAASQ,SAAW,CACjC,GAAIC,GAAiB1D,EAAKsC,KAAKqB,cAAeV,EAASQ,SACvD,KAAK,GAAIG,KAAYF,GAAgB,CACjC1D,EAAK6D,UAAWD,EAAUF,EAAgBE,GAC1C,YAGJ5D,GAAK8D,MAAMC,MACPjD,MAAU,wBACVkD,KAAYf,GAAYA,EAASgB,SAAatE,EAAa6D,MAAOtB,GAClE5B,SACI4D,MAAU,WACNlE,EAAK8D,MAAMK,gBA7B/BzB,OAAOC,KAAKC,MAAO,sBAAuB,gDAC1CX,GAAYA,OAwCpBQ,SAAU,SAAUP,GAChB,GAAIkC,GAAclC,EAAQR,OACtB2C,EAAc,GACdC,EAAc,IAClB,KAAM,GAAIC,KAAgBH,GAAa,CACnC,GAAII,GAAcJ,EAAYG,GAC1BX,EAAc3D,KAAKqC,KAAKmC,MAAOF,GAC/BG,EAAczE,KAAK0E,WAAYf,GAC/BgB,EAAc3E,KAAK4E,WAAYjB,EACnC,IAAMA,GAAagB,GAAcF,EAAjC,CAIA,IAAME,EAAUE,UAA2B,MAAfN,EAExB,MADAvE,MAAK4D,UAAWD,IACT,CAEX,IAAKY,GAAeA,EAAYO,MAAQ,CACpC,GAAIC,GAAIR,EAAYS,OAAOC,OACvBC,EAAMH,EAAI,GAAKR,EAAYS,OAAQ,IAAOT,EAAYS,OAAQ,GAAIE,GACtE,IAAKA,EACD,GAAmB,OAAdb,EACDA,EAAYa,MACT,IAAKb,IAAca,EAEtB,MADAlF,MAAK4D,UAAWD,EAAU,mFACnB,CAGf,IAAiB,KAAZS,EACDA,EAAUW,MACP,IAAKX,IAAYW,EAEpB,MADA/E,MAAK4D,UAAWD,EAAU,gHAAkHoB,EAAI,wDAA0DX,EAAU,UAC7M,OAtBX3B,QAAOC,KAAKC,MAAM,wBAAyB,oCA0BnD,OAAO,IAIf,QACIhD,KAAMA"}

static/scripts/bundled/login.bundled.js

@@ -1,2 +1,2 @@
-webpackJsonp([4],[function(e,r,n){(function(e){var r=n(1),a=r,i=n(60).GalaxyApp,o=n(10),t=n(5),l=n(62);window.app=function(n,c){window.Galaxy=new i(n,c),Galaxy.debug("login app");var d=encodeURI(n.redirect);if(!n.show_welcome_with_login){var w=r.param({use_panels:"True",redirect:d});return void(window.location.href=Galaxy.root+"user/login?"+w)}var p=new l.PageLayoutView(e.extend(n,{el:"body",center:new o.CenterPanel({el:"#center"}),right:new o.RightPanel({title:t("Login required"),el:"#right"})}));a(function(){var e=r.param({redirect:d}),a=Galaxy.root+"user/login?"+e;p.render(),p.center.$("#galaxy_main").prop("src",n.welcome_url),p.right.$(".unified-panel-body").css("overflow","hidden").html('<iframe src="'+a+'" frameborder="0" style="width: 100%; height: 100%;"/>')})}}).call(r,n(3))}]);
+webpackJsonp([4],[function(e,r,n){(function(e){var r=n(1),a=r,i=n(60).GalaxyApp,o=n(11),t=n(5),l=n(62);window.app=function(n,c){window.Galaxy=new i(n,c),Galaxy.debug("login app");var d=encodeURI(n.redirect);if(!n.show_welcome_with_login){var w=r.param({use_panels:"True",redirect:d});return void(window.location.href=Galaxy.root+"user/login?"+w)}var p=new l.PageLayoutView(e.extend(n,{el:"body",center:new o.CenterPanel({el:"#center"}),right:new o.RightPanel({title:t("Login required"),el:"#right"})}));a(function(){var e=r.param({redirect:d}),a=Galaxy.root+"user/login?"+e;p.render(),p.center.$("#galaxy_main").prop("src",n.welcome_url),p.right.$(".unified-panel-body").css("overflow","hidden").html('<iframe src="'+a+'" frameborder="0" style="width: 100%; height: 100%;"/>')})}}).call(r,n(3))}]);
 //# sourceMappingURL=login.bundled.js.map

static/scripts/mvc/history/history-contents.js

@@ -1,2 +1,2 @@
-define(["mvc/history/history-content-model","mvc/history/hda-model","mvc/history/hdca-model","mvc/base-mvc","utils/localization"],function(a,b,c,d){"use strict";var e="history",f=Backbone.Collection.extend(d.LoggableMixin).extend({_logNamespace:e,model:function(a,d){if("dataset"===a.history_content_type)return new b.HistoryDatasetAssociation(a,d);if("dataset_collection"===a.history_content_type){switch(a.collection_type){case"list":return new c.HistoryListDatasetCollection(a,d);case"paired":return new c.HistoryPairDatasetCollection(a,d);case"list:paired":return new c.HistoryListPairedDatasetCollection(a,d)}return{validationError:"Unknown collection_type: "+a.history_content_type}}return{validationError:"Unknown history_content_type: "+a.history_content_type}},initialize:function(a,b){b=b||{},this.historyId=b.historyId,this.model.prototype.idAttribute="type_id",this.on("all",function(){this.debug(this+".event:",arguments)})},urlRoot:Galaxy.root+"api/histories",url:function(){return this.urlRoot+"/"+this.historyId+"/contents"},ids:function(){return this.map(function(a){return a.get("id")})},notReady:function(){return this.filter(function(a){return!a.inReadyState()})},running:function(){var a=[];return this.each(function(b){var c=!b.inReadyState();c&&a.push(b.get("id"))}),a},getByHid:function(a){return _.first(this.filter(function(b){return b.get("hid")===a}))},getVisible:function(a,b,c){c=c||[],this.debug("checking isVisible");var d=new f(this.filter(function(c){return c.isVisible(a,b)}));return _.each(c,function(a){_.isFunction(a)&&(d=new f(d.filter(a)))}),d},hidden:function(){function a(a){return a.hidden()}return new f(this.filter(a))},deleted:function(){function a(a){return a.get("deleted")}return new f(this.filter(a))},haveDetails:function(){return this.all(function(a){return a.hasDetails()})},fetchAllDetails:function(a){a=a||{};var b={details:"all"};return a.data=a.data?_.extend(a.data,b):b,this.fetch(a)},ajaxQueue:function(a,b){var c=jQuery.Deferred(),d=this.length,e=[];if(!d)return c.resolve([]),c;var f=this.chain().reverse().map(function(g,h){return function(){var i=a.call(g,b);i.done(function(a){c.notify({curr:h,total:d,response:a,model:g})}),i.always(function(a){e.push(a),f.length?f.shift()():c.resolve(e)})}}).value();return f.shift()(),c},isCopyable:function(a){var b=["HistoryDatasetAssociation","HistoryDatasetCollectionAssociation"];return _.isObject(a)&&a.id&&_.contains(b,a.model_class)},copy:function(a){var b,c,d;_.isString(a)?(b=a,d="hda",c="dataset"):(b=a.id,d={HistoryDatasetAssociation:"hda",LibraryDatasetDatasetAssociation:"ldda",HistoryDatasetCollectionAssociation:"hdca"}[a.model_class]||"hda",c="hdca"===d?"dataset_collection":"dataset");var e=this,f=jQuery.post(this.url(),{content:b,source:d,type:c}).done(function(a){e.add([a])}).fail(function(){e.trigger("error",e,f,{},"Error copying contents",{type:c,id:b,source:d})});return f},matches:function(a){return this.filter(function(b){return b.matches(a)})},set:function(b,c){b=_.isArray(b)?b:[b],_.each(b,function(b){b.type_id&&b.get&&b.get("type_id")||(b.type_id=a.typeIdStr(b.history_content_type,b.id))}),Backbone.Collection.prototype.set.call(this,b,c)},createHDCA:function(a,b,d){var e=this,f={list:c.HistoryListDatasetCollection,paired:c.HistoryPairDatasetCollection},g=new f[b]({history_id:this.historyId,name:d,element_identifiers:a});return g.save().done(function(){e.add(g)}).fail(function(a,b,c){e.trigger("error",a,b,c)})},clone:function(){var a=Backbone.Collection.prototype.clone.call(this);return a.historyId=this.historyId,a},print:function(){var a=this;a.each(function(b){a.debug(b),b.elements&&a.debug("	 elements:",b.elements)})},toString:function(){return["HistoryContents(",[this.historyId,this.length].join(),")"].join("")}});return{HistoryContents:f}});
+define(["mvc/history/history-content-model","mvc/history/hda-model","mvc/history/hdca-model","mvc/dataset/states","mvc/base-mvc","utils/localization"],function(a,b,c,d,e){"use strict";var f="history",g=Backbone.Collection.extend(e.LoggableMixin).extend({_logNamespace:f,model:function(a,d){if("dataset"===a.history_content_type)return new b.HistoryDatasetAssociation(a,d);if("dataset_collection"===a.history_content_type){switch(a.collection_type){case"list":return new c.HistoryListDatasetCollection(a,d);case"paired":return new c.HistoryPairDatasetCollection(a,d);case"list:paired":return new c.HistoryListPairedDatasetCollection(a,d)}return{validationError:"Unknown collection_type: "+a.history_content_type}}return{validationError:"Unknown history_content_type: "+a.history_content_type}},initialize:function(a,b){b=b||{},this.historyId=b.historyId,this.model.prototype.idAttribute="type_id",this.on("all",function(){this.debug(this+".event:",arguments)})},urlRoot:Galaxy.root+"api/histories",url:function(){return this.urlRoot+"/"+this.historyId+"/contents"},ids:function(){return this.map(function(a){return a.get("id")})},notReady:function(){return this.filter(function(a){return!a.inReadyState()})},running:function(){function a(a){return!a.inReadyState()}return new g(this.filter(a))},getByHid:function(a){return _.first(this.filter(function(b){return b.get("hid")===a}))},getVisible:function(a,b,c){c=c||[],this.debug("checking isVisible");var d=new g(this.filter(function(c){return c.isVisible(a,b)}));return _.each(c,function(a){_.isFunction(a)&&(d=new g(d.filter(a)))}),d},hidden:function(){function a(a){return a.hidden()}return new g(this.filter(a))},deleted:function(){function a(a){return a.get("deleted")}return new g(this.filter(a))},visibleAndUndeleted:function(){function a(a){return a.get("visible")&&!a.get("deleted")}return new g(this.filter(a))},haveDetails:function(){return this.all(function(a){return a.hasDetails()})},fetch:function(a){return a=a||{},a.data=_.defaults(a.data||{},{v:"dev"}),Backbone.Collection.prototype.fetch.call(this,a)},fetchUpdated:function(a,b){return b=b||{},b.traditional=!0,b.data=[{name:"v",value:"dev"}],a&&(b.data=b.data.concat(this._filtersFromMap({"update_time-ge":a.toISOString()}))),b.merge=!0,b.remove=!1,this.fetch(b)},_filtersFromMap:function(a){var b=[];return _.each(a,function(a,c){b.push({name:"q",value:c}),b.push({name:"qv",value:a})}),b},fetchAllDetails:function(a){a=a||{};var b={details:"all"};return a.data=a.data?_.extend(a.data,b):b,this.fetch(a)},ajaxQueue:function(a,b){var c=jQuery.Deferred(),d=this.length,e=[];if(!d)return c.resolve([]),c;var f=this.chain().reverse().map(function(g,h){return function(){var i=a.call(g,b);i.done(function(a){c.notify({curr:h,total:d,response:a,model:g})}),i.always(function(a){e.push(a),f.length?f.shift()():c.resolve(e)})}}).value();return f.shift()(),c},isCopyable:function(a){var b=["HistoryDatasetAssociation","HistoryDatasetCollectionAssociation"];return _.isObject(a)&&a.id&&_.contains(b,a.model_class)},copy:function(a){var b,c,d;_.isString(a)?(b=a,d="hda",c="dataset"):(b=a.id,d={HistoryDatasetAssociation:"hda",LibraryDatasetDatasetAssociation:"ldda",HistoryDatasetCollectionAssociation:"hdca"}[a.model_class]||"hda",c="hdca"===d?"dataset_collection":"dataset");var e=this,f=jQuery.post(this.url(),{content:b,source:d,type:c}).done(function(a){e.add([a])}).fail(function(){e.trigger("error",e,f,{},"Error copying contents",{type:c,id:b,source:d})});return f},matches:function(a){return this.filter(function(b){return b.matches(a)})},set:function(b,c){b=_.isArray(b)?b:[b],_.each(b,function(b){b.type_id&&b.get&&b.get("type_id")||(b.type_id=a.typeIdStr(b.history_content_type,b.id))}),Backbone.Collection.prototype.set.call(this,b,c)},createHDCA:function(a,b,d){var e=this,f={list:c.HistoryListDatasetCollection,paired:c.HistoryPairDatasetCollection},g=new f[b]({history_id:this.historyId,name:d,element_identifiers:a});return g.save().done(function(){e.add(g)}).fail(function(a,b,c){e.trigger("error",a,b,c)})},clone:function(){var a=Backbone.Collection.prototype.clone.call(this);return a.historyId=this.historyId,a},print:function(){var a=this;a.each(function(b){a.debug(b),b.elements&&a.debug("	 elements:",b.elements)})},toString:function(){return["HistoryContents(",[this.historyId,this.length].join(),")"].join("")}});return{HistoryContents:g}});
 //# sourceMappingURL=../../../maps/mvc/history/history-contents.js.map

static/scripts/mvc/tool/tool-form.js

@@ -1,2 +1,2 @@
-define(["utils/utils","mvc/ui/ui-misc","mvc/tool/tool-form-base","mvc/tool/tool-template"],function(a,b,c,d){var e=c.extend({initialize:function(d){var e=this;c.prototype.initialize.call(this,a.merge({customize:function(a){a.buttons={execute:execute_btn=new b.Button({icon:"fa-check",tooltip:"Execute: "+a.name+" ("+a.version+")",title:"Execute",cls:"btn btn-primary",floating:"clear",onclick:function(){execute_btn.wait(),e.portlet.disable(),e.submit(a,function(){execute_btn.unwait(),e.portlet.enable()})}})},a.job_id&&a.job_remap&&(a.inputs.rerun_remap_job_id={label:"Resume dependencies from this job",name:"rerun_remap_job_id",type:"select",display:"radio",ignore:"__ignore__",value:"__ignore__",options:[["Yes",a.job_id],["No","__ignore__"]],help:"The previous run of this tool failed and other tools were waiting for it to finish successfully. Use this option to resume those tools using the new output(s) of this tool run."})}},d))},submit:function(b,c){var e=this,f={tool_id:b.id,tool_version:b.version,inputs:this.data.create()};return this.trigger("reset"),e.validate(f)?(Galaxy.emit.debug("tool-form::submit()","Validation complete.",f),void a.request({type:"POST",url:Galaxy.root+"api/tools",data:f,success:function(a){c&&c(),e.$el.empty().append(d.success(a)),parent.Galaxy&&parent.Galaxy.currHistoryPanel&&parent.Galaxy.currHistoryPanel.refreshContents()},error:function(a){if(c&&c(),Galaxy.emit.debug("tool-form::submit","Submission failed.",a),a&&a.err_data){var b=e.data.matchResponse(a.err_data);for(var g in b){e.highlight(g,b[g]);break}}else e.modal.show({title:"Job submission failed",body:a&&a.err_msg||d.error(f),buttons:{Close:function(){e.modal.hide()}}})}})):(Galaxy.emit.debug("tool-form::submit()","Submission canceled. Validation failed."),void(c&&c()))},validate:function(a){var b=a.inputs,c=-1,d=null;for(var e in b){var f=b[e],g=this.data.match(e),h=this.field_list[g],i=this.input_list[g];if(g&&i&&h){if(!i.optional&&null==f)return this.highlight(g),!1;if(f&&f.batch){var j=f.values.length,k=j>0&&f.values[0]&&f.values[0].src;if(k)if(null===d)d=k;else if(d!==k)return this.highlight(g,"Please select either dataset or dataset list fields for all batch mode fields."),!1;if(-1===c)c=j;else if(c!==j)return this.highlight(g,"Please make sure that you select the same number of inputs for all batch mode fields. This field contains <b>"+j+"</b> selection(s) while a previous field contains <b>"+c+"</b>."),!1}}else Galaxy.emit.debug("tool-form::validate()","Retrieving input objects failed.")}return!0}});return{View:e}});
+define(["utils/utils","mvc/ui/ui-misc","mvc/tool/tool-form-base","mvc/tool/tool-template"],function(a,b,c,d){var e=c.extend({initialize:function(d){var e=this;c.prototype.initialize.call(this,a.merge({customize:function(a){a.buttons={execute:execute_btn=new b.Button({icon:"fa-check",tooltip:"Execute: "+a.name+" ("+a.version+")",title:"Execute",cls:"ui-button btn btn-primary",floating:"clear",onclick:function(){execute_btn.wait(),e.portlet.disable(),e.submit(a,function(){execute_btn.unwait(),e.portlet.enable()})}})},a.job_id&&a.job_remap&&(a.inputs.rerun_remap_job_id={label:"Resume dependencies from this job",name:"rerun_remap_job_id",type:"select",display:"radio",ignore:"__ignore__",value:"__ignore__",options:[["Yes",a.job_id],["No","__ignore__"]],help:"The previous run of this tool failed and other tools were waiting for it to finish successfully. Use this option to resume those tools using the new output(s) of this tool run."})}},d))},submit:function(b,c){var e=this,f={tool_id:b.id,tool_version:b.version,inputs:this.data.create()};return this.trigger("reset"),e.validate(f)?(Galaxy.emit.debug("tool-form::submit()","Validation complete.",f),void a.request({type:"POST",url:Galaxy.root+"api/tools",data:f,success:function(a){c&&c(),e.$el.empty().append(d.success(a)),parent.Galaxy&&parent.Galaxy.currHistoryPanel&&parent.Galaxy.currHistoryPanel.refreshContents()},error:function(a){if(c&&c(),Galaxy.emit.debug("tool-form::submit","Submission failed.",a),a&&a.err_data){var b=e.data.matchResponse(a.err_data);for(var g in b){e.highlight(g,b[g]);break}}else e.modal.show({title:"Job submission failed",body:a&&a.err_msg||d.error(f),buttons:{Close:function(){e.modal.hide()}}})}})):(Galaxy.emit.debug("tool-form::submit()","Submission canceled. Validation failed."),void(c&&c()))},validate:function(a){var b=a.inputs,c=-1,d=null;for(var e in b){var f=b[e],g=this.data.match(e),h=this.field_list[g],i=this.input_list[g];if(g&&i&&h){if(!i.optional&&null==f)return this.highlight(g),!1;if(f&&f.batch){var j=f.values.length,k=j>0&&f.values[0]&&f.values[0].src;if(k)if(null===d)d=k;else if(d!==k)return this.highlight(g,"Please select either dataset or dataset list fields for all batch mode fields."),!1;if(-1===c)c=j;else if(c!==j)return this.highlight(g,"Please make sure that you select the same number of inputs for all batch mode fields. This field contains <b>"+j+"</b> selection(s) while a previous field contains <b>"+c+"</b>."),!1}}else Galaxy.emit.debug("tool-form::validate()","Retrieving input objects failed.")}return!0}});return{View:e}});
 //# sourceMappingURL=../../../maps/mvc/tool/tool-form.js.map

templates/webapps/galaxy/workflow/display.mako

@@ -30,6 +30,15 @@
       <% new_prefix = prefix + input.name + "|" %>
       ${row_for_param( input.test_param, group_values[ input.test_param.name ], other_values, prefix, step )}
       ${do_inputs( input.cases[ current_case ].inputs, group_values, new_prefix, step, other_values )}
+    %elif input.type == "section":
+      <% new_prefix = prefix + input.name + "|" %>
+      <% group_values = values[input.name] %>
+      <div class="form-title-row"><b>${input.title}:</b></div>
+      <div class="repeat-group">
+        <div class="repeat-group-item">
+          ${do_inputs( input.inputs, group_values, new_prefix, step, other_values )}
+        </div>
+      </div>
     %else:
       ${row_for_param( input, values[ input.name ], other_values, prefix, step )}
     %endif

test/unit/tools/test_column_parameters.py

@@ -25,22 +25,22 @@ class DataColumnParameterTestCase( BaseParameterTestCase ):
         # TODO: don't break abstraction, try setting null value instead
         return self.param.optional
 
-    def test_from_html(self):
-        value = self.param.from_html("3", self.trans, { "input_tsv": self.build_ready_hda()  } )
+    def test_from_json(self):
+        value = self.param.from_json("3", self.trans, { "input_tsv": self.build_ready_hda()  } )
         assert value == "3"
 
-    def test_from_html_strips_c(self):
-        value = self.param.from_html("c1", self.trans, { "input_tsv": self.build_ready_hda()  } )
+    def test_from_json_strips_c(self):
+        value = self.param.from_json("c1", self.trans, { "input_tsv": self.build_ready_hda()  } )
         assert value == "1"
 
-    def test_multiple_from_html(self):
+    def test_multiple_from_json(self):
         self.multiple = True
-        value = self.param.from_html("1,2,3", self.trans, { "input_tsv": self.build_ready_hda()  } )
+        value = self.param.from_json("1,2,3", self.trans, { "input_tsv": self.build_ready_hda()  } )
         assert value == ["1", "2", "3"]
 
-    def test_multiple_from_html_with_c(self):
+    def test_multiple_from_json_with_c(self):
         self.multiple = True
-        value = self.param.from_html("c1,c2,c3", self.trans, { "input_tsv": self.build_ready_hda()  } )
+        value = self.param.from_json("c1,c2,c3", self.trans, { "input_tsv": self.build_ready_hda()  } )
         assert value == ["1", "2", "3"]
 
     def test_get_initial_value_default(self):

test/unit/tools/test_data_parameters.py

@@ -7,9 +7,9 @@ from .test_parameter_parsing import BaseParameterTestCase
 class DataToolParameterTestCase( BaseParameterTestCase ):
 
     def test_to_python_none_values( self ):
-        assert None is self.param.to_python( None, self.app )
-        assert 'None' == self.param.to_python( 'None', self.app )
-        assert '' == self.param.to_python( '', self.app )
+        assert self.param.to_python( None, self.app ) is None
+        assert self.param.to_python( 'None', self.app ) is None
+        assert self.param.to_python( '', self.app ) is None
 
     def test_to_python_hda( self ):
         hda = self._new_hda()

test/unit/tools/test_select_parameters.py

@@ -10,7 +10,7 @@ class SelectToolParameterTestCase( BaseParameterTestCase ):
     def test_validated_values( self ):
         self.options_xml = '''<options><filter type="data_meta" ref="input_bam" key="dbkey"/></options>'''
         try:
-            self.param.from_html("42", self.trans, { "input_bam": model.HistoryDatasetAssociation() })
+            self.param.from_json("42", self.trans, { "input_bam": model.HistoryDatasetAssociation() })
         except ValueError, err:
             assert str(err) == "An invalid option was selected for my_name, '42', please verify."
             return
@@ -19,7 +19,7 @@ class SelectToolParameterTestCase( BaseParameterTestCase ):
     def test_validated_values_missing_dependency( self ):
         self.options_xml = '''<options><filter type="data_meta" ref="input_bam" key="dbkey"/></options>'''
         try:
-            self.param.from_html("42", self.trans)
+            self.param.from_json("42", self.trans)
         except AssertionError, err:
             assert str(err) == "Required dependency 'input_bam' not found in incoming values"
             return
@@ -28,12 +28,12 @@ class SelectToolParameterTestCase( BaseParameterTestCase ):
     def test_unvalidated_values( self ):
         self.options_xml = '''<options><filter type="data_meta" ref="input_bam" key="dbkey"/></options>'''
         self.trans.workflow_building_mode = True
-        assert self.param.from_html("42", self.trans) == "42"
+        assert self.param.from_json("42", self.trans) == "42"
 
     def test_validated_datasets( self ):
         self.options_xml = '''<options><filter type="data_meta" ref="input_bam" key="dbkey"/></options>'''
         try:
-            self.param.from_html( model.HistoryDatasetAssociation(), self.trans, { "input_bam": basic.RuntimeValue() } )
+            self.param.from_json( model.HistoryDatasetAssociation(), self.trans, { "input_bam": basic.RuntimeValue() } )
         except ValueError, err:
             assert str(err) == "Parameter my_name requires a value, but has no legal values defined."
             return
@@ -42,7 +42,7 @@ class SelectToolParameterTestCase( BaseParameterTestCase ):
     def test_unvalidated_datasets( self ):
         self.options_xml = '''<options><filter type="data_meta" ref="input_bam" key="dbkey"/></options>'''
         self.trans.workflow_building_mode = True
-        assert isinstance( self.param.from_html( model.HistoryDatasetAssociation(), self.trans, { "input_bam": basic.RuntimeValue() } ), model.HistoryDatasetAssociation )
+        assert isinstance( self.param.from_json( model.HistoryDatasetAssociation(), self.trans, { "input_bam": basic.RuntimeValue() } ), model.HistoryDatasetAssociation )
 
     def test_filter_param_value( self ):
         self.options_xml = '''<options from_data_table="test_table"><filter type="param_value" ref="input_bam" column="0" /></options>'''