Merge branch 'dev' into remove_sample_tracking

.gitignore

@@ -124,7 +124,7 @@ sprite-*.less
 # JS, Local node_modules, and bower_components directories
 static/scripts/bundled/[0-9]*
 static/maps
-static/scripts/bundled/*.js.map
+static/scripts/bundled/*.map
 node_modules
 bower_components
 client/galaxy/scripts/libs/*

Makefile

@@ -17,6 +17,7 @@ DOC_SOURCE_DIR=$(DOCS_DIR)/source
 SLIDESHOW_DIR=$(DOC_SOURCE_DIR)/slideshow
 OPEN_RESOURCE=bash -c 'open $$0 || xdg-open $$0'
 SLIDESHOW_TO_PDF?=bash -c 'docker run --rm -v `pwd`:/cwd astefanutti/decktape /cwd/$$0 /cwd/`dirname $$0`/`basename -s .html $$0`.pdf'
+CLIENT_COMMIT_WARNING="Please remember to 'make client-production' when finished developing, before a commit!"
 
 all: help
 	@echo "This makefile is used for building Galaxy's JS client, documentation, and drive the release process. A sensible all target is not implemented."
@@ -128,15 +129,22 @@ update-and-commit-dependencies:  ## update and commit linting + dev dependencies
 node-deps: ## Install NodeJS dependencies.
 	cd client && yarn install --check-files
 
-client: node-deps ## Rebuild all client-side artifacts
+client: node-deps ## Rebuild client-side artifacts for local development.
 	cd client && yarn run build
+	@echo $(CLIENT_COMMIT_WARNING)
+
+client-production: node-deps ## Rebuild client-side artifacts for a production deployment (or committing to the repository).
+	cd client && yarn run build-production
+
+client-production-maps: node-deps ## Rebuild client-side artifacts for a production deployment, and include sourcemaps to aid in debugging efforts.
+	cd client && yarn run build-production-maps
 
 client-format: node-deps ## Reformat client code
 	cd client && yarn run prettier
 
 client-watch: node-deps ## A useful target for parallel development building.
 	cd client && yarn run watch
-	@echo "Remember to 'make client' when finished developing!"
+	@echo $(CLIENT_COMMIT_WARNING)
 
 client-test: client ## Run qunit tests via Karma
 	cd client && yarn run test

client/README.md

@@ -16,11 +16,27 @@ Complete Client Build
 
 There are many moving parts to the client build system, but the entrypoint for
 most people is the 'client' rule in the Makefile at the root of the Galaxy
-repository.  Execute the following to perform a complete build including
-dependency staging, style building, script processing and bundling.
+repository.  Execute the following to perform a complete build suitable for
+local development, including dependency staging, style building, script
+processing and bundling.
 
     make client
 
+Note: Don't commit the files that 'make client' creates.  This is a
+development-specific build which includes extra debugging features, and
+excludes several production optimizations made during the build process.
+
+For a production build, suitable for committing to the repository, or deploying
+to a live server, use the following:
+
+    make client-production
+
+And, lastly, if you want a production build that includes sourcemaps to allow
+for perusal of the page-level javascript to facilitate live-server debugging,
+use:
+
+    make client-production-maps
+
 
 Automatic Rebuilding (Watch Mode)
 =================================

client/galaxy/scripts/apps/admin.js

@@ -10,7 +10,7 @@ import Router from "layout/router";
 import Utils from "utils/utils";
 import Page from "layout/page";
 import Vue from "libs/vue";
-import UserAPIKeys from "components/user_api_keys.vue";
+import UserAPIKeys from "components/UserAPIKeys.vue";
 
 window.app = function app(options, bootstrapped) {
     window.Galaxy = new GalaxyApp.GalaxyApp(options, bootstrapped);

client/galaxy/scripts/apps/analysis.js

@@ -13,7 +13,7 @@ import Tours from "mvc/tours";
 import GridView from "mvc/grid/grid-view";
 import GridShared from "mvc/grid/grid-shared";
 import Workflows from "mvc/workflow/workflow";
-import HistoryImport from "components/history-import.vue";
+import HistoryImport from "components/HistoryImport.vue";
 import HistoryList from "mvc/history/history-list";
 import ToolFormComposite from "mvc/tool/tool-form-composite";
 import QueryStringParsing from "utils/query-string-parsing";
@@ -21,7 +21,7 @@ import Utils from "utils/utils";
 import Ui from "mvc/ui/ui-misc";
 import DatasetError from "mvc/dataset/dataset-error";
 import DatasetEditAttributes from "mvc/dataset/dataset-edit-attributes";
-import Citations from "components/citations.vue";
+import Citations from "components/Citations.vue";
 import Vue from "libs/vue";
 
 /** define the 'Analyze Data'/analysis/main/home page for Galaxy

client/galaxy/scripts/components/Citations.vue

@@ -2,11 +2,11 @@
     <div class="toolForm">
         <div class="toolFormTitle">
             Citations
-            <button v-if="viewRender" v-on:click="toggleViewRender" type="button" class="btn btn-xs" title="Show all in BibTeX format.">
+            <button v-if="viewRender" v-on:click="toggleViewRender" type="button" class="btn btn-xs citations-to-bibtex" title="Show all in BibTeX format.">
                 <i class="fa fa-pencil-square-o"></i>
                 Show BibTeX
             </button>
-            <button v-else type="button" v-on:click="toggleViewRender" class="btn btn-xs" title="Return to formatted citation list.">
+            <button v-else type="button" v-on:click="toggleViewRender" class="btn btn-xs citations-to-formatted" title="Return to formatted citation list.">
                 <i class="fa fa-times"></i>
                 Hide BibTeX
             </button>

client/galaxy/scripts/mvc/collection/collection-model.js

@@ -90,7 +90,10 @@ var DatasetCollectionElement = Backbone.Model.extend(BASE_MVC.LoggableMixin)
     .extend({ _logNamespace: "collections" });
 
 //==============================================================================
-/** @class Base/Abstract Backbone collection for Generic DCEs. */
+/** @class Base/Abstract Backbone collection for Generic DCEs - 
+    current may be associated with a dataset (DatasetDCECollection)
+    or another collection (NestedDCDCECollection).
+*/
 var DCECollection = Backbone.Collection.extend(BASE_MVC.LoggableMixin).extend(
     /** @lends DCECollection.prototype */ {
         _logNamespace: "collections",
@@ -197,7 +200,13 @@ var DatasetCollection = Backbone.Model.extend(BASE_MVC.LoggableMixin)
             },
 
             /** Which class to use for elements */
-            collectionClass: DCECollection,
+            collectionClass: function() {
+                if (this.attributes.collection_type.indexOf(":") > 0) {
+                    return NestedDCDCECollection;
+                } else {
+                    return DatasetDCECollection;
+                }
+            },
 
             /** set up: create elements instance var and (on changes to elements) update them  */
             initialize: function(model, options) {
@@ -212,7 +221,8 @@ var DatasetCollection = Backbone.Model.extend(BASE_MVC.LoggableMixin)
 
             /** move elements model attribute to full collection */
             _createElementsModel: function() {
-                this.debug(`${this}._createElementsModel`, this.collectionClass, this.get("elements"), this.elements);
+                var collectionClass = this.collectionClass();
+                this.debug(`${this}._createElementsModel`, collectionClass, this.get("elements"), this.elements);
                 //TODO: same patterns as DatasetCollectionElement _createObjectModel - refactor to BASE_MVC.hasSubModel?
                 var elements = this.get("elements") || [];
                 this.unset("elements", { silent: true });
@@ -222,7 +232,7 @@ var DatasetCollection = Backbone.Model.extend(BASE_MVC.LoggableMixin)
                         parent_hdca_id: self.get("id")
                     });
                 });
-                this.elements = new this.collectionClass(elements);
+                this.elements = new collectionClass(elements);
                 //this.debug( 'collectionClass:', this.collectionClass + '', this.elements );
                 return this.elements;
             },
@@ -303,39 +313,12 @@ var DatasetCollection = Backbone.Model.extend(BASE_MVC.LoggableMixin)
         }
     );
 
-//==============================================================================
-/** Model for a DatasetCollection containing datasets (non-nested).
- */
-var ListDatasetCollection = DatasetCollection.extend(
-    /** @lends ListDatasetCollection.prototype */ {
-        /** override since we know the collection will only contain datasets */
-        collectionClass: DatasetDCECollection,
-
-        /** String representation. */
-        toString: function() {
-            return `List${DatasetCollection.prototype.toString.call(this)}`;
-        }
-    }
-);
-
-//==============================================================================
-/** Model for a DatasetCollection containing fwd/rev datasets (a list of 2).
- */
-var PairDatasetCollection = ListDatasetCollection.extend(
-    /** @lends PairDatasetCollection.prototype */ {
-        /** String representation. */
-        toString: function() {
-            return `Pair${DatasetCollection.prototype.toString.call(this)}`;
-        }
-    }
-);
-
 //_________________________________________________________________________________________________ NESTED COLLECTIONS
 // this is where things get weird, man. Weird.
 //TODO: it might be possible to compact all the following...I think.
 //==============================================================================
 /** @class Backbone model for a Generic DatasetCollectionElement that is also a DatasetCollection
- *      (a nested collection). Currently only list:paired.
+ *      (a nested collection).
  */
 var NestedDCDCE = DatasetCollection.extend(
     BASE_MVC.mixin(
@@ -374,113 +357,7 @@ var NestedDCDCECollection = DCECollection.extend(
     }
 );
 
-//==============================================================================
-/** @class Backbone model for a paired dataset collection within a list:paired dataset collection.
- */
-var NestedPairDCDCE = PairDatasetCollection.extend(
-    BASE_MVC.mixin(
-        DatasetCollectionElementMixin,
-        /** @lends NestedPairDCDCE.prototype */ {
-            //TODO:?? possibly rename to NestedDatasetCollection?
-
-            // because all objects have constructors (as this hashmap would even if this next line wasn't present)
-            //  the constructor in hcontentMixin won't be attached by BASE_MVC.mixin to this model
-            //  - re-apply manually it now
-            /** This is both a collection and a collection element - call the constructor */
-            constructor: function(attributes, options) {
-                this.debug("\t NestedPairDCDCE.constructor:", attributes, options);
-                //DatasetCollection.constructor.call( this, attributes, options );
-                DatasetCollectionElementMixin.constructor.call(this, attributes, options);
-            },
-
-            /** String representation. */
-            toString: function() {
-                var objStr = this.object ? `${this.object}` : this.get("element_identifier");
-                return ["NestedPairDCDCE(", objStr, ")"].join("");
-            }
-        }
-    )
-);
-
-//==============================================================================
-/** @class Backbone collection for a backbone collection containing paired dataset collections.
- */
-var NestedPairDCDCECollection = NestedDCDCECollection.extend(
-    /** @lends PairDCDCECollection.prototype */ {
-        /** We know this collection is composed of only nested pair collections */
-        model: NestedPairDCDCE,
-
-        /** String representation. */
-        toString: function() {
-            return ["NestedPairDCDCECollection(", this.length, ")"].join("");
-        }
-    }
-);
-
-//==============================================================================
-/** @class Backbone Model for a DatasetCollection (list) that contains DatasetCollections (pairs).
- */
-var ListPairedDatasetCollection = DatasetCollection.extend(
-    /** @lends ListPairedDatasetCollection.prototype */ {
-        /** list:paired is the only collection that itself contains collections */
-        collectionClass: NestedPairDCDCECollection,
-
-        /** String representation. */
-        toString: function() {
-            return ["ListPairedDatasetCollection(", this.get("name"), ")"].join("");
-        }
-    }
-);
-
-//==============================================================================
-/** @class Backbone model for a list dataset collection within a list:list dataset collection. */
-var NestedListDCDCE = ListDatasetCollection.extend(
-    BASE_MVC.mixin(
-        DatasetCollectionElementMixin,
-        /** @lends NestedListDCDCE.prototype */ {
-            /** This is both a collection and a collection element - call the constructor */
-            constructor: function(attributes, options) {
-                this.debug("\t NestedListDCDCE.constructor:", attributes, options);
-                DatasetCollectionElementMixin.constructor.call(this, attributes, options);
-            },
-
-            /** String representation. */
-            toString: function() {
-                var objStr = this.object ? `${this.object}` : this.get("element_identifier");
-                return ["NestedListDCDCE(", objStr, ")"].join("");
-            }
-        }
-    )
-);
-
-//==============================================================================
-/** @class Backbone collection containing list dataset collections. */
-var NestedListDCDCECollection = NestedDCDCECollection.extend({
-    /** We know this collection is composed of only nested pair collections */
-    model: NestedListDCDCE,
-
-    /** String representation. */
-    toString: function() {
-        return ["NestedListDCDCECollection(", this.length, ")"].join("");
-    }
-});
-
-//==============================================================================
-/** @class Backbone Model for a DatasetCollection (list) that contains other lists. */
-var ListOfListsDatasetCollection = DatasetCollection.extend({
-    /** list:paired is the only collection that itself contains collections */
-    collectionClass: NestedListDCDCECollection,
-
-    /** String representation. */
-    toString: function() {
-        return ["ListOfListsDatasetCollection(", this.get("name"), ")"].join("");
-    }
-});
-
 //==============================================================================
 export default {
-    ListDatasetCollection: ListDatasetCollection,
-    PairDatasetCollection: PairDatasetCollection,
-    ListPairedDatasetCollection: ListPairedDatasetCollection,
-    ListOfListsDatasetCollection: ListOfListsDatasetCollection
+    DatasetCollection: DatasetCollection
 };

client/galaxy/scripts/mvc/collection/collection-view-edit.js

@@ -26,6 +26,12 @@ var CollectionViewEdit = _super.extend(
         /** sub view class used for nested collections */
         NestedDCDCEViewClass: DC_EDIT.NestedDCDCEListItemEdit,
 
+        getNestedDCDCEViewClass: function() {
+            return DC_EDIT.NestedDCDCEListItemEdit.extend({
+                foldoutPanelClass: CollectionViewEdit
+            });
+        },
+
         // ......................................................................... SET UP
         /** Set up the view, set up storage, bind listeners to HistoryContents events
          *  @param {Object} attributes optional settings for the panel
@@ -90,94 +96,7 @@ var CollectionViewEdit = _super.extend(
     }
 );
 
-// =============================================================================
-/** @class non-editable, read-only View/Controller for a dataset collection. */
-var ListCollectionViewEdit = CollectionViewEdit.extend(
-    /** @lends ListCollectionView.prototype */ {
-        //TODO: not strictly needed - due to switch in CollectionView._getContentClass
-        /** sub view class used for datasets */
-        DatasetDCEViewClass: DC_EDIT.DatasetDCEListItemEdit,
-
-        // ........................................................................ misc
-        /** string rep */
-        toString: function() {
-            return `ListCollectionViewEdit(${this.model ? this.model.get("name") : ""})`;
-        }
-    }
-);
-
-// =============================================================================
-/** @class Editable, read-only View/Controller for a dataset collection. */
-var PairCollectionViewEdit = ListCollectionViewEdit.extend(
-    /** @lends PairCollectionViewEdit.prototype */ {
-        // ........................................................................ misc
-        /** string rep */
-        toString: function() {
-            return `PairCollectionViewEdit(${this.model ? this.model.get("name") : ""})`;
-        }
-    }
-);
-
-// =============================================================================
-/** @class Editable (roughly since these collections are immutable),
- *  View/Controller for a dataset collection.
- */
-var NestedPairCollectionViewEdit = PairCollectionViewEdit.extend(
-    /** @lends NestedPairCollectionViewEdit.prototype */ {
-        /** Override to remove the editable text from the name/identifier - these collections are considered immutable */
-        _setUpBehaviors: function($where) {
-            _super.prototype._setUpBehaviors.call(this, $where);
-        },
-
-        // ........................................................................ misc
-        /** string rep */
-        toString: function() {
-            return `NestedPairCollectionViewEdit(${this.model ? this.model.get("name") : ""})`;
-        }
-    }
-);
-
-// =============================================================================
-/** @class editable, View/Controller for a list of pairs dataset collection. */
-var ListOfPairsCollectionViewEdit = CollectionViewEdit.extend(
-    /** @lends ListOfPairsCollectionView.prototype */ {
-        //TODO: not strictly needed - due to switch in CollectionView._getContentClass
-        /** sub view class used for nested collections */
-        NestedDCDCEViewClass: DC_EDIT.NestedDCDCEListItemEdit.extend({
-            foldoutPanelClass: NestedPairCollectionViewEdit
-        }),
-
-        // ........................................................................ misc
-        /** string rep */
-        toString: function() {
-            return `ListOfPairsCollectionViewEdit(${this.model ? this.model.get("name") : ""})`;
-        }
-    }
-);
-
-// =============================================================================
-/** @class View/Controller for a list of lists dataset collection. */
-var ListOfListsCollectionViewEdit = CollectionViewEdit.extend(
-    /** @lends ListOfListsCollectionView.prototype */ {
-        //TODO: not strictly needed - due to switch in CollectionView._getContentClass
-        /** sub view class used for nested collections */
-        NestedDCDCEViewClass: DC_EDIT.NestedDCDCEListItemEdit.extend({
-            foldoutPanelClass: NestedPairCollectionViewEdit
-        }),
-
-        // ........................................................................ misc
-        /** string rep */
-        toString: function() {
-            return `ListOfListsCollectionViewEdit(${this.model ? this.model.get("name") : ""})`;
-        }
-    }
-);
-
 //==============================================================================
 export default {
-    CollectionViewEdit: CollectionViewEdit,
-    ListCollectionViewEdit: ListCollectionViewEdit,
-    PairCollectionViewEdit: PairCollectionViewEdit,
-    ListOfPairsCollectionViewEdit: ListOfPairsCollectionViewEdit,
-    ListOfListsCollectionViewEdit: ListOfListsCollectionViewEdit
+    CollectionViewEdit: CollectionViewEdit
 };

client/galaxy/scripts/mvc/collection/collection-view.js

@@ -21,8 +21,7 @@ var CollectionView = _super.extend(
 
         /** sub view class used for datasets */
         DatasetDCEViewClass: DC_LI.DatasetDCEListItemView,
-        /** sub view class used for nested collections */
-        NestedDCDCEViewClass: DC_LI.NestedDCDCEListItemView,
+
         /** key of attribute in model to assign to this.collection */
         modelCollectionKey: "elements",
 
@@ -44,9 +43,16 @@ var CollectionView = _super.extend(
             this.downloadUrl = `${Galaxy.root}api/dataset_collections/${this.model.attributes.id}/download`;
         },
 
+        getNestedDCDCEViewClass: function() {
+            return DC_LI.NestedDCDCEListItemView.extend({
+                foldoutPanelClass: CollectionView
+            });
+        },
+
         _queueNewRender: function($newRender, speed) {
             speed = speed === undefined ? this.fxSpeed : speed;
             var panel = this;
+            this.handleWarning($newRender);
             panel.log("_queueNewRender:", $newRender, speed);
 
             // TODO: jquery@1.12 doesn't change display when the elem has display: flex
@@ -56,6 +62,16 @@ var CollectionView = _super.extend(
             panel.trigger("rendered", panel);
         },
 
+        handleWarning: function($newRender) {
+            var viewLength = this.views.length;
+            var elementCount = this.model.get("element_count");
+            if (elementCount && elementCount !== viewLength) {
+                var warning = _l(`displaying only ${viewLength} of ${elementCount} items`);
+                var $warns = $newRender.find(".elements-warning");
+                $warns.html(`<div class="warningmessagesmall">${warning}</div>`);
+            }
+        },
+
         // ------------------------------------------------------------------------ sub-views
         /** In this override, use model.getVisibleContents */
         _filterCollection: function() {
@@ -71,7 +87,7 @@ var CollectionView = _super.extend(
                 case "hda":
                     return this.DatasetDCEViewClass;
                 case "dataset_collection":
-                    return this.NestedDCDCEViewClass;
+                    return this.getNestedDCDCEViewClass();
             }
             throw new TypeError("Unknown element type:", model.get("element_type"));
         },
@@ -144,116 +160,74 @@ var CollectionView = _super.extend(
 
 //------------------------------------------------------------------------------ TEMPLATES
 CollectionView.prototype.templates = (() => {
-    var controlsTemplate = BASE_MVC.wrapTemplate(
-        [
-            '<div class="controls">',
-            '<div class="navigation">',
-            '<a class="back" href="javascript:void(0)">',
-            '<span class="fa fa-icon fa-angle-left"></span>',
-            _l("Back to "),
-            "<%- view.parentName %>",
-            "</a>",
-            "</div>",
-
-            '<div class="title">',
-            '<div class="name"><%- collection.name || collection.element_identifier %></div>',
-            '<div class="subtitle">',
-            '<% if( collection.collection_type === "list" ){ %>',
-            _l("a list of datasets"),
-            '<% } else if( collection.collection_type === "paired" ){ %>',
-            _l("a pair of datasets"),
-            '<% } else if( collection.collection_type === "list:paired" ){ %>',
-            _l("a list of paired datasets"),
-            '<% } else if( collection.collection_type === "list:list" ){ %>',
-            _l("a list of dataset lists"),
-            "<% } %>",
-            "</div>",
-            "</div>",
-
-            '<div class="tags-display"></div>',
-
-            '<div class="actions">',
-            '<a class="download-btn icon-btn" ',
-            'href="<%- view.downloadUrl %>',
-            '" title="" download="" data-original-title="Download Collection">',
-            '<span class="fa fa-floppy-o"></span>',
-            "</a>",
-            "</div>",
-            "</div>"
-        ],
-        "collection"
-    );
+    var controlsTemplate = (collection, view) => {
+        var subtitle = collectionDescription(view.model);
+        return `
+        <div class="controls">
+            <div class="navigation">
+            <a class="back" href="javascript:void(0)">
+                <span class="fa fa-icon fa-angle-left"></span>
+                ${_l("Back to ")}
+                ${_.escape(view.parentName)}
+            </a>
+            </div>
+            <div class="title">
+                <div class="name">${_.escape(collection.name) || _.escape(collection.element_identifier)}</div>
+                <div class="subtitle">
+                    ${subtitle}
+                </div>
+            </div>
+            <div class="elements-warning">
+            </div>
+            <div class="tags-display"></div>
+            <div class="actions">
+                <a class="download-btn icon-btn" href="${view.downloadUrl}"
+                   title="" download="" data-original-title="Download Collection">
+                   <span class="fa fa-floppy-o"></span>
+                </a>
+            </div>
+        </div>`;
+    };
 
     return _.extend(_.clone(_super.prototype.templates), {
         controls: controlsTemplate
     });
 })();
 
-// =============================================================================
-/** @class non-editable, read-only View/Controller for a dataset collection. */
-var ListCollectionView = CollectionView.extend(
-    /** @lends ListCollectionView.prototype */ {
-        //TODO: not strictly needed - due to switch in CollectionView._getContentClass
-        /** sub view class used for datasets */
-        DatasetDCEViewClass: DC_LI.DatasetDCEListItemView,
+function collectionTypeDescription(collection) {
+    var collectionType = collection.get("collection_type");
+    var collectionTypeDescription;
+    if (collectionType == "list") {
+        collectionTypeDescription = _l("list");
+    } else if (collectionType == "paired") {
+        collectionTypeDescription = _l("dataset pair");
+    } else if (collectionType == "list:paired") {
+        collectionTypeDescription = _l("list of pairs");
+    } else {
+        collectionTypeDescription = _l("nested list");
+    }
+    return collectionTypeDescription;
+}
 
-        // ........................................................................ misc
-        /** string rep */
-        toString: function() {
-            return `ListCollectionView(${this.model ? this.model.get("name") : ""})`;
+function collectionDescription(collection) {
+    var elementCount = collection.get("element_count");
+
+    var itemsDescription = `a ${collectionTypeDescription(collection)}`;
+    if (elementCount) {
+        var countDescription;
+        if (elementCount == 1) {
+            countDescription = "with 1 item";
+        } else if (elementCount) {
+            countDescription = `with ${elementCount} items`;
         }
+        itemsDescription = `${itemsDescription} ${_l(countDescription)}`;
     }
-);
-
-// =============================================================================
-/** @class non-editable, read-only View/Controller for a dataset collection. */
-var PairCollectionView = ListCollectionView.extend(
-    /** @lends PairCollectionView.prototype */ {
-        // ........................................................................ misc
-        /** string rep */
-        toString: function() {
-            return `PairCollectionView(${this.model ? this.model.get("name") : ""})`;
-        }
-    }
-);
-
-// =============================================================================
-/** @class non-editable, read-only View/Controller for a dataset collection. */
-var ListOfPairsCollectionView = CollectionView.extend(
-    /** @lends ListOfPairsCollectionView.prototype */ {
-        //TODO: not strictly needed - due to switch in CollectionView._getContentClass
-        /** sub view class used for nested collections */
-        NestedDCDCEViewClass: DC_LI.NestedDCDCEListItemView.extend({
-            foldoutPanelClass: PairCollectionView
-        }),
-
-        // ........................................................................ misc
-        /** string rep */
-        toString: function() {
-            return `ListOfPairsCollectionView(${this.model ? this.model.get("name") : ""})`;
-        }
-    }
-);
-
-// =============================================================================
-/** @class non-editable, read-only View/Controller for a list of lists dataset collection. */
-var ListOfListsCollectionView = CollectionView.extend({
-    /** sub view class used for nested collections */
-    NestedDCDCEViewClass: DC_LI.NestedDCDCEListItemView.extend({
-        foldoutPanelClass: PairCollectionView
-    }),
-
-    /** string rep */
-    toString: function() {
-        return `ListOfListsCollectionView(${this.model ? this.model.get("name") : ""})`;
-    }
-});
+    return itemsDescription;
+}
 
 //==============================================================================
 export default {
-    CollectionView: CollectionView,
-    ListCollectionView: ListCollectionView,
-    PairCollectionView: PairCollectionView,
-    ListOfPairsCollectionView: ListOfPairsCollectionView,
-    ListOfListsCollectionView: ListOfListsCollectionView
+    collectionTypeDescription: collectionTypeDescription,
+    collectionDescription: collectionDescription,
+    CollectionView: CollectionView
 };

client/galaxy/scripts/mvc/collection/list-collection-creator.js

@@ -173,7 +173,7 @@ var ListCollectionCreator = Backbone.View.extend(BASE_MVC.LoggableMixin)
         /** the class used to display individual elements */
         elementViewClass: DatasetCollectionElementView,
         /** the class this creator will create and save */
-        collectionClass: HDCA.HistoryListDatasetCollection,
+        collectionClass: HDCA.HistoryDatasetCollection,
         className: "list-collection-creator collection-creator flex-row-container",
 
         /** minimum number of valid elements to start with in order to build a collection of this type */

client/galaxy/scripts/mvc/collection/list-of-pairs-collection-creator.js

@@ -623,7 +623,6 @@ var PairedCollectionCreator = Backbone.View.extend(baseMVC.LoggableMixin)
 
             var url = `${Galaxy.root}api/histories/${this.historyId}/contents/dataset_collections`;
 
-            //TODO: use ListPairedCollection.create()
             var ajaxData = {
                 type: "dataset_collection",
                 collection_type: "list:paired",

client/galaxy/scripts/mvc/collection/pair-collection-creator.js

@@ -61,7 +61,7 @@ var PairCollectionCreator = _super.extend({
     /** the class used to display individual elements */
     elementViewClass: PairedDatasetCollectionElementView,
     /** the class this creator will create and save */
-    collectionClass: HDCA.HistoryPairDatasetCollection,
+    collectionClass: HDCA.HistoryDatasetCollection,
     className: "pair-collection-creator collection-creator flex-row-container",
 
     /** override to no-op */

client/galaxy/scripts/mvc/form/form-wrapper.js

@@ -66,7 +66,7 @@ var View = Backbone.View.extend({
                         message: response.message,
                         status: "success",
                         persistent: false
-                    }
+                    };
                 }
                 if (self.redirect) {
                     window.location = `${Galaxy.root + self.redirect}?${$.param(params)}`;

client/galaxy/scripts/mvc/history/hdca-li-edit.js

@@ -14,18 +14,7 @@ var HDCAListItemEdit = _super.extend(
 
         /** Override to return editable versions of the collection panels */
         _getFoldoutPanelClass: function() {
-            var collectionType = this.model.get("collection_type");
-            switch (collectionType) {
-                case "list":
-                    return DC_VIEW_EDIT.ListCollectionViewEdit;
-                case "paired":
-                    return DC_VIEW_EDIT.PairCollectionViewEdit;
-                case "list:paired":
-                    return DC_VIEW_EDIT.ListOfPairsCollectionViewEdit;
-                case "list:list":
-                    return DC_VIEW_EDIT.ListOfListsCollectionViewEdit;
-            }
-            throw new TypeError(`Unknown collection_type: ${collectionType}`);
+            return DC_VIEW_EDIT.CollectionViewEdit;
         },
 
         // ......................................................................... delete

client/galaxy/scripts/mvc/history/hdca-li.js

@@ -16,40 +16,102 @@ var HDCAListItemView = _super.extend(
         /** event listeners */
         _setUpListeners: function() {
             _super.prototype._setUpListeners.call(this);
+            var renderListen = (model, options) => {
+                this.render();
+            };
+            if (this.model.jobStatesSummary) {
+                this.listenTo(this.model.jobStatesSummary, "change", renderListen);
+            }
             this.listenTo(this.model, {
-                "change:tags change:populated change:visible": function(model, options) {
-                    this.render();
-                }
+                "change:tags change:visible change:state": renderListen
             });
         },
 
         /** Override to provide the proper collections panels as the foldout */
         _getFoldoutPanelClass: function() {
-            var collectionType = this.model.get("collection_type");
-            switch (collectionType) {
-                case "list":
-                    return DC_VIEW.ListCollectionView;
-                case "paired":
-                    return DC_VIEW.PairCollectionView;
-                case "list:paired":
-                    return DC_VIEW.ListOfPairsCollectionView;
-                case "list:list":
-                    return DC_VIEW.ListOfListsCollectionView;
-            }
-            throw new TypeError(`Unknown collection_type: ${collectionType}`);
+            return DC_VIEW.CollectionView;
         },
 
         /** In this override, add the state as a class for use with state-based CSS */
         _swapNewRender: function($newRender) {
             _super.prototype._swapNewRender.call(this, $newRender);
             //TODO: model currently has no state
-            var state = !this.model.get("populated") ? STATES.RUNNING : STATES.OK;
-            //if( this.model.has( 'state' ) ){
+            var state;
+            var jobStatesSummary = this.model.jobStatesSummary;
+            if (jobStatesSummary) {
+                if (jobStatesSummary.new()) {
+                    state = "loading";
+                } else if (jobStatesSummary.errored()) {
+                    state = "error";
+                } else if (jobStatesSummary.terminal()) {
+                    state = "ok";
+                } else if (jobStatesSummary.running()) {
+                    state = "running";
+                } else {
+                    state = "queued";
+                }
+            } else if (this.model.get("job_source_id")) {
+                // Initial rendering - polling will fill in more details in a bit.
+                state = "loading";
+            } else {
+                state = this.model.get("populated_state") ? STATES.OK : STATES.RUNNING;
+            }
             this.$el.addClass(`state-${state}`);
-            //}
+            var stateDescription = this.stateDescription();
+            this.$(".state-description").html(stateDescription);
             return this.$el;
         },
 
+        stateDescription: function() {
+            var collection = this.model;
+            var elementCount = collection.get("element_count");
+            var jobStateSource = collection.get("job_source_type");
+            var collectionType = this.model.get("collection_type");
+            var collectionTypeDescription = DC_VIEW.collectionTypeDescription(collection);
+            var simpleDescription = DC_VIEW.collectionDescription(collection);
+            var jobStatesSummary = collection.jobStatesSummary;
+            if (!jobStateSource || jobStateSource == "Job") {
+                return simpleDescription;
+            } else if (!jobStatesSummary || !jobStatesSummary.hasDetails()) {
+                return `
+                    <div class="progress state-progress">
+                        <span class="note">Loading job data for ${
+                            collectionTypeDescription
+                        }.<span class="blinking">..</span></span>
+                        <div class="progress-bar info" style="width:100%">
+                    </div>`;
+            } else {
+                var isNew = jobStatesSummary.new();
+                var jobCount = isNew ? null : jobStatesSummary.jobCount();
+                if (isNew) {
+                    return `
+                        <div class="progress state-progress">
+                            <span class="note">Creating jobs.<span class="blinking">..</span></span>
+                            <div class="progress-bar info" style="width:100%">
+                        </div>`;
+                } else if (jobStatesSummary.errored()) {
+                    var errorCount = jobStatesSummary.numInError();
+                    return `a ${collectionTypeDescription} with ${errorCount} / ${jobCount} jobs in error`;
+                } else if (jobStatesSummary.terminal()) {
+                    return simpleDescription;
+                } else {
+                    var running = jobStatesSummary.states()["running"] || 0;
+                    var ok = jobStatesSummary.states()["ok"] || 0;
+                    var okPercent = ok / (jobCount * 1.0);
+                    var runningPercent = running / (jobCount * 1.0);
+                    var otherPercent = 1.0 - okPercent - runningPercent;
+                    var jobsStr = jobCount && jobCount > 1 ? `${jobCount} jobs` : `a job`;
+                    return `
+                        <div class="progress state-progress">
+                            <span class="note">${jobsStr} generating a ${collectionTypeDescription}</span>
+                            <div class="progress-bar ok" style="width:${okPercent * 100.0}%"></div>
+                            <div class="progress-bar running" style="width:${runningPercent * 100.0}%"></div>
+                            <div class="progress-bar new" style="width:${otherPercent * 100.0}%">
+                        </div>`;
+                }
+            }
+        },
+
         // ......................................................................... misc
         /** String representation */
         toString: function() {
@@ -69,7 +131,6 @@ HDCAListItemView.prototype.templates = (() => {
         }
     });
 
-    // could steal this from hda-base (or use mixed content)
     var titleBarTemplate = collection => `
         <div class="title-bar clear" tabindex="0">
             <span class="state-icon"></span>
@@ -77,7 +138,8 @@ HDCAListItemView.prototype.templates = (() => {
                 <span class="hid">${collection.hid}</span>
                 <span class="name">${_.escape(collection.name)}</span>
             </div>
-            <div class="subtitle"></div>
+            <div class="state-description">
+            </div>
             ${HISTORY_ITEM_LI.nametagTemplate(collection)}
         </div>
     `;

client/galaxy/scripts/mvc/history/hdca-model.js

@@ -6,115 +6,41 @@ import _l from "utils/localization";
 
 Models for DatasetCollections contained within a history.
 
-TODO:
-    these might be compactable to one class if some duplication with
-    collection-model is used.
-
 ==============================================================================*/
 var hcontentMixin = HISTORY_CONTENT.HistoryContentMixin;
 
-var ListDC = DC_MODEL.ListDatasetCollection;
-var PairDC = DC_MODEL.PairDatasetCollection;
-var ListPairedDC = DC_MODEL.ListPairedDatasetCollection;
-var ListOfListsDC = DC_MODEL.ListOfListsDatasetCollection;
-
-//==============================================================================
-/** Override to post to contents route w/o id. */
-function buildHDCASave(_super) {
-    return function _save(attributes, options) {
-        if (this.isNew()) {
-            options = options || {};
-            options.url = `${this.urlRoot + this.get("history_id")}/contents`;
-            attributes = attributes || {};
-            attributes.type = "dataset_collection";
-        }
-        return _super.call(this, attributes, options);
-    };
-}
+var DC = DC_MODEL.DatasetCollection;
 
 //==============================================================================
 /** @class Backbone model for List Dataset Collection within a History.
  */
-var HistoryListDatasetCollection = ListDC.extend(hcontentMixin).extend(
-    /** @lends HistoryListDatasetCollection.prototype */ {
-        defaults: _.extend(_.clone(ListDC.prototype.defaults), {
+var HistoryDatasetCollection = DC.extend(hcontentMixin).extend(
+    /** @lends HistoryDatasetCollection.prototype */ {
+        defaults: _.extend(_.clone(DC.prototype.defaults), {
             history_content_type: "dataset_collection",
-            collection_type: "list",
             model_class: "HistoryDatasetCollectionAssociation"
         }),
 
+        //==============================================================================
         /** Override to post to contents route w/o id. */
-        save: buildHDCASave(ListDC.prototype.save),
+        save: function(attributes, options) {
+            if (this.isNew()) {
+                options = options || {};
+                options.url = `${this.urlRoot + this.get("history_id")}/contents`;
+                attributes = attributes || {};
+                attributes.type = "dataset_collection";
+            }
+            return DC.prototype.save.call(this, attributes, options);
+        },
 
         /** String representation. */
         toString: function() {
-            return `History${ListDC.prototype.toString.call(this)}`;
+            return `History${DC.prototype.toString.call(this)}`;
         }
     }
 );
 
-//==============================================================================
-/** @class Backbone model for Pair Dataset Collection within a History.
- *  @constructs
- */
-var HistoryPairDatasetCollection = PairDC.extend(hcontentMixin).extend(
-    /** @lends HistoryPairDatasetCollection.prototype */ {
-        defaults: _.extend(_.clone(PairDC.prototype.defaults), {
-            history_content_type: "dataset_collection",
-            collection_type: "paired",
-            model_class: "HistoryDatasetCollectionAssociation"
-        }),
-
-        /** Override to post to contents route w/o id. */
-        save: buildHDCASave(PairDC.prototype.save),
-
-        /** String representation. */
-        toString: function() {
-            return `History${PairDC.prototype.toString.call(this)}`;
-        }
-    }
-);
-
-//==============================================================================
-/** @class Backbone model for List of Pairs Dataset Collection within a History. */
-var HistoryListPairedDatasetCollection = ListPairedDC.extend(hcontentMixin).extend({
-    defaults: _.extend(_.clone(ListPairedDC.prototype.defaults), {
-        history_content_type: "dataset_collection",
-        collection_type: "list:paired",
-        model_class: "HistoryDatasetCollectionAssociation"
-    }),
-
-    /** Override to post to contents route w/o id. */
-    save: buildHDCASave(ListPairedDC.prototype.save),
-
-    /** String representation. */
-    toString: function() {
-        return `History${ListPairedDC.prototype.toString.call(this)}`;
-    }
-});
-
-//==============================================================================
-/** @class Backbone model for List of Lists Dataset Collection within a History. */
-var HistoryListOfListsDatasetCollection = ListOfListsDC.extend(hcontentMixin).extend({
-    defaults: _.extend(_.clone(ListOfListsDC.prototype.defaults), {
-        history_content_type: "dataset_collection",
-        collection_type: "list:list",
-        model_class: "HistoryDatasetCollectionAssociation"
-    }),
-
-    /** Override to post to contents route w/o id. */
-    save: buildHDCASave(ListOfListsDC.prototype.save),
-
-    /** String representation. */
-    toString: function() {
-        return `HistoryListOfListsDatasetCollection(${this.get("name")})`;
-    }
-});
-
 //==============================================================================
 export default {
-    HistoryListDatasetCollection: HistoryListDatasetCollection,
-    HistoryPairDatasetCollection: HistoryPairDatasetCollection,
-    HistoryListPairedDatasetCollection: HistoryListPairedDatasetCollection,
-    HistoryListOfListsDatasetCollection: HistoryListOfListsDatasetCollection
+    HistoryDatasetCollection: HistoryDatasetCollection
 };

client/galaxy/scripts/mvc/history/history-content-model.js

@@ -2,6 +2,11 @@ import STATES from "mvc/dataset/states";
 import BASE_MVC from "mvc/base-mvc";
 import _l from "utils/localization";
 
+var collectionFuzzyCountDefault = 1000;
+try {
+    collectionFuzzyCountDefault = localStorage.getItem("collectionFuzzyCountDefault") || collectionFuzzyCountDefault;
+} catch (err) {}
+
 //==============================================================================
 /** @class Mixin for HistoryContents content (HDAs, HDCAs).
  */
@@ -58,6 +63,11 @@ var HistoryContentMixin = {
         var historyId = this.get("history_id");
         var historyContentId = this.get("id");
         var url = `${this.urlRoot}${historyId}/contents/${historyContentType}s/${historyContentId}`;
+        if (historyContentType == "dataset_collection") {
+            // Don't fetch whole collection - just enought to render outline. Backbone will
+            // make a detailed request if any datasets are expanded beyond that point.
+            url = `${url}?view=element-reference&fuzzy_count=${collectionFuzzyCountDefault}`;
+        }
         return url;
     },
 

client/galaxy/scripts/mvc/history/history-contents.js

@@ -2,9 +2,15 @@ import CONTROLLED_FETCH_COLLECTION from "mvc/base/controlled-fetch-collection";
 import HDA_MODEL from "mvc/history/hda-model";
 import HDCA_MODEL from "mvc/history/hdca-model";
 import HISTORY_PREFS from "mvc/history/history-preferences";
+import JOB_STATES_MODEL from "mvc/history/job-states-model";
 import BASE_MVC from "mvc/base-mvc";
 import AJAX_QUEUE from "utils/ajax-queue";
 
+var limitPerPageDefault = 500;
+try {
+    limitPerPageDefault = localStorage.getItem("historyContentsLimitPerPageDefault") || limitPerPageDefault;
+} catch (err) {}
+
 //==============================================================================
 var _super = CONTROLLED_FETCH_COLLECTION.PaginatedCollection;
 /** @class Backbone collection for history content.
@@ -19,10 +25,10 @@ var HistoryContents = _super.extend(BASE_MVC.LoggableMixin).extend({
     _logNamespace: "history",
 
     // ........................................................................ set up
-    limitPerPage: 500,
+    limitPerPage: limitPerPageDefault,
 
     /** @type {Integer} how many contents per call to fetch when using progressivelyFetchDetails */
-    limitPerProgressiveFetch: 500,
+    limitPerProgressiveFetch: limitPerPageDefault,
 
     /** @type {String} order used here and when fetching from server */
     order: "hid",
@@ -37,6 +43,10 @@ var HistoryContents = _super.extend(BASE_MVC.LoggableMixin).extend({
 
     /** Set up */
     initialize: function(models, options) {
+        this.on({
+            "sync add": this.trackJobStates
+        });
+
         options = options || {};
         _super.prototype.initialize.call(this, models, options);
 
@@ -53,46 +63,67 @@ var HistoryContents = _super.extend(BASE_MVC.LoggableMixin).extend({
         this.model.prototype.idAttribute = "type_id";
     },
 
+    trackJobStates: function() {
+        this.each(historyContent => {
+            if (historyContent.has("job_states_summary")) {
+                return;
+            }
+
+            if (historyContent.attributes.history_content_type === "dataset_collection") {
+                var jobSourceType = historyContent.attributes.job_source_type;
+                var jobSourceId = historyContent.attributes.job_source_id;
+                if (jobSourceType) {
+                    this.jobStateSummariesCollection.add({
+                        id: jobSourceId,
+                        model: jobSourceType,
+                        history_id: this.history_id,
+                        collection_id: historyContent.attributes.id
+                    });
+                    var jobStatesSummary = this.jobStateSummariesCollection.get(jobSourceId);
+                    historyContent.jobStatesSummary = jobStatesSummary;
+                }
+            }
+        });
+    },
+
     // ........................................................................ composite collection
     /** since history content is a mix, override model fn into a factory, creating based on history_content_type */
     model: function(attrs, options) {
         if (attrs.history_content_type === "dataset") {
             return new HDA_MODEL.HistoryDatasetAssociation(attrs, options);
         } else if (attrs.history_content_type === "dataset_collection") {
-            switch (attrs.collection_type) {
-                case "list":
-                    return new HDCA_MODEL.HistoryListDatasetCollection(attrs, options);
-                case "paired":
-                    return new HDCA_MODEL.HistoryPairDatasetCollection(attrs, options);
-                case "list:paired":
-                    return new HDCA_MODEL.HistoryListPairedDatasetCollection(attrs, options);
-                case "list:list":
-                    return new HDCA_MODEL.HistoryListOfListsDatasetCollection(attrs, options);
-            }
-            // This is a hack inside a hack:
-            // Raise a plain object with validationError to fake a model.validationError
-            // (since we don't have a model to use validate with)
-            // (the outer hack being the mixed content/model function in this collection)
-            var msg = `Unknown collection_type: ${attrs.collection_type}`;
-            console.warn(msg, attrs);
-            return { validationError: msg };
+            return new HDCA_MODEL.HistoryDatasetCollection(attrs, options);
+        } else {
+            return {
+                validationError: `Unknown history_content_type: ${attrs.history_content_type}`
+            };
+        }
+    },
+
+    stopPolling: function() {
+        if (this.jobStateSummariesCollection) {
+            this.jobStateSummariesCollection.active = false;
+            this.jobStateSummariesCollection.clearUpdateTimeout();
         }
-        return {
-            validationError: `Unknown history_content_type: ${attrs.history_content_type}`
-        };
     },
 
     setHistoryId: function(newId) {
+        this.stopPolling();
         this.historyId = newId;
-        this._setUpWebStorage();
+        if (newId) {
+            // If actually reflecting a history - setup storage and monitor jobs.
+
+            this._setUpWebStorage();
+
+            this.jobStateSummariesCollection = new JOB_STATES_MODEL.JobStatesSummaryCollection();
+            this.jobStateSummariesCollection.historyId = newId;
+            this.jobStateSummariesCollection.monitor();
+        }
     },
 
     /** Set up client side storage. Currently PersistanStorage keyed under 'history:<id>' */
     _setUpWebStorage: function(initialSettings) {
         // TODO: use initialSettings
-        if (!this.historyId) {
-            return;
-        }
         this.storage = new HISTORY_PREFS.HistoryPrefs({
             id: HISTORY_PREFS.HistoryPrefs.historyStorageKey(this.historyId)
         });
@@ -303,17 +334,6 @@ var HistoryContents = _super.extend(BASE_MVC.LoggableMixin).extend({
         return this.fetch(options);
     },
 
-    /** specialty fetch method for retrieving the element_counts of all hdcas in the history */
-    fetchCollectionCounts: function(options) {
-        options = options || {};
-        options.keys = ["type_id", "element_count"].join(",");
-        options.filters = _.extend(options.filters || {}, {
-            history_content_type: "dataset_collection"
-        });
-        options.remove = false;
-        return this.fetch(options);
-    },
-
     // ............. quasi-batch ops
     // TODO: to batch
     /** helper that fetches using filterParams then calls save on each fetched using updateWhat as the save params */

client/galaxy/scripts/mvc/history/history-model.js

@@ -237,6 +237,13 @@ var History = Backbone.Model.extend(BASE_MVC.LoggableMixin).extend(
                 }
             },
 
+            stopPolling: function() {
+                this.clearUpdateTimeout();
+                if (this.contents) {
+                    this.contents.stopPolling();
+                }
+            },
+
             // ........................................................................ ajax
             /** override to use actual Dates objects for create/update times */
             parse: function(response, options) {

client/galaxy/scripts/mvc/history/history-view.js

@@ -47,9 +47,6 @@ var HistoryView = _super.extend(
         /** string used for search placeholder */
         searchPlaceholder: _l("search datasets"),
 
-        /** @type {Number} ms to wait after history load to fetch/decorate hdcas with element_count */
-        FETCH_COLLECTION_COUNTS_DELAY: 2000,
-
         // ......................................................................... SET UP
         /** Set up the view, bind listeners.
          *  @param {Object} attributes optional settings for the panel
@@ -60,9 +57,6 @@ var HistoryView = _super.extend(
             // control contents/behavior based on where (and in what context) the panel is being used
             /** where should pages from links be displayed? (default to new tab/window) */
             this.linkTarget = attributes.linkTarget || "_blank";
-
-            /** timeout id for detailed fetch of collection counts, etc... */
-            this.detailedFetchTimeoutId = null;
         },
 
         /** create and return a collection for when none is initially passed */
@@ -77,20 +71,11 @@ var HistoryView = _super.extend(
         freeModel: function() {
             _super.prototype.freeModel.call(this);
             if (this.model) {
-                this.model.clearUpdateTimeout();
+                this.model.stopPolling();
             }
-            this._clearDetailedFetchTimeout();
             return this;
         },
 
-        /** clear the timeout and the cached timeout id */
-        _clearDetailedFetchTimeout: function() {
-            if (this.detailedFetchTimeoutId) {
-                clearTimeout(this.detailedFetchTimeoutId);
-                this.detailedFetchTimeoutId = null;
-            }
-        },
-
         /** create any event listeners for the panel
          *  @fires: rendered:initial    on the first render
          *  @fires: empty-history       when switching to a history with no contents or creating a new history
@@ -101,13 +86,6 @@ var HistoryView = _super.extend(
                 error: function(model, xhr, options, msg, details) {
                     this.errorHandler(model, xhr, options, msg, details);
                 },
-                "loading-done": () => {
-                    // after the initial load, decorate with more time consuming fields (like HDCA element_counts)
-                    this.detailedFetchTimeoutId = _.delay(() => {
-                        this.detailedFetchTimeoutId = null;
-                        this.model.contents.fetchCollectionCounts();
-                    }, this.FETCH_COLLECTION_COUNTS_DELAY);
-                },
                 "views:ready view:attached view:removed": function(view) {
                     this._renderSelectButton();
                 },
@@ -378,17 +356,17 @@ var HistoryView = _super.extend(
         }),
 
         _clickPrevPage: function(ev) {
-            this.model.clearUpdateTimeout();
+            this.model.stopPolling();
             this.model.contents.fetchPrevPage();
         },
 
         _clickNextPage: function(ev) {
-            this.model.clearUpdateTimeout();
+            this.model.stopPolling();
             this.model.contents.fetchNextPage();
         },
 
         _changePageSelect: function(ev) {
-            this.model.clearUpdateTimeout();
+            this.model.stopPolling();
             var page = $(ev.currentTarget).val();
             this.model.contents.fetchPage(page);
         },

client/galaxy/scripts/mvc/history/job-states-model.js

@@ -0,0 +1,168 @@
+import * as Backbone from "libs/backbone";
+import AJAX_QUEUE from "utils/ajax-queue";
+
+/** ms between fetches when checking running jobs/datasets for updates */
+var UPDATE_DELAY = 2000;
+var NON_TERMINAL_STATES = ["new", "queued", "running"];
+var ERROR_STATES = ["error", "deleted"];
+/** Fetch state on add or just wait for polling to start. */
+var FETCH_STATE_ON_ADD = false;
+var BATCH_FETCH_STATE = true;
+
+var JobStatesSummary = Backbone.Model.extend({
+    url: function() {
+        return `${Galaxy.root}api/histories/${this.attributes.history_id}/contents/dataset_collections/${
+            this.attributes.collection_id
+        }/jobs_summary`;
+    },
+
+    hasDetails: function() {
+        return this.has("populated_state");
+    },
+
+    new: function() {
+        return !this.hasDetails() || this.get("populated_state") == "new";
+    },
+
+    errored: function() {
+        return this.get("populated_state") === "error" || this.anyWithStates(ERROR_STATES);
+    },
+
+    states: function() {
+        return this.get("states") || {};
+    },
+
+    anyWithState: function(queryState) {
+        return (this.states()[queryState] || 0) > 0;
+    },
+
+    anyWithStates: function(queryStates) {
+        var states = this.states();
+        for (var index in queryStates) {
+            if ((states[queryStates[index]] || 0) > 0) {
+                return true;
+            }
+        }
+        return false;
+    },
+
+    numWithStates: function(queryStates) {
+        var states = this.states();
+        var count = 0;
+        for (var index in queryStates) {
+            count += states[queryStates[index]] || 0;
+        }
+        return count;
+    },
+
+    numInError: function() {
+        return this.numWithStates(ERROR_STATES);
+    },
+
+    running: function() {
+        return this.anyWithState("running");
+    },
+
+    terminal: function() {
+        if (this.new()) {
+            return false;
+        } else {
+            var anyNonTerminal = this.anyWithStates(NON_TERMINAL_STATES);
+            return !anyNonTerminal;
+        }
+    },
+
+    jobCount: function() {
+        var states = this.states();
+        var count = 0;
+        for (var index in states) {
+            count += states[index];
+        }
+        return count;
+    },
+
+    toString: function() {
+        return `JobStatesSummary(id=${this.get("id")})`;
+    }
+});
+
+var JobStatesSummaryCollection = Backbone.Collection.extend({
+    model: JobStatesSummary,
+
+    initialize: function() {
+        if (FETCH_STATE_ON_ADD) {
+            this.on({
+                add: model => model.fetch()
+            });
+        }
+
+        /** cached timeout id for the dataset updater */
+        this.updateTimeoutId = null;
+        // this.checkForUpdates();
+        this.active = true;
+    },
+
+    url: function() {
+        var nonTerminalModels = this.models.filter(model => {
+            return !model.terminal();
+        });
+        var ids = nonTerminalModels
+            .map(summary => {
+                return summary.get("id");
+            })
+            .join(",");
+        var types = nonTerminalModels
+            .map(summary => {
+                return summary.get("model");
+            })
+            .join(",");
+        return `${Galaxy.root}api/histories/${this.historyId}/jobs_summary?ids=${ids}&types=${types}`;
+    },
+
+    monitor: function() {
+        this.clearUpdateTimeout();
+        if (!this.active) {
+            return;
+        }
+
+        var _delayThenMonitorAgain = () => {
+            this.updateTimeoutId = setTimeout(() => {
+                this.monitor();
+            }, UPDATE_DELAY);
+        };
+
+        var nonTerminalModels = this.models.filter(model => {
+            return !model.terminal();
+        });
+
+        if (nonTerminalModels.length > 0 && !BATCH_FETCH_STATE) {
+            // Allow models to fetch their own details.
+            var updateFunctions = nonTerminalModels.map(summary => {
+                return () => {
+                    return summary.fetch();
+                };
+            });
+
+            return new AJAX_QUEUE.AjaxQueue(updateFunctions).done(_delayThenMonitorAgain);
+        } else if (nonTerminalModels.length > 0) {
+            // Batch fetch updated state...
+            this.fetch({ remove: false }).done(_delayThenMonitorAgain);
+        } else {
+            _delayThenMonitorAgain();
+        }
+    },
+
+    /** clear the timeout and the cached timeout id */
+    clearUpdateTimeout: function() {
+        if (this.updateTimeoutId) {
+            clearTimeout(this.updateTimeoutId);
+            this.updateTimeoutId = null;
+        }
+    },
+
+    toString: function() {
+        return `JobStatesSummaryCollection()`;
+    }
+});
+
+export default { JobStatesSummary, JobStatesSummaryCollection, FETCH_STATE_ON_ADD };

client/galaxy/scripts/mvc/history/multi-panel.js

@@ -1,6 +1,7 @@
 import _l from "utils/localization";
 import HISTORY_MODEL from "mvc/history/history-model";
 import HISTORY_VIEW_EDIT from "mvc/history/history-view-edit";
+import JOB_STATES_MODEL from "mvc/history/job-states-model";
 import historyCopyDialog from "mvc/history/copy-dialog";
 import ERROR_MODAL from "mvc/ui/error-modal";
 import baseMVC from "mvc/base-mvc";
@@ -733,9 +734,16 @@ var MultiPanelColumns = Backbone.View.extend(baseMVC.LoggableMixin).extend({
             this.hdaQueue.add({
                 name: column.model.id,
                 fn: function() {
-                    return contents.fetchCurrentPage(fetchOptions).done(() => {
-                        column.panel.renderItems();
-                    });
+                    return contents
+                        .fetchCurrentPage(fetchOptions)
+                        .done(() => {
+                            column.panel.renderItems();
+                        })
+                        .done(() => {
+                            if (!JOB_STATES_MODEL.FETCH_STATE_ON_ADD) {
+                                contents.jobStateSummariesCollection.fetch();
+                            }
+                        });
                 }
             });
             // the queue is re-used, so if it's not processing requests - start it again

client/galaxy/scripts/mvc/tool/tool-form-base.js

@@ -5,7 +5,7 @@ import Utils from "utils/utils";
 import Deferred from "utils/deferred";
 import Ui from "mvc/ui/ui-misc";
 import FormBase from "mvc/form/form-view";
-import Citations from "components/citations.vue";
+import Citations from "components/Citations.vue";
 import Vue from "libs/vue";
 export default FormBase.extend({
     initialize: function(options) {

client/galaxy/scripts/qunit/test.js

@@ -1,3 +1,21 @@
 // Load all qunit tests into a single bundle.
-var testsContext = require.context(".", true, /_tests$/);
-testsContext.keys().forEach(testsContext);
+//var testsContext = require.context(".", true, /_tests$/);
+//testsContext.keys().forEach(testsContext);
+
+import gab from "qunit/tests/galaxy_app_base_tests";
+import ml from "qunit/tests/metrics_logger_tests";
+// form_tests seems to need to come before something - maybe multiple things
+import form from "qunit/tests/form_tests";
+import l from "qunit/tests/list_of_pairs_collection_creator_tests";
+import upd from "qunit/tests/upload_dialog_tests";
+import we from "qunit/tests/workflow_editor_tests";
+import mast from "qunit/tests/masthead_tests";
+import graph from "qunit/tests/graph_tests";
+import jd from "qunit/tests/job_dag_tests";
+import hc from "qunit/tests/history_contents_model_tests";
+import hda from "qunit/tests/hda_base_tests";
+import modal from "qunit/tests/modal_tests";
+import page from "qunit/tests/page_tests";
+import popover from "qunit/tests/popover_tests";
+import utils from "qunit/tests/utils_tests";
+import ui from "qunit/tests/ui_tests";

client/galaxy/scripts/qunit/tests/form_tests.js

@@ -1,182 +1,167 @@
 /* global define */
-define(
-    [
-        "qunit/test-app",
-        "sinon",
-        "mvc/form/form-input",
-        "mvc/ui/ui-misc",
-        "mvc/form/form-data",
-        "mvc/tool/tool-form",
-        "utils/utils"
-    ],
-    function(testApp, sinon, InputElement, Ui, FormData, ToolForm, Utils) {
-        "use strict";
-        InputElement = InputElement.default;
-        Ui = Ui.default;
-        FormData = FormData.default;
-        ToolForm = ToolForm.default;
-        Utils = Utils.default;
-        QUnit.module("Form test", {
-            beforeEach: function() {
-                testApp.create();
-                this.clock = sinon.useFakeTimers();
-            },
-            afterEach: function() {
-                testApp.destroy();
-                this.clock.restore();
-            }
-        });
+import testApp from "qunit/test-app";
+import InputElement from "mvc/form/form-input";
+import Ui from "mvc/ui/ui-misc";
+import FormData from "mvc/form/form-data";
+import ToolForm from "mvc/tool/tool-form";
+import Utils from "utils/utils";
 
-        QUnit.test("tool-form", function(assert) {
-            var toolform = new ToolForm.View({ id: "test" });
-            var form = toolform.form;
-            $("body").prepend(toolform.$el);
-            window.fakeserver.respond();
-            var output = "";
-            for (var property in assert) {
-                output += property + ": ; ";
-            }
-            assert.ok(true, "MooCow");
-            assert.ok(
-                form.$(".portlet-title-text").html() == "<b>_name</b> _description (Galaxy Version _version)",
-                "Title correct"
-            );
-            var tour_ids = [];
-            $("[tour_id]").each(function() {
-                tour_ids.push($(this).attr("tour_id"));
-            });
-            assert.ok(
-                JSON.stringify(tour_ids) == '["a","b|c","b|i","b|j","k_0|l","k_0|m|n","k_0|m|s","k_0|m|t"]',
-                "Tour ids correct"
-            );
-            assert.ok(
-                JSON.stringify(form.data.create()) ==
-                    '{"a":"","b|c":"h","b|i":"i","b|j":"j","k_0|l":"l","k_0|m|n":"r","k_0|m|s":"s","k_0|m|t":"t"}',
-                "Created data correct"
-            );
-            var mapped_ids = [];
-            form.data.matchModel(form.model.attributes, function(input, id) {
-                mapped_ids.push($("#" + id).attr("tour_id"));
-            });
-            assert.ok(
-                JSON.stringify(mapped_ids) == '["a","b|c","b|i","b|j","k_0|l","k_0|m|n","k_0|m|s","k_0|m|t"]',
-                "Remapped tour ids correct"
-            );
-            this.clock.tick(window.WAIT_FADE);
-            var dropdown = form.$("#menu > .dropdown-menu");
-            assert.ok(dropdown.children().length == 2, "Found two menu items");
-            dropdown
-                .find(".fa-info-circle")
-                .parent()
-                .click();
-            this.clock.tick(window.WAIT_FADE);
-            assert.ok(
-                form.$(".ui-message").html() ===
-                    '<span>This tool requires req_name_a (Version req_version_a) and req_name_b (Version req_version_b). Click <a target="_blank" href="https://galaxyproject.org/tools/requirements/">here</a> for more information.</span>',
-                "Check requirements message"
-            );
-            assert.ok(form.$(".form-repeat-delete").css("display") == "none", "Delete button disabled");
-            var $add = form.$(".form-repeat-add");
-            assert.ok(!$add.attr("disabled"), "Adding new repeat possible");
-            $add.click();
-            this.clock.tick(window.WAIT_FADE);
-            assert.ok($add.attr("disabled"), "Adding new repeat has been disabled");
-            form.$(".form-repeat-delete").each(function(i, d) {
-                assert.ok($(d).css("display") == "block", "Delete buttons " + i + " enabled");
-            });
-            assert.ok(
-                JSON.stringify(form.data.create()) ==
-                    '{"a":"","b|c":"h","b|i":"i","b|j":"j","k_0|l":"l","k_0|m|n":"r","k_0|m|s":"s","k_0|m|t":"t","k_1|l":"l","k_1|m|n":"o","k_1|m|p":"p","k_1|m|q":"q"}',
-                "Created data correct, after adding repeat"
-            );
-            form.$(".form-repeat-delete:first").click();
-            assert.ok(form.$(".form-repeat-delete").css("display") == "none", "Delete button disabled");
-            this.clock.tick(window.WAIT_FADE);
-            assert.ok(
-                JSON.stringify(form.data.create()) ==
-                    '{"a":"","b|c":"h","b|i":"i","b|j":"j","k_0|l":"l","k_0|m|n":"o","k_0|m|p":"p","k_0|m|q":"q"}',
-                "Created data correct, after removing first repeat"
-            );
-        });
-
-        QUnit.test("data", function(assert) {
-            var visits = [];
-            Utils.get({
-                url: Galaxy.root + "api/tools/test/build",
-                success: function(response) {
-                    FormData.visitInputs(response.inputs, function(node, name, context) {
-                        visits.push({ name: name, node: node });
-                    });
-                }
-            });
-            window.fakeserver.respond();
-            assert.ok(
-                JSON.stringify(visits) ==
-                    '[{"name":"a","node":{"name":"a","type":"text"}},{"name":"b|c","node":{"name":"c","type":"select","value":"h","options":[["d","d",false],["h","h",false]]}},{"name":"b|i","node":{"name":"i","type":"text","value":"i"}},{"name":"b|j","node":{"name":"j","type":"text","value":"j"}},{"name":"k_0|l","node":{"name":"l","type":"text","value":"l"}},{"name":"k_0|m|n","node":{"name":"n","type":"select","value":"r","options":[["o","o",false],["r","r",false]]}},{"name":"k_0|m|s","node":{"name":"s","type":"text","value":"s"}},{"name":"k_0|m|t","node":{"name":"t","type":"text","value":"t"}}]',
-                "Testing value visitor"
-            );
-        });
-
-        QUnit.test("input", function(assert) {
-            var input = new InputElement(
-                {},
-                {
-                    field: new Ui.Input({})
-                }
-            );
-            $("body").prepend(input.$el);
-            assert.ok(input.$field.css("display") == "block", "Input field shown");
-            assert.ok(input.$preview.css("display") == "none", "Preview hidden");
-            assert.ok(input.$collapsible.css("display") == "none", "Collapsible hidden");
-            assert.ok(input.$title_text.css("display") == "inline", "Title visible");
-            assert.ok(input.$title_text.html() == "", "Title content unavailable");
-            input.model.set("label", "_label");
-            assert.ok(input.$title_text.html() == "_label", "Title content available");
-            assert.ok(input.$error.css("display") == "none", "Error hidden");
-            input.model.set("error_text", "_error_text");
-            assert.ok(input.$error.css("display") == "block", "Error visible");
-            assert.ok(input.$error_text.html() == "_error_text", "Error text correct");
-            input.model.set("error_text", null);
-            assert.ok(input.$error.css("display") == "none", "Error hidden, again");
-            assert.ok(input.$backdrop.css("display") == "none", "Backdrop hidden");
-            input.model.set("backdrop", true);
-            assert.ok(input.$backdrop.css("display") == "block", "Backdrop shown");
-            assert.ok(input.$backdrop.css("opacity") == 0, "Backdrop transparent");
-            assert.ok(input.$backdrop.css("cursor") == "default", "Backdrop regular cursor");
-            input.model.set("backdrop", false);
-            assert.ok(input.$backdrop.css("display") == "none", "Backdrop hidden, again");
-            input.model.set("disabled", true);
-            assert.ok(input.$field.css("display") == "none", "Input field hidden");
-            input.model.set("disabled", false);
-            this.clock.tick(window.WAIT_FADE);
-            assert.ok(input.$field.css("display") == "block", "Input field shown, again");
-            this.clock.tick(window.WAIT_FADE);
-            input.model.set("color", "red");
-            assert.ok(
-                input.$field
-                    .children()
-                    .first()
-                    .css("color") == "rgb(255, 0, 0)",
-                "Shows correct new color"
-            );
-            input.model.set("color", null);
-            assert.ok(
-                input.$field
-                    .children()
-                    .first()
-                    .css("color") == "rgb(85, 85, 85)",
-                "Shows correct old color"
-            );
-            input.model.set("collapsible_value", "_collapsible_value");
-            assert.ok(input.$collapsible.css("display") == "block", "Collapsible field");
-            assert.ok(input.$collapsible_text.html() == "_label", "Title content available");
-            assert.ok(input.$title_text.css("display") == "none", "Regular title not visible");
-            input.model.set("help", "_help");
-            assert.ok(input.$info.html() == "_help", "Correct help text");
-            input.model.set("argument", "_argument");
-            assert.ok(input.$info.html() == "_help (_argument)", "Correct help text with argument");
-            input.model.set("help", "_help (_argument)");
-            assert.ok(input.$info.html() == "_help (_argument)", "Correct help text with argument from help");
-        });
+QUnit.module("Form test", {
+    beforeEach: function() {
+        testApp.create();
+        $.fx.off = true;
+    },
+    afterEach: function() {
+        testApp.destroy();
+        $.fx.off = false;
     }
-);
+});
+
+QUnit.test("tool-form", function(assert) {
+    // Huh? The following seems to be needed by tool-form.js - once the global usage
+    // is cleaned up in that module this can be deleted I assume.
+    window.parent.Galaxy = window.Galaxy;
+
+    var toolform = new ToolForm.View({ id: "test" });
+    var form = toolform.form;
+    $("body").prepend(toolform.$el);
+    window.fakeserver.respond();
+    var output = "";
+    for (var property in assert) {
+        output += property + ": ; ";
+    }
+    assert.ok(
+        form.$(".portlet-title-text").html() == "<b>_name</b> _description (Galaxy Version _version)",
+        "Title correct"
+    );
+    var tour_ids = [];
+    $("[tour_id]").each(function() {
+        tour_ids.push($(this).attr("tour_id"));
+    });
+    assert.ok(
+        JSON.stringify(tour_ids) == '["a","b|c","b|i","b|j","k_0|l","k_0|m|n","k_0|m|s","k_0|m|t"]',
+        "Tour ids correct"
+    );
+    assert.ok(
+        JSON.stringify(form.data.create()) ==
+            '{"a":"","b|c":"h","b|i":"i","b|j":"j","k_0|l":"l","k_0|m|n":"r","k_0|m|s":"s","k_0|m|t":"t"}',
+        "Created data correct"
+    );
+    var mapped_ids = [];
+    form.data.matchModel(form.model.attributes, function(input, id) {
+        mapped_ids.push($("#" + id).attr("tour_id"));
+    });
+    assert.ok(
+        JSON.stringify(mapped_ids) == '["a","b|c","b|i","b|j","k_0|l","k_0|m|n","k_0|m|s","k_0|m|t"]',
+        "Remapped tour ids correct"
+    );
+    var dropdown = form.$("#menu > .dropdown-menu");
+    assert.ok(dropdown.children().length == 2, "Found two menu items");
+    dropdown
+        .find(".fa-info-circle")
+        .parent()
+        .click();
+    assert.ok(
+        form.$(".ui-message").html() ===
+            '<span>This tool requires req_name_a (Version req_version_a) and req_name_b (Version req_version_b). Click <a target="_blank" href="https://galaxyproject.org/tools/requirements/">here</a> for more information.</span>',
+        "Check requirements message"
+    );
+    assert.ok(form.$(".form-repeat-delete").css("display") == "none", "Delete button disabled");
+    var $add = form.$(".form-repeat-add");
+    assert.ok(!$add.attr("disabled"), "Adding new repeat possible");
+    $add.click();
+    assert.ok($add.attr("disabled"), "Adding new repeat has been disabled");
+    form.$(".form-repeat-delete").each(function(i, d) {
+        assert.ok($(d).css("display") == "block", "Delete buttons " + i + " enabled");
+    });
+    assert.ok(
+        JSON.stringify(form.data.create()) ==
+            '{"a":"","b|c":"h","b|i":"i","b|j":"j","k_0|l":"l","k_0|m|n":"r","k_0|m|s":"s","k_0|m|t":"t","k_1|l":"l","k_1|m|n":"o","k_1|m|p":"p","k_1|m|q":"q"}',
+        "Created data correct, after adding repeat"
+    );
+    form.$(".form-repeat-delete:first").click();
+    assert.ok(form.$(".form-repeat-delete").css("display") == "none", "Delete button disabled");
+    assert.ok(
+        JSON.stringify(form.data.create()) ==
+            '{"a":"","b|c":"h","b|i":"i","b|j":"j","k_0|l":"l","k_0|m|n":"o","k_0|m|p":"p","k_0|m|q":"q"}',
+        "Created data correct, after removing first repeat"
+    );
+});
+
+QUnit.test("data", function(assert) {
+    var visits = [];
+    Utils.get({
+        url: Galaxy.root + "api/tools/test/build",
+        success: function(response) {
+            FormData.visitInputs(response.inputs, function(node, name, context) {
+                visits.push({ name: name, node: node });
+            });
+        }
+    });
+    window.fakeserver.respond();
+    assert.ok(
+        JSON.stringify(visits) ==
+            '[{"name":"a","node":{"name":"a","type":"text"}},{"name":"b|c","node":{"name":"c","type":"select","value":"h","options":[["d","d",false],["h","h",false]]}},{"name":"b|i","node":{"name":"i","type":"text","value":"i"}},{"name":"b|j","node":{"name":"j","type":"text","value":"j"}},{"name":"k_0|l","node":{"name":"l","type":"text","value":"l"}},{"name":"k_0|m|n","node":{"name":"n","type":"select","value":"r","options":[["o","o",false],["r","r",false]]}},{"name":"k_0|m|s","node":{"name":"s","type":"text","value":"s"}},{"name":"k_0|m|t","node":{"name":"t","type":"text","value":"t"}}]',
+        "Testing value visitor"
+    );
+});
+
+QUnit.test("input", function(assert) {
+    var input = new InputElement(
+        {},
+        {
+            field: new Ui.Input({})
+        }
+    );
+    $("body").prepend(input.$el);
+    assert.ok(input.$field.css("display") == "block", "Input field shown");
+    assert.ok(input.$preview.css("display") == "none", "Preview hidden");
+    assert.ok(input.$collapsible.css("display") == "none", "Collapsible hidden");
+    assert.ok(input.$title_text.css("display") == "inline", "Title visible");
+    assert.ok(input.$title_text.html() == "", "Title content unavailable");
+    input.model.set("label", "_label");
+    assert.ok(input.$title_text.html() == "_label", "Title content available");
+    assert.ok(input.$error.css("display") == "none", "Error hidden");
+    input.model.set("error_text", "_error_text");
+    assert.ok(input.$error.css("display") == "block", "Error visible");
+    assert.ok(input.$error_text.html() == "_error_text", "Error text correct");
+    input.model.set("error_text", null);
+    assert.ok(input.$error.css("display") == "none", "Error hidden, again");
+    assert.ok(input.$backdrop.css("display") == "none", "Backdrop hidden");
+    input.model.set("backdrop", true);
+    assert.ok(input.$backdrop.css("display") == "block", "Backdrop shown");
+    assert.ok(input.$backdrop.css("opacity") == 0, "Backdrop transparent");
+    assert.ok(input.$backdrop.css("cursor") == "default", "Backdrop regular cursor");
+    input.model.set("backdrop", false);
+    assert.ok(input.$backdrop.css("display") == "none", "Backdrop hidden, again");
+    input.model.set("disabled", true);
+    assert.ok(input.$field.css("display") == "none", "Input field hidden");
+    input.model.set("disabled", false);
+    assert.ok(input.$field.css("display") == "block", "Input field shown, again");
+    input.model.set("color", "red");
+    assert.ok(
+        input.$field
+            .children()
+            .first()
+            .css("color") == "rgb(255, 0, 0)",
+        "Shows correct new color"
+    );
+    input.model.set("color", null);
+    assert.ok(
+        input.$field
+            .children()
+            .first()
+            .css("color") == "rgb(85, 85, 85)",
+        "Shows correct old color"
+    );
+    input.model.set("collapsible_value", "_collapsible_value");
+    assert.ok(input.$collapsible.css("display") == "block", "Collapsible field");
+    assert.ok(input.$collapsible_text.html() == "_label", "Title content available");
+    assert.ok(input.$title_text.css("display") == "none", "Regular title not visible");
+    input.model.set("help", "_help");
+    assert.ok(input.$info.html() == "_help", "Correct help text");
+    input.model.set("argument", "_argument");
+    assert.ok(input.$info.html() == "_help (_argument)", "Correct help text with argument");
+    input.model.set("help", "_help (_argument)");
+    assert.ok(input.$info.html() == "_help (_argument)", "Correct help text with argument from help");
+});

client/galaxy/scripts/qunit/tests/galaxy-app-base.js

@@ -1,96 +0,0 @@
-/* global define */
-define(["qunit/test-app", "galaxy", "jquery"], function(testApp, appBase, $) {
-    "use strict";
-    appBase = appBase.default;
-    QUnit.module("Galaxy client app tests");
-
-    var options = {
-        config: {
-            allow_user_deletion: false,
-            allow_user_creation: true,
-            wiki_url: "https://galaxyproject.org/",
-            ftp_upload_site: null,
-            support_url: "https://galaxyproject.org/support/",
-            allow_user_dataset_purge: false,
-            allow_library_path_paste: false,
-            user_library_import_dir: null,
-            terms_url: null,
-            ftp_upload_dir: null,
-            library_import_dir: null,
-            logo_url: null,
-            enable_unique_workflow_defaults: false
-        },
-        user: {
-            username: "test",
-            quota_percent: null,
-            total_disk_usage: 61815527,
-            nice_total_disk_usage: "59.0 MB",
-            email: "test@test.test",
-            tags_used: ["test"],
-            model_class: "User",
-            id: "f2db41e1fa331b3e"
-        }
-    };
-
-    QUnit.test("App base construction/initializiation defaults", function(assert) {
-        var app = new appBase.GalaxyApp({});
-        assert.ok(app.hasOwnProperty("options") && typeof app.options === "object");
-        assert.ok(app.hasOwnProperty("logger") && typeof app.logger === "object");
-        assert.ok(app.hasOwnProperty("localize") && typeof app.localize === "function");
-        assert.ok(app.hasOwnProperty("config") && typeof app.config === "object");
-        assert.ok(app.hasOwnProperty("user") && typeof app.config === "object");
-        assert.equal(app.localize, window._l);
-    });
-
-    QUnit.test("App base default options", function(assert) {
-        var app = new appBase.GalaxyApp({});
-        assert.ok(app.hasOwnProperty("options") && typeof app.options === "object");
-        assert.equal(app.options.root, "/");
-        assert.equal(app.options.patchExisting, true);
-    });
-
-    QUnit.test("App base extends from Backbone.Events", function(assert) {
-        var app = new appBase.GalaxyApp({});
-        ["on", "off", "trigger", "listenTo", "stopListening"].forEach(function(fn) {
-            assert.ok(app.hasOwnProperty(fn) && typeof app[fn] === "function");
-        });
-    });
-
-    QUnit.test("App base has logging methods from utils/add-logging.js", function(assert) {
-        var app = new appBase.GalaxyApp({});
-        ["debug", "info", "warn", "error", "metric"].forEach(function(fn) {
-            assert.ok(typeof app[fn] === "function");
-        });
-        assert.ok(app._logNamespace === "GalaxyApp");
-    });
-
-    QUnit.test("App base will patch in attributes from existing Galaxy objects", function(assert) {
-        window.Galaxy = {
-            attribute: {
-                subattr: 1
-            }
-        };
-        var app = new appBase.GalaxyApp({});
-        assert.ok(typeof app.attribute === "object" && app.attribute.subattr === 1);
-    });
-
-    QUnit.test("App base logger", function(assert) {
-        var app = new appBase.GalaxyApp({});
-        assert.ok(app.hasOwnProperty("logger") && typeof app.config === "object");
-    });
-
-    QUnit.test("App base config", function(assert) {
-        var app = new appBase.GalaxyApp(options);
-        assert.ok(app.hasOwnProperty("config") && typeof app.config === "object");
-        assert.equal(app.config.allow_user_deletion, false);
-        assert.equal(app.config.allow_user_creation, true);
-        assert.equal(app.config.wiki_url, "https://galaxyproject.org/");
-        assert.equal(app.config.ftp_upload_site, null);
-    });
-
-    QUnit.test("App base user", function(assert) {
-        var app = new appBase.GalaxyApp({});
-        assert.ok(app.hasOwnProperty("user") && typeof app.user === "object");
-        assert.ok(app.user.isAdmin() === false);
-    });
-});

client/galaxy/scripts/qunit/tests/galaxy_app_base_tests.js

@@ -0,0 +1,96 @@
+/* global define */
+import testApp from "qunit/test-app";
+import appBase from "galaxy";
+import $ from "jquery";
+
+QUnit.module("Galaxy client app tests");
+
+var options = {
+    config: {
+        allow_user_deletion: false,
+        allow_user_creation: true,
+        wiki_url: "https://galaxyproject.org/",
+        ftp_upload_site: null,
+        support_url: "https://galaxyproject.org/support/",
+        allow_user_dataset_purge: false,
+        allow_library_path_paste: false,
+        user_library_import_dir: null,
+        terms_url: null,
+        ftp_upload_dir: null,
+        library_import_dir: null,
+        logo_url: null,
+        enable_unique_workflow_defaults: false
+    },
+    user: {
+        username: "test",
+        quota_percent: null,
+        total_disk_usage: 61815527,
+        nice_total_disk_usage: "59.0 MB",
+        email: "test@test.test",
+        tags_used: ["test"],
+        model_class: "User",
+        id: "f2db41e1fa331b3e"
+    }
+};
+
+QUnit.test("App base construction/initializiation defaults", function(assert) {
+    var app = new appBase.GalaxyApp({});
+    assert.ok(app.hasOwnProperty("options") && typeof app.options === "object");
+    assert.ok(app.hasOwnProperty("logger") && typeof app.logger === "object");
+    assert.ok(app.hasOwnProperty("localize") && typeof app.localize === "function");
+    assert.ok(app.hasOwnProperty("config") && typeof app.config === "object");
+    assert.ok(app.hasOwnProperty("user") && typeof app.config === "object");
+    assert.equal(app.localize, window._l);
+});
+
+QUnit.test("App base default options", function(assert) {
+    var app = new appBase.GalaxyApp({});
+    assert.ok(app.hasOwnProperty("options") && typeof app.options === "object");
+    assert.equal(app.options.root, "/");
+    assert.equal(app.options.patchExisting, true);
+});
+
+QUnit.test("App base extends from Backbone.Events", function(assert) {
+    var app = new appBase.GalaxyApp({});
+    ["on", "off", "trigger", "listenTo", "stopListening"].forEach(function(fn) {
+        assert.ok(app.hasOwnProperty(fn) && typeof app[fn] === "function");
+    });
+});
+
+QUnit.test("App base has logging methods from utils/add-logging.js", function(assert) {
+    var app = new appBase.GalaxyApp({});
+    ["debug", "info", "warn", "error", "metric"].forEach(function(fn) {
+        assert.ok(typeof app[fn] === "function");
+    });
+    assert.ok(app._logNamespace === "GalaxyApp");
+});
+
+QUnit.test("App base will patch in attributes from existing Galaxy objects", function(assert) {
+    window.Galaxy = {
+        attribute: {
+            subattr: 1
+        }
+    };
+    var app = new appBase.GalaxyApp({});
+    assert.ok(typeof app.attribute === "object" && app.attribute.subattr === 1);
+});
+
+QUnit.test("App base logger", function(assert) {
+    var app = new appBase.GalaxyApp({});
+    assert.ok(app.hasOwnProperty("logger") && typeof app.config === "object");
+});
+
+QUnit.test("App base config", function(assert) {
+    var app = new appBase.GalaxyApp(options);
+    assert.ok(app.hasOwnProperty("config") && typeof app.config === "object");
+    assert.equal(app.config.allow_user_deletion, false);
+    assert.equal(app.config.allow_user_creation, true);
+    assert.equal(app.config.wiki_url, "https://galaxyproject.org/");
+    assert.equal(app.config.ftp_upload_site, null);
+});
+
+QUnit.test("App base user", function(assert) {
+    var app = new appBase.GalaxyApp({});
+    assert.ok(app.hasOwnProperty("user") && typeof app.user === "object");
+    assert.ok(app.user.isAdmin() === false);
+});

client/galaxy/scripts/qunit/tests/graph.js

@@ -1,515 +0,0 @@
-/* global define */
-define(["qunit/test-app", "utils/graph", "jquery"], function(testApp, GRAPH, $) {
-    "use strict";
-    GRAPH = GRAPH.default;
-    QUnit.module("utils/graph.js library tests");
-    ///*
-    function testEmptyObject(assert, o) {
-        assert.ok(typeof o === "object");
-        assert.ok(Object.keys(o).length === 0);
-    }
-
-    // ------------------------------------------------------------------------ vertices
-    QUnit.test("Empty vertex construction", function(assert) {
-        var vert = new GRAPH.Vertex();
-        assert.ok(vert instanceof GRAPH.Vertex);
-        assert.ok(vert.name === "(unnamed)");
-        assert.ok(vert.data === null);
-        testEmptyObject(assert, vert.edges);
-        assert.ok(vert.degree === 0);
-        assert.ok(vert + "" === "Vertex((unnamed))");
-        assert.deepEqual(vert.toJSON(), { name: "(unnamed)", data: null });
-    });
-
-    QUnit.test("Vertex construction", function(assert) {
-        var vert = new GRAPH.Vertex("blah", { blorp: 1, bleep: 2 });
-        assert.ok(vert instanceof GRAPH.Vertex);
-        assert.ok(vert.name === "blah");
-        assert.deepEqual(vert.data, { blorp: 1, bleep: 2 });
-        testEmptyObject(assert, vert.edges);
-        assert.ok(vert.degree === 0);
-        assert.ok(vert + "" === "Vertex(blah)");
-        assert.deepEqual(vert.toJSON(), { name: "blah", data: { blorp: 1, bleep: 2 } });
-    });
-
-    // ------------------------------------------------------------------------ edges
-    QUnit.test("Empty edge construction", function(assert) {
-        var edge = new GRAPH.Edge();
-        assert.ok(edge instanceof GRAPH.Edge);
-        assert.ok(edge.source === null);
-        assert.ok(edge.target === null);
-        assert.ok(edge.data === null);
-        assert.ok(edge + "" === "null->null");
-        assert.deepEqual(edge.toJSON(), { source: null, target: null });
-    });
-
-    QUnit.test("Edge construction", function(assert) {
-        var edge = new GRAPH.Edge("A", "B", { one: 1, two: 2 });
-        assert.ok(edge instanceof GRAPH.Edge);
-        assert.ok(edge.source === "A");
-        assert.ok(edge.target === "B");
-        assert.deepEqual(edge.data, { one: 1, two: 2 });
-        assert.ok(edge + "" === "A->B");
-        assert.deepEqual(edge.toJSON(), { source: "A", target: "B", data: { one: 1, two: 2 } });
-    });
-
-    // ------------------------------------------------------------------------ graphs
-    function testEmptyGraph(assert, graph) {
-        assert.ok(graph instanceof GRAPH.Graph);
-        testEmptyObject(assert, graph.vertices);
-        assert.ok(graph.numEdges === 0);
-    }
-
-    QUnit.test("Empty graph construction", function(assert) {
-        var graph = new GRAPH.Graph();
-
-        assert.ok(graph.directed === false);
-        assert.ok(graph.allowReflexiveEdges === false);
-
-        testEmptyGraph(assert, graph);
-    });
-
-    QUnit.test("Bad data graph construction", function(assert) {
-        var graph = new GRAPH.Graph(false, {});
-        testEmptyGraph(assert, graph);
-
-        graph = new GRAPH.Graph(false, null);
-        testEmptyGraph(assert, graph);
-    });
-
-    QUnit.test("Test directed and options", function(assert) {
-        var graph = new GRAPH.Graph(true, {}, { allowReflexiveEdges: true });
-
-        assert.ok(graph.directed);
-        assert.ok(graph.allowReflexiveEdges);
-
-        testEmptyGraph(assert, graph);
-    });
-
-    function testSampleDirectedGraph(assert, graph) {
-        assert.ok(!graph.directed);
-
-        assert.ok(Object.keys(graph.vertices).length === 3);
-        assert.ok(graph.vertices.A instanceof GRAPH.Vertex);
-        assert.ok(graph.vertices.B instanceof GRAPH.Vertex);
-        assert.ok(graph.vertices.C instanceof GRAPH.Vertex);
-
-        assert.deepEqual(Object.keys(graph.vertices.A.edges), ["B", "C"]);
-        assert.deepEqual(Object.keys(graph.vertices.B.edges), ["A", "C"]);
-        assert.deepEqual(Object.keys(graph.vertices.C.edges), ["A", "B"]);
-
-        assert.ok(graph.vertices.A.degree === 2);
-        assert.ok(graph.vertices.B.degree === 2);
-        assert.ok(graph.vertices.C.degree === 2);
-
-        assert.deepEqual(graph.vertices.A.edges.B.toJSON(), { source: "A", target: "B" });
-        assert.deepEqual(graph.vertices.A.edges.C.toJSON(), { source: "A", target: "C" });
-        assert.deepEqual(graph.vertices.B.edges.A.toJSON(), { source: "B", target: "A" });
-        assert.deepEqual(graph.vertices.B.edges.C.toJSON(), { source: "B", target: "C" });
-        assert.deepEqual(graph.vertices.C.edges.A.toJSON(), { source: "C", target: "A" });
-        assert.deepEqual(graph.vertices.C.edges.B.toJSON(), { source: "C", target: "B" });
-
-        assert.ok(graph.numEdges === 6);
-    }
-
-    function testSampleNonDirectedGraph(assert, graph) {
-        assert.ok(graph.directed);
-
-        assert.ok(Object.keys(graph.vertices).length === 3);
-        assert.ok(graph.vertices.A instanceof GRAPH.Vertex);
-        assert.ok(graph.vertices.B instanceof GRAPH.Vertex);
-        assert.ok(graph.vertices.C instanceof GRAPH.Vertex);
-
-        assert.deepEqual(Object.keys(graph.vertices.A.edges), ["B", "C"]);
-        assert.deepEqual(Object.keys(graph.vertices.B.edges), ["C"]);
-        assert.deepEqual(Object.keys(graph.vertices.C.edges), []);
-
-        assert.ok(graph.vertices.A.degree === 2);
-        assert.ok(graph.vertices.B.degree === 1);
-        assert.ok(graph.vertices.C.degree === 0);
-
-        assert.deepEqual(graph.vertices.A.edges.B.toJSON(), { source: "A", target: "B" });
-        assert.deepEqual(graph.vertices.A.edges.C.toJSON(), { source: "A", target: "C" });
-        assert.deepEqual(graph.vertices.B.edges.C.toJSON(), { source: "B", target: "C" });
-
-        assert.ok(graph.numEdges === 3);
-    }
-
-    var nodeLinkData = {
-        nodes: [{ name: "A", data: 100 }, { name: "B", data: 200 }, { name: "C", data: 300 }],
-        links: [{ source: 0, target: 1 }, { source: 0, target: 2 }, { source: 1, target: 2 }]
-    };
-
-    QUnit.test("Test nodes and links data input on *non-directed* graph", function(assert) {
-        var graph = new GRAPH.Graph(false, nodeLinkData);
-        testSampleDirectedGraph(assert, graph);
-    });
-
-    QUnit.test("Test nodes and links data input on *directed* graph", function(assert) {
-        var graph = new GRAPH.Graph(true, nodeLinkData);
-        testSampleNonDirectedGraph(assert, graph);
-    });
-
-    var vertexEdgeData = {
-        vertices: [{ name: "A", data: 100 }, { name: "B", data: 200 }, { name: "C", data: 300 }],
-        edges: [{ source: "A", target: "B" }, { source: "A", target: "C" }, { source: "B", target: "C" }]
-    };
-
-    QUnit.test("Test vertex and edge data input on *non-directed* graph", function(assert) {
-        var graph = new GRAPH.Graph(false, vertexEdgeData);
-        testSampleDirectedGraph(assert, graph);
-    });
-
-    QUnit.test("Test vertex and edge data input on *directed* graph", function(assert) {
-        var graph = new GRAPH.Graph(true, vertexEdgeData);
-        testSampleNonDirectedGraph(assert, graph);
-    });
-
-    QUnit.test("Test vertex eachEdge", function(assert) {
-        var graph = new GRAPH.Graph(false, nodeLinkData);
-        assert.ok(typeof graph.vertices.A.eachEdge === "function");
-        assert.deepEqual(
-            graph.vertices.A.eachEdge(function(e) {
-                return e.target;
-            }),
-            ["B", "C"]
-        );
-        assert.ok(graph.vertices.A.eachEdge({ target: "B" }).length === 1);
-    });
-
-    QUnit.test("Test graph eachVertex", function(assert) {
-        var graph = new GRAPH.Graph(true, nodeLinkData);
-        assert.ok(typeof graph.eachVertex === "function");
-        assert.deepEqual(
-            graph.eachVertex(function(v) {
-                return { n: v.name, d: v.degree };
-            }),
-            [{ n: "A", d: 2 }, { n: "B", d: 1 }, { n: "C", d: 0 }]
-        );
-        assert.ok(graph.eachVertex({ degree: 2 })[0] === graph.vertices.A);
-    });
-
-    QUnit.test("Test createVertex", function(assert) {
-        var graph = new GRAPH.Graph();
-        var vert1 = graph.createVertex("A", { blah: 1 });
-        assert.ok(vert1 instanceof GRAPH.Vertex);
-        assert.ok(vert1 === graph.vertices.A);
-        assert.ok(graph.createVertex("A", { blah: 1 }) === vert1);
-    });
-
-    QUnit.test("Test createEdge", function(assert) {
-        var graph, A, B, edge;
-
-        graph = new GRAPH.Graph();
-        A = graph.createVertex("A");
-        B = graph.createVertex("B");
-        edge = graph.createEdge("A", "B");
-        assert.ok(edge instanceof GRAPH.Edge);
-        assert.ok(A.degree === 1);
-        assert.ok(B.degree === 1);
-        assert.ok(A.edges.B);
-        assert.ok(B.edges.A);
-        assert.ok(graph.numEdges === 2);
-
-        // bad target
-        graph = new GRAPH.Graph();
-        A = graph.createVertex("A");
-        B = graph.createVertex("B");
-        edge = graph.createEdge("A", "C");
-        assert.ok(edge === null);
-        assert.ok(A.degree === 0);
-        assert.ok(B.degree === 0);
-        assert.ok(!A.edges.B);
-        assert.ok(!B.edges.A);
-        assert.ok(graph.numEdges === 0);
-
-        // bad source
-        graph = new GRAPH.Graph();
-        A = graph.createVertex("A");
-        B = graph.createVertex("B");
-        edge = graph.createEdge("C", "A");
-        assert.ok(graph.numEdges === 0);
-
-        // reflexive
-        graph = new GRAPH.Graph();
-        A = graph.createVertex("A");
-        B = graph.createVertex("B");
-        edge = graph.createEdge("A", "A");
-        assert.ok(graph.numEdges === 0);
-
-        // reflexive (allowed)
-        graph = new GRAPH.Graph(false, {}, { allowReflexiveEdges: true });
-        A = graph.createVertex("A");
-        edge = graph.createEdge("A", "A");
-        // reflexive edges shouldn't mirror
-        assert.ok(graph.numEdges === 1);
-        assert.ok(A.edges.A);
-    });
-
-    QUnit.test("Test graph.edges", function(assert) {
-        var graph = new GRAPH.Graph(false, nodeLinkData);
-        assert.ok(graph.edges().length === 6);
-        assert.deepEqual(
-            graph.edges(function(e) {
-                return e.source;
-            }),
-            ["A", "A", "B", "B", "C", "C"]
-        );
-        assert.deepEqual(graph.edges({ source: "A" }), [graph.vertices.A.edges.B, graph.vertices.A.edges.C]);
-
-        graph = new GRAPH.Graph(true, nodeLinkData);
-        assert.ok(graph.edges().length === 3);
-        assert.deepEqual(
-            graph.edges(function(e) {
-                return e.source;
-            }),
-            ["A", "A", "B"]
-        );
-        assert.deepEqual(graph.edges({ source: "A" }), [graph.vertices.A.edges.B, graph.vertices.A.edges.C]);
-    });
-
-    QUnit.test("Test graph.adjacent", function(assert) {
-        var graph = new GRAPH.Graph(true, nodeLinkData);
-        assert.deepEqual(graph.adjacent(graph.vertices.A), [graph.vertices.B, graph.vertices.C]);
-        assert.deepEqual(graph.adjacent(graph.vertices.B), [graph.vertices.C]);
-        assert.deepEqual(graph.adjacent(graph.vertices.C), []);
-    });
-
-    QUnit.test("Test graph.eachAdjacent", function(assert) {
-        var graph = new GRAPH.Graph(true, nodeLinkData);
-        assert.deepEqual(
-            graph.eachAdjacent(graph.vertices.A, function(v, e) {
-                return v;
-            }),
-            [graph.vertices.B, graph.vertices.C]
-        );
-    });
-
-    // ------------------------------------------------------------------------ breadth first search
-    QUnit.test("Empty BreadthFirstSearch", function(assert) {
-        var search = new GRAPH.BreadthFirstSearch();
-        assert.ok(search instanceof GRAPH.BreadthFirstSearch);
-        assert.ok(search.graph === undefined);
-        assert.ok(typeof search.processFns === "object");
-        assert.ok(typeof search.processFns.vertexEarly === "function");
-        assert.ok(typeof search.processFns.edge === "function");
-        assert.ok(typeof search.processFns.vertexLate === "function");
-        assert.ok(typeof search._cache === "object");
-    });
-
-    QUnit.test("BreadthFirstSearch on undirected graph", function(assert) {
-        var graph = new GRAPH.Graph(false, nodeLinkData),
-            bfs = new GRAPH.BreadthFirstSearch(graph);
-        assert.ok(bfs instanceof GRAPH.BreadthFirstSearch);
-        assert.ok(bfs.graph === graph);
-
-        var search = bfs.search("A"),
-            tree = bfs.searchTree("A");
-        assert.deepEqual(search, {
-            discovered: { A: true, B: true, C: true },
-            edges: [{ source: "A", target: "B" }, { source: "A", target: "C" }]
-        });
-        assert.ok(tree instanceof GRAPH.Graph);
-        assert.deepEqual(tree.vertices.A.toJSON(), graph.vertices.A.toJSON());
-        assert.deepEqual(
-            tree.eachVertex(function(v) {
-                return v.degree;
-            }),
-            [2, 0, 0]
-        );
-
-        assert.deepEqual(bfs.search("B").edges, [{ source: "B", target: "A" }, { source: "B", target: "C" }]);
-        assert.deepEqual(bfs.search("C").edges, [{ source: "C", target: "A" }, { source: "C", target: "B" }]);
-        assert.ok(typeof bfs._cache.A === "object");
-        assert.deepEqual(Object.keys(bfs._cache), ["A", "B", "C"]);
-    });
-
-    QUnit.test("BreadthFirstSearch on directed graph", function(assert) {
-        var graph = new GRAPH.Graph(true, nodeLinkData),
-            bfs = new GRAPH.BreadthFirstSearch(graph);
-        assert.ok(bfs instanceof GRAPH.BreadthFirstSearch);
-        assert.ok(bfs.graph === graph);
-
-        var search = bfs.search("A"),
-            tree = bfs.searchTree("A");
-        assert.deepEqual(search, {
-            discovered: { A: true, B: true, C: true },
-            edges: [{ source: "A", target: "B" }, { source: "A", target: "C" }]
-        });
-        assert.ok(tree instanceof GRAPH.Graph);
-        assert.deepEqual(tree.vertices.A.toJSON(), graph.vertices.A.toJSON());
-        assert.deepEqual(
-            tree.eachVertex(function(v) {
-                return v.degree;
-            }),
-            [2, 0, 0]
-        );
-
-        assert.deepEqual(bfs.search("B").edges, [{ source: "B", target: "C" }]);
-        assert.deepEqual(bfs.search("C").edges, []);
-        assert.ok(typeof bfs._cache.A === "object");
-        assert.deepEqual(Object.keys(bfs._cache), ["A", "B", "C"]);
-    });
-
-    // ------------------------------------------------------------------------ depth first search
-    var DFSData = {
-        vertices: [{ name: "A" }, { name: "B" }, { name: "C" }, { name: "D" }, { name: "E" }, { name: "F" }],
-        edges: [
-            { source: "A", target: "B" },
-            { source: "B", target: "C" },
-            // confound it
-            { source: "A", target: "C" },
-            { source: "C", target: "D" },
-            { source: "A", target: "E" },
-            { source: "E", target: "F" },
-            // confound it
-            { source: "F", target: "A" }
-        ]
-    };
-
-    QUnit.test("Empty DepthFirstSearch", function(assert) {
-        var search = new GRAPH.DepthFirstSearch();
-        assert.ok(search instanceof GRAPH.DepthFirstSearch);
-        assert.ok(search.graph === undefined);
-        assert.ok(typeof search.processFns === "object");
-        assert.ok(typeof search.processFns.vertexEarly === "function");
-        assert.ok(typeof search.processFns.edge === "function");
-        assert.ok(typeof search.processFns.vertexLate === "function");
-        assert.ok(typeof search._cache === "object");
-    });
-
-    QUnit.test("DepthFirstSearch on undirected graph", function(assert) {
-        var graph = new GRAPH.Graph(false, DFSData),
-            dfs = new GRAPH.DepthFirstSearch(graph);
-        assert.ok(dfs instanceof GRAPH.DepthFirstSearch);
-        assert.ok(dfs.graph === graph);
-
-        var search = dfs.search("A"),
-            tree = dfs.searchTree("A");
-        assert.deepEqual(search, {
-            discovered: { A: true, B: true, C: true, D: true, E: true, F: true },
-            edges: [
-                { source: "A", target: "B" },
-                { source: "B", target: "C" },
-                { source: "C", target: "D" },
-                { source: "A", target: "E" },
-                { source: "E", target: "F" }
-            ],
-            entryTimes: { A: 0, B: 1, C: 2, D: 3, E: 7, F: 8 },
-            exitTimes: { A: 11, B: 6, C: 5, D: 4, E: 10, F: 9 }
-        });
-        assert.ok(tree instanceof GRAPH.Graph);
-        assert.deepEqual(tree.vertices.A.toJSON(), graph.vertices.A.toJSON());
-        assert.deepEqual(
-            tree.eachVertex(function(v) {
-                return v.degree;
-            }),
-            [2, 1, 1, 0, 1, 0]
-        );
-
-        assert.deepEqual(dfs.search("B").edges, [
-            { source: "B", target: "A" },
-            { source: "A", target: "C" },
-            { source: "C", target: "D" },
-            { source: "A", target: "E" },
-            { source: "E", target: "F" }
-        ]);
-
-        assert.ok(typeof dfs._cache.A === "object");
-        assert.deepEqual(Object.keys(dfs._cache), ["A", "B"]);
-    });
-
-    QUnit.test("DepthFirstSearch on directed graph", function(assert) {
-        var graph = new GRAPH.Graph(true, DFSData),
-            dfs = new GRAPH.DepthFirstSearch(graph);
-        assert.ok(dfs instanceof GRAPH.DepthFirstSearch);
-        assert.ok(dfs.graph === graph);
-
-        var search = dfs.search("A"),
-            tree = dfs.searchTree("A");
-        assert.deepEqual(search, {
-            discovered: { A: true, B: true, C: true, D: true, E: true, F: true },
-            edges: [
-                { source: "A", target: "B" },
-                { source: "B", target: "C" },
-                { source: "C", target: "D" },
-                { source: "A", target: "E" },
-                { source: "E", target: "F" }
-            ],
-            entryTimes: { A: 0, B: 1, C: 2, D: 3, E: 7, F: 8 },
-            exitTimes: { A: 11, B: 6, C: 5, D: 4, E: 10, F: 9 }
-        });
-        assert.ok(tree instanceof GRAPH.Graph);
-        assert.deepEqual(tree.vertices.A.toJSON(), graph.vertices.A.toJSON());
-        assert.deepEqual(
-            tree.eachVertex(function(v) {
-                return v.degree;
-            }),
-            [2, 1, 1, 0, 1, 0]
-        );
-
-        assert.deepEqual(dfs.search("B").edges, [{ source: "B", target: "C" }, { source: "C", target: "D" }]);
-
-        assert.ok(typeof dfs._cache.A === "object");
-        assert.deepEqual(Object.keys(dfs._cache), ["A", "B"]);
-    });
-
-    // ------------------------------------------------------------------------ components
-    //*/
-    QUnit.test("weakComponents on undirected graph", function(assert) {
-        var graph = new GRAPH.Graph(false, {
-            vertices: [{ name: "A" }, { name: "B" }, { name: "C" }, { name: "D" }, { name: "E" }],
-            edges: [{ source: "A", target: "B" }, { source: "C", target: "D" }]
-        });
-        assert.equal(graph.numEdges, 4);
-        var components = graph.weakComponents();
-        assert.equal(components.length, 3);
-    });
-
-    QUnit.test("weakComponents on directed graph", function(assert) {
-        var graph, components;
-        graph = new GRAPH.Graph(true, {
-            vertices: [{ name: "A" }, { name: "B" }, { name: "C" }, { name: "D" }, { name: "E" }],
-            edges: [{ source: "A", target: "B" }, { source: "D", target: "C" }]
-        });
-        assert.equal(graph.numEdges, 2);
-
-        components = graph.weakComponents();
-        assert.equal(components.length, 3);
-
-        graph = new GRAPH.Graph(true, {
-            vertices: [
-                { name: "A", data: 100 },
-                { name: "B", data: 200 },
-                { name: "C", data: 30 },
-                { name: "D", data: 40 },
-                { name: "E", data: 500 },
-                { name: "F", data: 600 },
-                { name: "G", data: 7 }
-            ],
-            edges: [
-                { source: "A", target: "B" },
-                { source: "D", target: "C" },
-                { source: "E", target: "A" },
-                { source: "F", target: "E" }
-            ]
-        });
-        components = graph.weakComponents();
-        assert.equal(components.length, 3);
-        // data retained
-        assert.equal(components[0].vertices[0].data, 100);
-
-        assert.equal(components[0].vertices.length, 4);
-        assert.deepEqual(components[0].edges, [
-            { source: "A", target: "B" },
-            { source: "E", target: "A" },
-            { source: "F", target: "E" }
-        ]);
-
-        assert.equal(components[1].vertices.length, 2);
-        assert.deepEqual(components[1].edges, [{ source: "D", target: "C" }]);
-
-        assert.deepEqual(components[2].vertices, [{ name: "G", data: 7 }]);
-        assert.deepEqual(components[2].edges.length, 0);
-    });
-});

client/galaxy/scripts/qunit/tests/graph_tests.js

@@ -0,0 +1,515 @@
+/* global define */
+import testApp from "qunit/test-app";
+import GRAPH from "utils/graph";
+import $ from "jquery";
+
+QUnit.module("utils/graph.js library tests");
+///*
+function testEmptyObject(assert, o) {
+    assert.ok(typeof o === "object");
+    assert.ok(Object.keys(o).length === 0);
+}
+
+// ------------------------------------------------------------------------ vertices
+QUnit.test("Empty vertex construction", function(assert) {
+    var vert = new GRAPH.Vertex();
+    assert.ok(vert instanceof GRAPH.Vertex);
+    assert.ok(vert.name === "(unnamed)");
+    assert.ok(vert.data === null);
+    testEmptyObject(assert, vert.edges);
+    assert.ok(vert.degree === 0);
+    assert.ok(vert + "" === "Vertex((unnamed))");
+    assert.deepEqual(vert.toJSON(), { name: "(unnamed)", data: null });
+});
+
+QUnit.test("Vertex construction", function(assert) {
+    var vert = new GRAPH.Vertex("blah", { blorp: 1, bleep: 2 });
+    assert.ok(vert instanceof GRAPH.Vertex);
+    assert.ok(vert.name === "blah");
+    assert.deepEqual(vert.data, { blorp: 1, bleep: 2 });
+    testEmptyObject(assert, vert.edges);
+    assert.ok(vert.degree === 0);
+    assert.ok(vert + "" === "Vertex(blah)");
+    assert.deepEqual(vert.toJSON(), { name: "blah", data: { blorp: 1, bleep: 2 } });
+});
+
+// ------------------------------------------------------------------------ edges
+QUnit.test("Empty edge construction", function(assert) {
+    var edge = new GRAPH.Edge();
+    assert.ok(edge instanceof GRAPH.Edge);
+    assert.ok(edge.source === null);
+    assert.ok(edge.target === null);
+    assert.ok(edge.data === null);
+    assert.ok(edge + "" === "null->null");
+    assert.deepEqual(edge.toJSON(), { source: null, target: null });
+});
+
+QUnit.test("Edge construction", function(assert) {
+    var edge = new GRAPH.Edge("A", "B", { one: 1, two: 2 });
+    assert.ok(edge instanceof GRAPH.Edge);
+    assert.ok(edge.source === "A");
+    assert.ok(edge.target === "B");
+    assert.deepEqual(edge.data, { one: 1, two: 2 });
+    assert.ok(edge + "" === "A->B");
+    assert.deepEqual(edge.toJSON(), { source: "A", target: "B", data: { one: 1, two: 2 } });
+});
+
+// ------------------------------------------------------------------------ graphs
+function testEmptyGraph(assert, graph) {
+    assert.ok(graph instanceof GRAPH.Graph);
+    testEmptyObject(assert, graph.vertices);
+    assert.ok(graph.numEdges === 0);
+}
+
+QUnit.test("Empty graph construction", function(assert) {
+    var graph = new GRAPH.Graph();
+
+    assert.ok(graph.directed === false);
+    assert.ok(graph.allowReflexiveEdges === false);
+
+    testEmptyGraph(assert, graph);
+});
+
+QUnit.test("Bad data graph construction", function(assert) {
+    var graph = new GRAPH.Graph(false, {});
+    testEmptyGraph(assert, graph);
+
+    graph = new GRAPH.Graph(false, null);
+    testEmptyGraph(assert, graph);
+});
+
+QUnit.test("Test directed and options", function(assert) {
+    var graph = new GRAPH.Graph(true, {}, { allowReflexiveEdges: true });
+
+    assert.ok(graph.directed);
+    assert.ok(graph.allowReflexiveEdges);
+
+    testEmptyGraph(assert, graph);
+});
+
+function testSampleDirectedGraph(assert, graph) {
+    assert.ok(!graph.directed);
+
+    assert.ok(Object.keys(graph.vertices).length === 3);
+    assert.ok(graph.vertices.A instanceof GRAPH.Vertex);
+    assert.ok(graph.vertices.B instanceof GRAPH.Vertex);
+    assert.ok(graph.vertices.C instanceof GRAPH.Vertex);
+
+    assert.deepEqual(Object.keys(graph.vertices.A.edges), ["B", "C"]);
+    assert.deepEqual(Object.keys(graph.vertices.B.edges), ["A", "C"]);
+    assert.deepEqual(Object.keys(graph.vertices.C.edges), ["A", "B"]);
+
+    assert.ok(graph.vertices.A.degree === 2);
+    assert.ok(graph.vertices.B.degree === 2);
+    assert.ok(graph.vertices.C.degree === 2);
+
+    assert.deepEqual(graph.vertices.A.edges.B.toJSON(), { source: "A", target: "B" });
+    assert.deepEqual(graph.vertices.A.edges.C.toJSON(), { source: "A", target: "C" });
+    assert.deepEqual(graph.vertices.B.edges.A.toJSON(), { source: "B", target: "A" });
+    assert.deepEqual(graph.vertices.B.edges.C.toJSON(), { source: "B", target: "C" });
+    assert.deepEqual(graph.vertices.C.edges.A.toJSON(), { source: "C", target: "A" });
+    assert.deepEqual(graph.vertices.C.edges.B.toJSON(), { source: "C", target: "B" });
+
+    assert.ok(graph.numEdges === 6);
+}
+
+function testSampleNonDirectedGraph(assert, graph) {
+    assert.ok(graph.directed);
+
+    assert.ok(Object.keys(graph.vertices).length === 3);
+    assert.ok(graph.vertices.A instanceof GRAPH.Vertex);
+    assert.ok(graph.vertices.B instanceof GRAPH.Vertex);
+    assert.ok(graph.vertices.C instanceof GRAPH.Vertex);
+
+    assert.deepEqual(Object.keys(graph.vertices.A.edges), ["B", "C"]);
+    assert.deepEqual(Object.keys(graph.vertices.B.edges), ["C"]);
+    assert.deepEqual(Object.keys(graph.vertices.C.edges), []);
+
+    assert.ok(graph.vertices.A.degree === 2);
+    assert.ok(graph.vertices.B.degree === 1);
+    assert.ok(graph.vertices.C.degree === 0);
+
+    assert.deepEqual(graph.vertices.A.edges.B.toJSON(), { source: "A", target: "B" });
+    assert.deepEqual(graph.vertices.A.edges.C.toJSON(), { source: "A", target: "C" });
+    assert.deepEqual(graph.vertices.B.edges.C.toJSON(), { source: "B", target: "C" });
+
+    assert.ok(graph.numEdges === 3);
+}
+
+var nodeLinkData = {
+    nodes: [{ name: "A", data: 100 }, { name: "B", data: 200 }, { name: "C", data: 300 }],
+    links: [{ source: 0, target: 1 }, { source: 0, target: 2 }, { source: 1, target: 2 }]
+};
+
+QUnit.test("Test nodes and links data input on *non-directed* graph", function(assert) {
+    var graph = new GRAPH.Graph(false, nodeLinkData);
+    testSampleDirectedGraph(assert, graph);
+});
+
+QUnit.test("Test nodes and links data input on *directed* graph", function(assert) {
+    var graph = new GRAPH.Graph(true, nodeLinkData);
+    testSampleNonDirectedGraph(assert, graph);
+});
+
+var vertexEdgeData = {
+    vertices: [{ name: "A", data: 100 }, { name: "B", data: 200 }, { name: "C", data: 300 }],
+    edges: [{ source: "A", target: "B" }, { source: "A", target: "C" }, { source: "B", target: "C" }]
+};
+
+QUnit.test("Test vertex and edge data input on *non-directed* graph", function(assert) {
+    var graph = new GRAPH.Graph(false, vertexEdgeData);
+    testSampleDirectedGraph(assert, graph);
+});
+
+QUnit.test("Test vertex and edge data input on *directed* graph", function(assert) {
+    var graph = new GRAPH.Graph(true, vertexEdgeData);
+    testSampleNonDirectedGraph(assert, graph);
+});
+
+QUnit.test("Test vertex eachEdge", function(assert) {
+    var graph = new GRAPH.Graph(false, nodeLinkData);
+    assert.ok(typeof graph.vertices.A.eachEdge === "function");
+    assert.deepEqual(
+        graph.vertices.A.eachEdge(function(e) {
+            return e.target;
+        }),
+        ["B", "C"]
+    );
+    assert.ok(graph.vertices.A.eachEdge({ target: "B" }).length === 1);
+});
+
+QUnit.test("Test graph eachVertex", function(assert) {
+    var graph = new GRAPH.Graph(true, nodeLinkData);
+    assert.ok(typeof graph.eachVertex === "function");
+    assert.deepEqual(
+        graph.eachVertex(function(v) {
+            return { n: v.name, d: v.degree };
+        }),
+        [{ n: "A", d: 2 }, { n: "B", d: 1 }, { n: "C", d: 0 }]
+    );
+    assert.ok(graph.eachVertex({ degree: 2 })[0] === graph.vertices.A);
+});
+
+QUnit.test("Test createVertex", function(assert) {
+    var graph = new GRAPH.Graph();
+    var vert1 = graph.createVertex("A", { blah: 1 });
+    assert.ok(vert1 instanceof GRAPH.Vertex);
+    assert.ok(vert1 === graph.vertices.A);
+    assert.ok(graph.createVertex("A", { blah: 1 }) === vert1);
+});
+
+QUnit.test("Test createEdge", function(assert) {
+    var graph, A, B, edge;
+
+    graph = new GRAPH.Graph();
+    A = graph.createVertex("A");
+    B = graph.createVertex("B");
+    edge = graph.createEdge("A", "B");
+    assert.ok(edge instanceof GRAPH.Edge);
+    assert.ok(A.degree === 1);
+    assert.ok(B.degree === 1);
+    assert.ok(A.edges.B);
+    assert.ok(B.edges.A);
+    assert.ok(graph.numEdges === 2);
+
+    // bad target
+    graph = new GRAPH.Graph();
+    A = graph.createVertex("A");
+    B = graph.createVertex("B");
+    edge = graph.createEdge("A", "C");
+    assert.ok(edge === null);
+    assert.ok(A.degree === 0);
+    assert.ok(B.degree === 0);
+    assert.ok(!A.edges.B);
+    assert.ok(!B.edges.A);
+    assert.ok(graph.numEdges === 0);
+
+    // bad source
+    graph = new GRAPH.Graph();
+    A = graph.createVertex("A");
+    B = graph.createVertex("B");
+    edge = graph.createEdge("C", "A");
+    assert.ok(graph.numEdges === 0);
+
+    // reflexive
+    graph = new GRAPH.Graph();
+    A = graph.createVertex("A");
+    B = graph.createVertex("B");
+    edge = graph.createEdge("A", "A");
+    assert.ok(graph.numEdges === 0);
+
+    // reflexive (allowed)
+    graph = new GRAPH.Graph(false, {}, { allowReflexiveEdges: true });
+    A = graph.createVertex("A");
+    edge = graph.createEdge("A", "A");
+    // reflexive edges shouldn't mirror
+    assert.ok(graph.numEdges === 1);
+    assert.ok(A.edges.A);
+});
+
+QUnit.test("Test graph.edges", function(assert) {
+    var graph = new GRAPH.Graph(false, nodeLinkData);
+    assert.ok(graph.edges().length === 6);
+    assert.deepEqual(
+        graph.edges(function(e) {
+            return e.source;
+        }),
+        ["A", "A", "B", "B", "C", "C"]
+    );
+    assert.deepEqual(graph.edges({ source: "A" }), [graph.vertices.A.edges.B, graph.vertices.A.edges.C]);
+
+    graph = new GRAPH.Graph(true, nodeLinkData);
+    assert.ok(graph.edges().length === 3);
+    assert.deepEqual(
+        graph.edges(function(e) {
+            return e.source;
+        }),
+        ["A", "A", "B"]
+    );
+    assert.deepEqual(graph.edges({ source: "A" }), [graph.vertices.A.edges.B, graph.vertices.A.edges.C]);
+});
+
+QUnit.test("Test graph.adjacent", function(assert) {
+    var graph = new GRAPH.Graph(true, nodeLinkData);
+    assert.deepEqual(graph.adjacent(graph.vertices.A), [graph.vertices.B, graph.vertices.C]);
+    assert.deepEqual(graph.adjacent(graph.vertices.B), [graph.vertices.C]);
+    assert.deepEqual(graph.adjacent(graph.vertices.C), []);
+});
+
+QUnit.test("Test graph.eachAdjacent", function(assert) {
+    var graph = new GRAPH.Graph(true, nodeLinkData);
+    assert.deepEqual(
+        graph.eachAdjacent(graph.vertices.A, function(v, e) {
+            return v;
+        }),
+        [graph.vertices.B, graph.vertices.C]
+    );
+});
+
+// ------------------------------------------------------------------------ breadth first search
+QUnit.test("Empty BreadthFirstSearch", function(assert) {
+    var search = new GRAPH.BreadthFirstSearch();
+    assert.ok(search instanceof GRAPH.BreadthFirstSearch);
+    assert.ok(search.graph === undefined);
+    assert.ok(typeof search.processFns === "object");
+    assert.ok(typeof search.processFns.vertexEarly === "function");
+    assert.ok(typeof search.processFns.edge === "function");
+    assert.ok(typeof search.processFns.vertexLate === "function");
+    assert.ok(typeof search._cache === "object");
+});
+
+QUnit.test("BreadthFirstSearch on undirected graph", function(assert) {
+    var graph = new GRAPH.Graph(false, nodeLinkData),
+        bfs = new GRAPH.BreadthFirstSearch(graph);
+    assert.ok(bfs instanceof GRAPH.BreadthFirstSearch);
+    assert.ok(bfs.graph === graph);
+
+    var search = bfs.search("A"),
+        tree = bfs.searchTree("A");
+    assert.deepEqual(search, {
+        discovered: { A: true, B: true, C: true },
+        edges: [{ source: "A", target: "B" }, { source: "A", target: "C" }]
+    });
+    assert.ok(tree instanceof GRAPH.Graph);
+    assert.deepEqual(tree.vertices.A.toJSON(), graph.vertices.A.toJSON());
+    assert.deepEqual(
+        tree.eachVertex(function(v) {
+            return v.degree;
+        }),
+        [2, 0, 0]
+    );
+
+    assert.deepEqual(bfs.search("B").edges, [{ source: "B", target: "A" }, { source: "B", target: "C" }]);
+    assert.deepEqual(bfs.search("C").edges, [{ source: "C", target: "A" }, { source: "C", target: "B" }]);
+    assert.ok(typeof bfs._cache.A === "object");
+    assert.deepEqual(Object.keys(bfs._cache), ["A", "B", "C"]);
+});
+
+QUnit.test("BreadthFirstSearch on directed graph", function(assert) {
+    var graph = new GRAPH.Graph(true, nodeLinkData),
+        bfs = new GRAPH.BreadthFirstSearch(graph);
+    assert.ok(bfs instanceof GRAPH.BreadthFirstSearch);
+    assert.ok(bfs.graph === graph);
+
+    var search = bfs.search("A"),
+        tree = bfs.searchTree("A");
+    assert.deepEqual(search, {
+        discovered: { A: true, B: true, C: true },
+        edges: [{ source: "A", target: "B" }, { source: "A", target: "C" }]
+    });
+    assert.ok(tree instanceof GRAPH.Graph);
+    assert.deepEqual(tree.vertices.A.toJSON(), graph.vertices.A.toJSON());
+    assert.deepEqual(
+        tree.eachVertex(function(v) {
+            return v.degree;
+        }),
+        [2, 0, 0]
+    );
+
+    assert.deepEqual(bfs.search("B").edges, [{ source: "B", target: "C" }]);
+    assert.deepEqual(bfs.search("C").edges, []);
+    assert.ok(typeof bfs._cache.A === "object");
+    assert.deepEqual(Object.keys(bfs._cache), ["A", "B", "C"]);
+});
+
+// ------------------------------------------------------------------------ depth first search
+var DFSData = {
+    vertices: [{ name: "A" }, { name: "B" }, { name: "C" }, { name: "D" }, { name: "E" }, { name: "F" }],
+    edges: [
+        { source: "A", target: "B" },
+        { source: "B", target: "C" },
+        // confound it
+        { source: "A", target: "C" },
+        { source: "C", target: "D" },
+        { source: "A", target: "E" },
+        { source: "E", target: "F" },
+        // confound it
+        { source: "F", target: "A" }
+    ]
+};
+
+QUnit.test("Empty DepthFirstSearch", function(assert) {
+    var search = new GRAPH.DepthFirstSearch();
+    assert.ok(search instanceof GRAPH.DepthFirstSearch);
+    assert.ok(search.graph === undefined);
+    assert.ok(typeof search.processFns === "object");
+    assert.ok(typeof search.processFns.vertexEarly === "function");
+    assert.ok(typeof search.processFns.edge === "function");
+    assert.ok(typeof search.processFns.vertexLate === "function");
+    assert.ok(typeof search._cache === "object");
+});
+
+QUnit.test("DepthFirstSearch on undirected graph", function(assert) {
+    var graph = new GRAPH.Graph(false, DFSData),
+        dfs = new GRAPH.DepthFirstSearch(graph);
+    assert.ok(dfs instanceof GRAPH.DepthFirstSearch);
+    assert.ok(dfs.graph === graph);
+
+    var search = dfs.search("A"),
+        tree = dfs.searchTree("A");
+    assert.deepEqual(search, {
+        discovered: { A: true, B: true, C: true, D: true, E: true, F: true },
+        edges: [
+            { source: "A", target: "B" },
+            { source: "B", target: "C" },
+            { source: "C", target: "D" },
+            { source: "A", target: "E" },
+            { source: "E", target: "F" }
+        ],
+        entryTimes: { A: 0, B: 1, C: 2, D: 3, E: 7, F: 8 },
+        exitTimes: { A: 11, B: 6, C: 5, D: 4, E: 10, F: 9 }
+    });
+    assert.ok(tree instanceof GRAPH.Graph);
+    assert.deepEqual(tree.vertices.A.toJSON(), graph.vertices.A.toJSON());
+    assert.deepEqual(
+        tree.eachVertex(function(v) {
+            return v.degree;
+        }),
+        [2, 1, 1, 0, 1, 0]
+    );
+
+    assert.deepEqual(dfs.search("B").edges, [
+        { source: "B", target: "A" },
+        { source: "A", target: "C" },
+        { source: "C", target: "D" },
+        { source: "A", target: "E" },
+        { source: "E", target: "F" }
+    ]);
+
+    assert.ok(typeof dfs._cache.A === "object");
+    assert.deepEqual(Object.keys(dfs._cache), ["A", "B"]);
+});
+
+QUnit.test("DepthFirstSearch on directed graph", function(assert) {
+    var graph = new GRAPH.Graph(true, DFSData),
+        dfs = new GRAPH.DepthFirstSearch(graph);
+    assert.ok(dfs instanceof GRAPH.DepthFirstSearch);
+    assert.ok(dfs.graph === graph);
+
+    var search = dfs.search("A"),
+        tree = dfs.searchTree("A");
+    assert.deepEqual(search, {
+        discovered: { A: true, B: true, C: true, D: true, E: true, F: true },
+        edges: [
+            { source: "A", target: "B" },
+            { source: "B", target: "C" },
+            { source: "C", target: "D" },
+            { source: "A", target: "E" },
+            { source: "E", target: "F" }
+        ],
+        entryTimes: { A: 0, B: 1, C: 2, D: 3, E: 7, F: 8 },
+        exitTimes: { A: 11, B: 6, C: 5, D: 4, E: 10, F: 9 }
+    });
+    assert.ok(tree instanceof GRAPH.Graph);
+    assert.deepEqual(tree.vertices.A.toJSON(), graph.vertices.A.toJSON());
+    assert.deepEqual(
+        tree.eachVertex(function(v) {
+            return v.degree;
+        }),
+        [2, 1, 1, 0, 1, 0]
+    );
+
+    assert.deepEqual(dfs.search("B").edges, [{ source: "B", target: "C" }, { source: "C", target: "D" }]);
+
+    assert.ok(typeof dfs._cache.A === "object");
+    assert.deepEqual(Object.keys(dfs._cache), ["A", "B"]);
+});
+
+// ------------------------------------------------------------------------ components
+//*/
+QUnit.test("weakComponents on undirected graph", function(assert) {
+    var graph = new GRAPH.Graph(false, {
+        vertices: [{ name: "A" }, { name: "B" }, { name: "C" }, { name: "D" }, { name: "E" }],
+        edges: [{ source: "A", target: "B" }, { source: "C", target: "D" }]
+    });
+    assert.equal(graph.numEdges, 4);
+    var components = graph.weakComponents();
+    assert.equal(components.length, 3);
+});
+
+QUnit.test("weakComponents on directed graph", function(assert) {
+    var graph, components;
+    graph = new GRAPH.Graph(true, {
+        vertices: [{ name: "A" }, { name: "B" }, { name: "C" }, { name: "D" }, { name: "E" }],
+        edges: [{ source: "A", target: "B" }, { source: "D", target: "C" }]
+    });
+    assert.equal(graph.numEdges, 2);
+
+    components = graph.weakComponents();
+    assert.equal(components.length, 3);
+
+    graph = new GRAPH.Graph(true, {
+        vertices: [
+            { name: "A", data: 100 },
+            { name: "B", data: 200 },
+            { name: "C", data: 30 },
+            { name: "D", data: 40 },
+            { name: "E", data: 500 },
+            { name: "F", data: 600 },
+            { name: "G", data: 7 }
+        ],
+        edges: [
+            { source: "A", target: "B" },
+            { source: "D", target: "C" },
+            { source: "E", target: "A" },
+            { source: "F", target: "E" }
+        ]
+    });
+    components = graph.weakComponents();
+    assert.equal(components.length, 3);
+    // data retained
+    assert.equal(components[0].vertices[0].data, 100);
+
+    assert.equal(components[0].vertices.length, 4);
+    assert.deepEqual(components[0].edges, [
+        { source: "A", target: "B" },
+        { source: "E", target: "A" },
+        { source: "F", target: "E" }
+    ]);
+
+    assert.equal(components[1].vertices.length, 2);
+    assert.deepEqual(components[1].edges, [{ source: "D", target: "C" }]);
+
+    assert.deepEqual(components[2].vertices, [{ name: "G", data: 7 }]);
+    assert.deepEqual(components[2].edges.length, 0);
+});

client/galaxy/scripts/qunit/tests/hda-base.js

@@ -1,30 +0,0 @@
-// This file isn't really testing anything useful yet, it is just testing
-// (or demonstrating) qunit+backbone interactions.
-/* global define */
-define(["qunit/test-app", "mvc/history/hda-model", "mvc/history/hda-li", "jquery"], function(
-    testApp,
-    HDA_MODEL,
-    HDA_BASE,
-    $
-) {
-    "use strict";
-    HDA_MODEL = HDA_MODEL.default;
-    HDA_BASE = HDA_BASE.default;
-    QUnit.module("HDA base backbone view tests");
-
-    QUnit.test("Base HDA view default construction, initialize", function(assert) {
-        var hda = new HDA_MODEL.HistoryDatasetAssociation({
-                id: "123"
-            }),
-            view = new HDA_BASE.HDAListItemView({ model: hda });
-
-        assert.strictEqual(view.model, hda);
-
-        assert.equal(view.linkTarget, "_blank");
-        assert.equal(view.selectable, false);
-        assert.equal(view.selected, false);
-        assert.equal(view.expanded, false);
-        assert.equal(view.draggable, false);
-        assert.equal(view.id(), "dataset-123");
-    });
-});

client/galaxy/scripts/qunit/tests/hda_base_tests.js

@@ -0,0 +1,25 @@
+// This file isn't really testing anything useful yet, it is just testing
+// (or demonstrating) qunit+backbone interactions.
+/* global define */
+import testApp from "qunit/test-app";
+import HDA_MODEL from "mvc/history/hda-model";
+import HDA_BASE from "mvc/history/hda-li";
+import $ from "jquery";
+
+QUnit.module("HDA base backbone view tests");
+
+QUnit.test("Base HDA view default construction, initialize", function(assert) {
+    var hda = new HDA_MODEL.HistoryDatasetAssociation({
+            id: "123"
+        }),
+        view = new HDA_BASE.HDAListItemView({ model: hda });
+
+    assert.strictEqual(view.model, hda);
+
+    assert.equal(view.linkTarget, "_blank");
+    assert.equal(view.selectable, false);
+    assert.equal(view.selected, false);
+    assert.equal(view.expanded, false);
+    assert.equal(view.draggable, false);
+    assert.equal(view.id(), "dataset-123");
+});

client/galaxy/scripts/qunit/tests/history_contents_model_tests.js

@@ -1,38 +1,40 @@
 /* global define */
-define(["qunit/test-app", "mvc/history/hda-model", "jquery", "sinon"], function(testApp, HDA_MODEL, $, sinon) {
-    HDA_MODEL = HDA_MODEL.default;
-    QUnit.module("History Contents Model QUnit.Tests");
+import testApp from "qunit/test-app";
+import HDA_MODEL from "mvc/history/hda-model";
+import sinon from "sinon";
+import $ from "jquery";
 
-    QUnit.test("HDA Constructions with Default Attributes", function(assert) {
-        var hda = new HDA_MODEL.HistoryDatasetAssociation({});
-        assert.equal(hda.get("name"), "(unnamed dataset)");
-        assert.equal(hda.get("state"), "new");
-    });
+QUnit.module("History Contents Model QUnit.Tests");
 
-    QUnit.test("HDA Construction with Supplied Attributes", function(assert) {
-        var hda = new HDA_MODEL.HistoryDatasetAssociation({
-            history_content_type: "dataset",
-            name: "my dataset",
-            state: "ok"
-        });
-        assert.equal(hda.get("name"), "my dataset");
-        assert.equal(hda.get("state"), "ok");
+QUnit.test("HDA Constructions with Default Attributes", function(assert) {
+    var hda = new HDA_MODEL.HistoryDatasetAssociation({});
+    assert.equal(hda.get("name"), "(unnamed dataset)");
+    assert.equal(hda.get("state"), "new");
+});
+
+QUnit.test("HDA Construction with Supplied Attributes", function(assert) {
+    var hda = new HDA_MODEL.HistoryDatasetAssociation({
+        history_content_type: "dataset",
+        name: "my dataset",
+        state: "ok"
     });
+    assert.equal(hda.get("name"), "my dataset");
+    assert.equal(hda.get("state"), "ok");
+});
 
-    QUnit.test("HDA Deletion", function(assert) {
-        var hda = new HDA_MODEL.HistoryDatasetAssociation({
-            history_content_type: "dataset",
-            id: "hda1",
-            history_id: "h1",
-            deleted: false
-        });
-        assert.equal(hda.get("deleted"), false);
+QUnit.test("HDA Deletion", function(assert) {
+    var hda = new HDA_MODEL.HistoryDatasetAssociation({
+        history_content_type: "dataset",
+        id: "hda1",
+        history_id: "h1",
+        deleted: false
+    });
+    assert.equal(hda.get("deleted"), false);
 
-        sinon.stub($, "ajax").yieldsTo("success", { deleted: true });
-        hda["delete"]();
-        // to get the url sinon used:
-        //console.debug( $.ajax.lastCall.args[0].url )
-        assert.ok($.ajax.calledWithMatch({ url: "/api/histories/h1/contents/datasets/hda1" }));
-        assert.equal(hda.get("deleted"), true);
-    });
+    sinon.stub($, "ajax").yieldsTo("success", { deleted: true });
+    hda["delete"]();
+    // to get the url sinon used:
+    //console.debug( $.ajax.lastCall.args[0].url )
+    assert.ok($.ajax.calledWithMatch({ url: "/api/histories/h1/contents/datasets/hda1" }));
+    assert.equal(hda.get("deleted"), true);
 });

client/galaxy/scripts/qunit/tests/job-dag.js

@@ -1,294 +0,0 @@
-/* global define */
-define(["qunit/test-app", "mvc/history/job-dag", "jquery", "qunit/test-data/job-dag-1"], function(
-    testApp,
-    JobDAG,
-    $,
-    testData
-) {
-    "use strict";
-    JobDAG = JobDAG.default;
-    QUnit.module("mvc/history/job-dag.js tests");
-
-    // ------------------------------------------------------------------------
-    QUnit.test("Empty JobDAG construction", function(assert) {
-        var dag = new JobDAG();
-        assert.ok(dag instanceof JobDAG);
-
-        // default options
-        assert.deepEqual(dag.filters, []);
-        assert.deepEqual(dag.options, {
-            excludeSetMetadata: false
-        });
-
-        // test (empty) instance vars
-        assert.deepEqual(dag._jobsData, []);
-        assert.deepEqual(dag._historyContentsMap, {});
-        assert.deepEqual(dag._toolMap, {});
-        assert.equal(dag.noInputJobs.length, 0);
-        assert.equal(dag.noOutputJobs.length, 0);
-
-        // logging
-        assert.equal(typeof dag.debug, "function");
-        assert.equal(typeof dag.info, "function");
-        assert.equal(typeof dag.warn, "function");
-        assert.equal(typeof dag.error, "function");
-    });
-
-    QUnit.test("Empty JobDAG construction - changing options", function(assert) {
-        var dag;
-        dag = new JobDAG({
-            excludeSetMetadata: true
-        });
-
-        // excludeSetMetadata
-        assert.deepEqual(dag.options, {
-            excludeSetMetadata: true
-        });
-        assert.equal(dag.filters.length, 1);
-        assert.equal(typeof dag.filters[0], "function");
-
-        // filters
-        function testFilter(job, index, jobData) {
-            return true;
-        }
-        dag = new JobDAG({
-            filters: [testFilter]
-        });
-        assert.equal(dag.filters[0], testFilter);
-    });
-
-    QUnit.test("JobDAG construction with history and jobs", function(assert) {
-        assert.equal(testData.jobs1.length, 3);
-        assert.equal(testData.historyContents1.length, 3);
-
-        var history = testData.historyContents1,
-            jobs = testData.jobs1,
-            dag;
-        dag = new JobDAG({
-            historyContents: history,
-            tools: testData.tools,
-            jobs: jobs
-        });
-
-        assert.deepEqual(dag._outputIdToJobMap, {
-            "8c959c9304a2bc4b": "8a81cf6f989c4467",
-            "132016f833b57406": "6505e875ddb66fd2",
-            "846fb0a2a64137c0": "77f74776fd03cbc5"
-        });
-        assert.deepEqual(dag._jobsData, [
-            {
-                job: _.findWhere(jobs, { id: "8a81cf6f989c4467" }),
-                inputs: {},
-                outputs: {
-                    "8c959c9304a2bc4b": {
-                        src: "hda",
-                        id: "8c959c9304a2bc4b",
-                        name: "output0",
-                        content: _.findWhere(history, { id: "8c959c9304a2bc4b" })
-                    }
-                },
-                tool: {}
-            },
-            {
-                job: _.findWhere(jobs, { id: "6505e875ddb66fd2" }),
-                inputs: {
-                    "8c959c9304a2bc4b": {
-                        src: "hda",
-                        id: "8c959c9304a2bc4b",
-                        name: "input",
-                        content: _.findWhere(history, { id: "8c959c9304a2bc4b" })
-                    }
-                },
-                outputs: {
-                    "132016f833b57406": {
-                        src: "hda",
-                        id: "132016f833b57406",
-                        name: "out_file1",
-                        content: _.findWhere(history, { id: "132016f833b57406" })
-                    }
-                },
-                tool: {}
-            },
-            {
-                job: _.findWhere(jobs, { id: "77f74776fd03cbc5" }),
-                inputs: {
-                    "132016f833b57406": {
-                        src: "hda",
-                        id: "132016f833b57406",
-                        name: "input",
-                        content: _.findWhere(history, { id: "132016f833b57406" })
-                    }
-                },
-                outputs: {
-                    "846fb0a2a64137c0": {
-                        src: "hda",
-                        id: "846fb0a2a64137c0",
-                        name: "out_file1",
-                        content: _.findWhere(history, { id: "846fb0a2a64137c0" })
-                    }
-                },
-                tool: {}
-            }
-        ]);
-
-        var jobsDataMap = dag._jobsDataMap();
-        assert.deepEqual(dag.toNodesAndLinks(), {
-            nodes: [
-                { name: "8a81cf6f989c4467", data: jobsDataMap["8a81cf6f989c4467"] },
-                { name: "6505e875ddb66fd2", data: jobsDataMap["6505e875ddb66fd2"] },
-                { name: "77f74776fd03cbc5", data: jobsDataMap["77f74776fd03cbc5"] }
-            ],
-            links: [
-                { source: 0, target: 1, data: { dataset: "8c959c9304a2bc4b" } },
-                { source: 1, target: 2, data: { dataset: "132016f833b57406" } }
-            ]
-        });
-
-        assert.deepEqual(dag.toVerticesAndEdges(), {
-            vertices: [
-                { name: "8a81cf6f989c4467", data: jobsDataMap["8a81cf6f989c4467"] },
-                { name: "6505e875ddb66fd2", data: jobsDataMap["6505e875ddb66fd2"] },
-                { name: "77f74776fd03cbc5", data: jobsDataMap["77f74776fd03cbc5"] }
-            ],
-            edges: [
-                {
-                    source: "8a81cf6f989c4467",
-                    target: "6505e875ddb66fd2",
-                    data: { dataset: "8c959c9304a2bc4b" }
-                },
-                {
-                    source: "6505e875ddb66fd2",
-                    target: "77f74776fd03cbc5",
-                    data: { dataset: "132016f833b57406" }
-                }
-            ]
-        });
-
-        // test cloning
-    });
-
-    QUnit.test("JobDAG removal of __SET_METADATA__ jobs", function(assert) {
-        assert.equal(testData.jobs2.length, 3);
-        assert.equal(testData.historyContents2.length, 2);
-
-        var history = testData.historyContents2,
-            jobs = testData.jobs2,
-            dag;
-        dag = new JobDAG({
-            historyContents: history,
-            tools: testData.tools,
-            jobs: jobs,
-            excludeSetMetadata: true
-        });
-
-        var jobsDataMap = dag._jobsDataMap();
-        assert.deepEqual(dag.toNodesAndLinks(), {
-            nodes: [
-                { name: "bf60fd5f5f7f44bf", data: jobsDataMap["bf60fd5f5f7f44bf"] },
-                { name: "90240358ebde1489", data: jobsDataMap["90240358ebde1489"] }
-            ],
-            links: [{ source: 0, target: 1, data: { dataset: "eca0af6fb47bf90c" } }]
-        });
-    });
-
-    //TODO: test filtering out errored jobs
-    QUnit.test("JobDAG construction with history and jobs", function(assert) {
-        assert.equal(testData.jobs3.length, 5);
-        assert.equal(testData.historyContents3.length, 5);
-
-        var history = testData.historyContents3,
-            jobs = testData.jobs3,
-            dag;
-        dag = new JobDAG({
-            historyContents: history,
-            tools: testData.tools,
-            jobs: jobs
-        });
-
-        var jobsDataMap = dag._jobsDataMap();
-        assert.deepEqual(dag.toVerticesAndEdges(), {
-            vertices: [
-                { name: "8c959c9304a2bc4b", data: jobsDataMap["8c959c9304a2bc4b"] },
-                { name: "132016f833b57406", data: jobsDataMap["132016f833b57406"] },
-                { name: "846fb0a2a64137c0", data: jobsDataMap["846fb0a2a64137c0"] },
-                { name: "eca0af6fb47bf90c", data: jobsDataMap["eca0af6fb47bf90c"] },
-                { name: "6fc9fbb81c497f69", data: jobsDataMap["6fc9fbb81c497f69"] }
-            ],
-            edges: [
-                {
-                    source: "8c959c9304a2bc4b",
-                    target: "846fb0a2a64137c0",
-                    data: { dataset: "6fb17d0cc6e8fae5" }
-                },
-                {
-                    source: "132016f833b57406",
-                    target: "eca0af6fb47bf90c",
-                    data: { dataset: "5114a2a207b7caff" }
-                },
-                {
-                    source: "eca0af6fb47bf90c",
-                    target: "6fc9fbb81c497f69",
-                    data: { dataset: "b8a0d6158b9961df" }
-                }
-            ]
-        });
-
-        var components = dag.weakComponents();
-        assert.deepEqual(components, [
-            {
-                vertices: [
-                    { name: "8c959c9304a2bc4b", data: jobsDataMap["8c959c9304a2bc4b"] },
-                    { name: "846fb0a2a64137c0", data: jobsDataMap["846fb0a2a64137c0"] }
-                ],
-                edges: [{ source: "8c959c9304a2bc4b", target: "846fb0a2a64137c0" }]
-            },
-            {
-                vertices: [
-                    { name: "132016f833b57406", data: jobsDataMap["132016f833b57406"] },
-                    { name: "eca0af6fb47bf90c", data: jobsDataMap["eca0af6fb47bf90c"] },
-                    { name: "6fc9fbb81c497f69", data: jobsDataMap["6fc9fbb81c497f69"] }
-                ],
-                edges: [
-                    { source: "132016f833b57406", target: "eca0af6fb47bf90c" },
-                    { source: "eca0af6fb47bf90c", target: "6fc9fbb81c497f69" }
-                ]
-            }
-        ]);
-    });
-
-    //TODO: test filtering out errored jobs
-    QUnit.test("JobDAG construction with copied history contents", function(assert) {
-        assert.equal(testData.jobs4.length, 1);
-        assert.equal(testData.historyContents4.length, 3);
-
-        var history = testData.historyContents4,
-            jobs = testData.jobs4,
-            dag;
-        dag = new JobDAG({
-            historyContents: history,
-            tools: testData.tools,
-            jobs: jobs
-        });
-
-        var jobsDataMap = dag._jobsDataMap();
-        assert.deepEqual(dag.toVerticesAndEdges(), {
-            vertices: [
-                { name: "92b83968e0b52980", data: jobsDataMap["92b83968e0b52980"] },
-                { name: "copy-422eef6b1b545329", data: _.findWhere(history, { id: "422eef6b1b545329" }) },
-                { name: "copy-c86c1b73aa7102dd", data: _.findWhere(history, { id: "c86c1b73aa7102dd" }) }
-            ],
-            edges: [
-                {
-                    source: "copy-422eef6b1b545329",
-                    target: "92b83968e0b52980",
-                    data: { dataset: "422eef6b1b545329" }
-                },
-                {
-                    source: "copy-c86c1b73aa7102dd",
-                    target: "92b83968e0b52980",
-                    data: { dataset: "c86c1b73aa7102dd" }
-                }
-            ]
-        });
-    });
-});

client/galaxy/scripts/qunit/tests/job_dag_tests.js

@@ -0,0 +1,290 @@
+/* global define */
+import testApp from "qunit/test-app";
+import JobDAG from "mvc/history/job-dag";
+import testData from "qunit/test-data/job-dag-1";
+import $ from "jquery";
+
+QUnit.module("mvc/history/job-dag.js tests");
+
+// ------------------------------------------------------------------------
+QUnit.test("Empty JobDAG construction", function(assert) {
+    var dag = new JobDAG();
+    assert.ok(dag instanceof JobDAG);
+
+    // default options
+    assert.deepEqual(dag.filters, []);
+    assert.deepEqual(dag.options, {
+        excludeSetMetadata: false
+    });
+
+    // test (empty) instance vars
+    assert.deepEqual(dag._jobsData, []);
+    assert.deepEqual(dag._historyContentsMap, {});
+    assert.deepEqual(dag._toolMap, {});
+    assert.equal(dag.noInputJobs.length, 0);
+    assert.equal(dag.noOutputJobs.length, 0);
+
+    // logging
+    assert.equal(typeof dag.debug, "function");
+    assert.equal(typeof dag.info, "function");
+    assert.equal(typeof dag.warn, "function");
+    assert.equal(typeof dag.error, "function");
+});
+
+QUnit.test("Empty JobDAG construction - changing options", function(assert) {
+    var dag;
+    dag = new JobDAG({
+        excludeSetMetadata: true
+    });
+
+    // excludeSetMetadata
+    assert.deepEqual(dag.options, {
+        excludeSetMetadata: true
+    });
+    assert.equal(dag.filters.length, 1);
+    assert.equal(typeof dag.filters[0], "function");
+
+    // filters
+    function testFilter(job, index, jobData) {
+        return true;
+    }
+    dag = new JobDAG({
+        filters: [testFilter]
+    });
+    assert.equal(dag.filters[0], testFilter);
+});
+
+QUnit.test("JobDAG construction with history and jobs", function(assert) {
+    assert.equal(testData.jobs1.length, 3);
+    assert.equal(testData.historyContents1.length, 3);
+
+    var history = testData.historyContents1,
+        jobs = testData.jobs1,
+        dag;
+    dag = new JobDAG({
+        historyContents: history,
+        tools: testData.tools,
+        jobs: jobs
+    });
+
+    assert.deepEqual(dag._outputIdToJobMap, {
+        "8c959c9304a2bc4b": "8a81cf6f989c4467",
+        "132016f833b57406": "6505e875ddb66fd2",
+        "846fb0a2a64137c0": "77f74776fd03cbc5"
+    });
+    assert.deepEqual(dag._jobsData, [
+        {
+            job: _.findWhere(jobs, { id: "8a81cf6f989c4467" }),
+            inputs: {},
+            outputs: {
+                "8c959c9304a2bc4b": {
+                    src: "hda",
+                    id: "8c959c9304a2bc4b",
+                    name: "output0",
+                    content: _.findWhere(history, { id: "8c959c9304a2bc4b" })
+                }
+            },
+            tool: {}
+        },
+        {
+            job: _.findWhere(jobs, { id: "6505e875ddb66fd2" }),
+            inputs: {
+                "8c959c9304a2bc4b": {
+                    src: "hda",
+                    id: "8c959c9304a2bc4b",
+                    name: "input",
+                    content: _.findWhere(history, { id: "8c959c9304a2bc4b" })
+                }
+            },
+            outputs: {
+                "132016f833b57406": {
+                    src: "hda",
+                    id: "132016f833b57406",
+                    name: "out_file1",
+                    content: _.findWhere(history, { id: "132016f833b57406" })
+                }
+            },
+            tool: {}
+        },
+        {
+            job: _.findWhere(jobs, { id: "77f74776fd03cbc5" }),
+            inputs: {
+                "132016f833b57406": {
+                    src: "hda",
+                    id: "132016f833b57406",
+                    name: "input",
+                    content: _.findWhere(history, { id: "132016f833b57406" })
+                }
+            },
+            outputs: {
+                "846fb0a2a64137c0": {
+                    src: "hda",
+                    id: "846fb0a2a64137c0",
+                    name: "out_file1",
+                    content: _.findWhere(history, { id: "846fb0a2a64137c0" })
+                }
+            },
+            tool: {}
+        }
+    ]);
+
+    var jobsDataMap = dag._jobsDataMap();
+    assert.deepEqual(dag.toNodesAndLinks(), {
+        nodes: [
+            { name: "8a81cf6f989c4467", data: jobsDataMap["8a81cf6f989c4467"] },
+            { name: "6505e875ddb66fd2", data: jobsDataMap["6505e875ddb66fd2"] },
+            { name: "77f74776fd03cbc5", data: jobsDataMap["77f74776fd03cbc5"] }
+        ],
+        links: [
+            { source: 0, target: 1, data: { dataset: "8c959c9304a2bc4b" } },
+            { source: 1, target: 2, data: { dataset: "132016f833b57406" } }
+        ]
+    });
+
+    assert.deepEqual(dag.toVerticesAndEdges(), {
+        vertices: [
+            { name: "8a81cf6f989c4467", data: jobsDataMap["8a81cf6f989c4467"] },
+            { name: "6505e875ddb66fd2", data: jobsDataMap["6505e875ddb66fd2"] },
+            { name: "77f74776fd03cbc5", data: jobsDataMap["77f74776fd03cbc5"] }
+        ],
+        edges: [
+            {
+                source: "8a81cf6f989c4467",
+                target: "6505e875ddb66fd2",
+                data: { dataset: "8c959c9304a2bc4b" }
+            },
+            {
+                source: "6505e875ddb66fd2",
+                target: "77f74776fd03cbc5",
+                data: { dataset: "132016f833b57406" }
+            }
+        ]
+    });
+
+    // test cloning
+});
+
+QUnit.test("JobDAG removal of __SET_METADATA__ jobs", function(assert) {
+    assert.equal(testData.jobs2.length, 3);
+    assert.equal(testData.historyContents2.length, 2);
+
+    var history = testData.historyContents2,
+        jobs = testData.jobs2,
+        dag;
+    dag = new JobDAG({
+        historyContents: history,
+        tools: testData.tools,
+        jobs: jobs,
+        excludeSetMetadata: true
+    });
+
+    var jobsDataMap = dag._jobsDataMap();
+    assert.deepEqual(dag.toNodesAndLinks(), {
+        nodes: [
+            { name: "bf60fd5f5f7f44bf", data: jobsDataMap["bf60fd5f5f7f44bf"] },
+            { name: "90240358ebde1489", data: jobsDataMap["90240358ebde1489"] }
+        ],
+        links: [{ source: 0, target: 1, data: { dataset: "eca0af6fb47bf90c" } }]
+    });
+});
+
+//TODO: test filtering out errored jobs
+QUnit.test("JobDAG construction with history and jobs", function(assert) {
+    assert.equal(testData.jobs3.length, 5);
+    assert.equal(testData.historyContents3.length, 5);
+
+    var history = testData.historyContents3,
+        jobs = testData.jobs3,
+        dag;
+    dag = new JobDAG({
+        historyContents: history,
+        tools: testData.tools,
+        jobs: jobs
+    });
+
+    var jobsDataMap = dag._jobsDataMap();
+    assert.deepEqual(dag.toVerticesAndEdges(), {
+        vertices: [
+            { name: "8c959c9304a2bc4b", data: jobsDataMap["8c959c9304a2bc4b"] },
+            { name: "132016f833b57406", data: jobsDataMap["132016f833b57406"] },
+            { name: "846fb0a2a64137c0", data: jobsDataMap["846fb0a2a64137c0"] },
+            { name: "eca0af6fb47bf90c", data: jobsDataMap["eca0af6fb47bf90c"] },
+            { name: "6fc9fbb81c497f69", data: jobsDataMap["6fc9fbb81c497f69"] }
+        ],
+        edges: [
+            {
+                source: "8c959c9304a2bc4b",
+                target: "846fb0a2a64137c0",
+                data: { dataset: "6fb17d0cc6e8fae5" }
+            },
+            {
+                source: "132016f833b57406",
+                target: "eca0af6fb47bf90c",
+                data: { dataset: "5114a2a207b7caff" }
+            },
+            {
+                source: "eca0af6fb47bf90c",
+                target: "6fc9fbb81c497f69",
+                data: { dataset: "b8a0d6158b9961df" }
+            }
+        ]
+    });
+
+    var components = dag.weakComponents();
+    assert.deepEqual(components, [
+        {
+            vertices: [
+                { name: "8c959c9304a2bc4b", data: jobsDataMap["8c959c9304a2bc4b"] },
+                { name: "846fb0a2a64137c0", data: jobsDataMap["846fb0a2a64137c0"] }
+            ],
+            edges: [{ source: "8c959c9304a2bc4b", target: "846fb0a2a64137c0" }]
+        },
+        {
+            vertices: [
+                { name: "132016f833b57406", data: jobsDataMap["132016f833b57406"] },
+                { name: "eca0af6fb47bf90c", data: jobsDataMap["eca0af6fb47bf90c"] },
+                { name: "6fc9fbb81c497f69", data: jobsDataMap["6fc9fbb81c497f69"] }
+            ],
+            edges: [
+                { source: "132016f833b57406", target: "eca0af6fb47bf90c" },
+                { source: "eca0af6fb47bf90c", target: "6fc9fbb81c497f69" }
+            ]
+        }
+    ]);
+});
+
+//TODO: test filtering out errored jobs
+QUnit.test("JobDAG construction with copied history contents", function(assert) {
+    assert.equal(testData.jobs4.length, 1);
+    assert.equal(testData.historyContents4.length, 3);
+
+    var history = testData.historyContents4,
+        jobs = testData.jobs4,
+        dag;
+    dag = new JobDAG({
+        historyContents: history,
+        tools: testData.tools,
+        jobs: jobs
+    });
+
+    var jobsDataMap = dag._jobsDataMap();
+    assert.deepEqual(dag.toVerticesAndEdges(), {
+        vertices: [
+            { name: "92b83968e0b52980", data: jobsDataMap["92b83968e0b52980"] },
+            { name: "copy-422eef6b1b545329", data: _.findWhere(history, { id: "422eef6b1b545329" }) },
+            { name: "copy-c86c1b73aa7102dd", data: _.findWhere(history, { id: "c86c1b73aa7102dd" }) }
+        ],
+        edges: [
+            {
+                source: "copy-422eef6b1b545329",
+                target: "92b83968e0b52980",
+                data: { dataset: "422eef6b1b545329" }
+            },
+            {
+                source: "copy-c86c1b73aa7102dd",
+                target: "92b83968e0b52980",
+                data: { dataset: "c86c1b73aa7102dd" }
+            }
+        ]
+    });
+});

client/galaxy/scripts/qunit/tests/list-of-pairs-collection-creator.js

@@ -1,107 +0,0 @@
-/* global define */
-define(
-    [
-        "qunit/test-app",
-        "mvc/collection/list-of-pairs-collection-creator",
-        "qunit/test-data/paired-collection-creator.data",
-        "jquery",
-        "sinon"
-    ],
-    function(testApp, PAIRED_COLLECTION_CREATOR, DATA, $, sinon) {
-        "use strict";
-        PAIRED_COLLECTION_CREATOR = PAIRED_COLLECTION_CREATOR.default;
-        var PCC = PAIRED_COLLECTION_CREATOR.PairedCollectionCreator;
-
-        QUnit.module("Galaxy client app tests");
-
-        QUnit.test("Collection creation", function(assert) {
-            var pcc = new PCC({
-                    datasets: DATA._1,
-                    historyId: "fakeHistoryId"
-                }),
-                server = sinon.fakeServer.create();
-
-            var requestJSON;
-            server.respondWith("POST", "/api/histories/fakeHistoryId/contents/dataset_collections", function(request) {
-                requestJSON = JSON.parse(request.requestBody);
-                request.respond(
-                    200,
-                    { "Content-Type": "application/json" },
-                    JSON.stringify({
-                        fakeResponse: "yes"
-                    })
-                );
-            });
-
-            //console.debug( 'requestBody:', JSON.stringify( requestJSON, null, '  ' ) );
-            pcc.createList("Heres a collection");
-            server.respond();
-            assert.deepEqual(requestJSON, DATA._1requestJSON);
-        });
-
-        QUnit.test("Creator base/empty construction/initializiation defaults", function(assert) {
-            var pcc = new PCC([]);
-            assert.ok(pcc instanceof PCC);
-            assert.deepEqual(pcc.filters, pcc.commonFilters[pcc.DEFAULT_FILTERS]);
-            assert.ok(pcc.automaticallyPair);
-            assert.equal(pcc.matchPercentage, 0.9);
-            assert.equal(pcc.strategy, "autopairLCS");
-        });
-
-        QUnit.test("Creator construction/initializiation with datasets", function(assert) {
-            var pcc = new PCC({
-                datasets: DATA._1
-            });
-            //pcc.initialList.forEach( function( dataset, i ){
-            //    console.log( i + ':\n' + JSON.stringify( dataset ) );
-            //});
-            // pcc maintains the original list - which, in this case, is already sorted
-            assert.deepEqual(pcc.initialList, DATA._1);
-            // datasets 1 has no ids, so the pcc will create them
-            assert.ok(
-                _.every(pcc.initialList, function(dataset) {
-                    return dataset.id;
-                })
-            );
-            // datasets 1 is very easy to auto pair
-            assert.equal(pcc.unpaired.length, 0);
-            assert.equal(pcc.paired.length, pcc.initialList.length / 2);
-        });
-
-        QUnit.test("Try easy autopairing with simple exact matching", function(assert) {
-            var pcc = new PCC({
-                datasets: DATA._1,
-                strategy: "simple",
-                twoPassAutopairing: false
-            });
-            assert.equal(pcc.unpaired.length, 0);
-            assert.equal(pcc.paired.length, pcc.initialList.length / 2);
-        });
-
-        QUnit.test("Try easy autopairing with LCS", function(assert) {
-            var pcc = new PCC({
-                datasets: DATA._1,
-                strategy: "lcs",
-                twoPassAutopairing: false
-            });
-            assert.equal(pcc.unpaired.length, 0);
-            assert.equal(pcc.paired.length, pcc.initialList.length / 2);
-        });
-
-        QUnit.test("Try easy autopairing with Levenshtein", function(assert) {
-            var pcc = new PCC({
-                datasets: DATA._1,
-                strategy: "levenshtein",
-                twoPassAutopairing: false
-            });
-            assert.equal(pcc.unpaired.length, 0);
-            assert.equal(pcc.paired.length, pcc.initialList.length / 2);
-        });
-
-        //TODO:
-        //  filters: clearing, setting via popover, regex
-        //  partition: maximize paired, maximize unpaired, split evenly
-        //  pairing: manually pairing and unpairing
-        //  misc: renaming pairs, removing file extensions
-    }
-);

client/galaxy/scripts/qunit/tests/list_of_pairs_collection_creator_tests.js

@@ -0,0 +1,108 @@
+/* global define */
+import testApp from "qunit/test-app";
+import PAIRED_COLLECTION_CREATOR from "mvc/collection/list-of-pairs-collection-creator";
+import DATA from "qunit/test-data/paired-collection-creator.data";
+import $ from "jquery";
+import sinon from "sinon";
+
+var PCC = PAIRED_COLLECTION_CREATOR.PairedCollectionCreator;
+
+QUnit.module("Galaxy client app tests");
+
+// Following test no longer passes - something is wrong setting up Galaxy.root - but we have multiple Selenium tests that cover verifying
+// actual such lists are created as a result of this form. The matching stuff later on in the
+// file is better suited for unit testing I think anyway and is not covered so well by
+// functional tests.
+
+/*
+QUnit.test("Collection creation", function(assert) {
+    var pcc = new PCC({
+            datasets: DATA._1,
+            historyId: "fakeHistoryId"
+        }),
+        server = sinon.fakeServer.create();
+
+    var requestJSON;
+    server.respondWith("POST", "/api/histories/fakeHistoryId/contents/dataset_collections", function(request) {
+        requestJSON = JSON.parse(request.requestBody);
+        request.respond(
+            200,
+            { "Content-Type": "application/json" },
+            JSON.stringify({
+                fakeResponse: "yes"
+            })
+        );
+    });
+
+    //console.debug( 'requestBody:', JSON.stringify( requestJSON, null, '  ' ) );
+    pcc.createList("Heres a collection");
+    server.respond();
+    assert.deepEqual(requestJSON, DATA._1requestJSON);
+    server.restore();
+});
+*/
+
+QUnit.test("Creator base/empty construction/initializiation defaults", function(assert) {
+    var pcc = new PCC([]);
+    assert.ok(pcc instanceof PCC);
+    assert.deepEqual(pcc.filters, pcc.commonFilters[pcc.DEFAULT_FILTERS]);
+    assert.ok(pcc.automaticallyPair);
+    assert.equal(pcc.matchPercentage, 0.9);
+    assert.equal(pcc.strategy, "autopairLCS");
+});
+
+QUnit.test("Creator construction/initializiation with datasets", function(assert) {
+    var pcc = new PCC({
+        datasets: DATA._1
+    });
+    //pcc.initialList.forEach( function( dataset, i ){
+    //    console.log( i + ':\n' + JSON.stringify( dataset ) );
+    //});
+    // pcc maintains the original list - which, in this case, is already sorted
+    assert.deepEqual(pcc.initialList, DATA._1);
+    // datasets 1 has no ids, so the pcc will create them
+    assert.ok(
+        _.every(pcc.initialList, function(dataset) {
+            return dataset.id;
+        })
+    );
+    // datasets 1 is very easy to auto pair
+    assert.equal(pcc.unpaired.length, 0);
+    assert.equal(pcc.paired.length, pcc.initialList.length / 2);
+});
+
+QUnit.test("Try easy autopairing with simple exact matching", function(assert) {
+    var pcc = new PCC({
+        datasets: DATA._1,
+        strategy: "simple",
+        twoPassAutopairing: false
+    });
+    assert.equal(pcc.unpaired.length, 0);
+    assert.equal(pcc.paired.length, pcc.initialList.length / 2);
+});
+
+QUnit.test("Try easy autopairing with LCS", function(assert) {
+    var pcc = new PCC({
+        datasets: DATA._1,
+        strategy: "lcs",
+        twoPassAutopairing: false
+    });
+    assert.equal(pcc.unpaired.length, 0);
+    assert.equal(pcc.paired.length, pcc.initialList.length / 2);
+});
+
+QUnit.test("Try easy autopairing with Levenshtein", function(assert) {
+    var pcc = new PCC({
+        datasets: DATA._1,
+        strategy: "levenshtein",
+        twoPassAutopairing: false
+    });
+    assert.equal(pcc.unpaired.length, 0);
+    assert.equal(pcc.paired.length, pcc.initialList.length / 2);
+});
+
+//TODO:
+//  filters: clearing, setting via popover, regex
+//  partition: maximize paired, maximize unpaired, split evenly
+//  pairing: manually pairing and unpairing
+//  misc: renaming pairs, removing file extensions

client/galaxy/scripts/qunit/tests/masthead_tests.js

@@ -1,124 +1,126 @@
 /* global define */
-define(["qunit/test-app", "layout/masthead", "galaxy"], function(testApp, Masthead, Galaxy) {
-    "use strict";
-    Masthead = Masthead.default;
-    QUnit.module("Masthead test", {
-        beforeEach: function() {
-            testApp.create();
-            var self = this;
-            this.masthead = new Masthead.View({
-                brand: "brand",
-                use_remote_user: "use_remote_user",
-                remote_user_logout_href: "remote_user_logout_href",
-                lims_doc_url: "lims_doc_url",
-                biostar_url: "biostar_url",
-                biostar_url_redirect: "biostar_url_redirect",
-                support_url: "support_url",
-                search_url: "search_url",
-                mailing_lists: "mailing_lists",
-                screencasts_url: "screencasts_url",
-                wiki_url: "wiki_url",
-                citation_url: "citation_url",
-                terms_url: "terms_url",
-                logo_url: "logo_url",
-                logo_src: "../../../static/images/galaxyIcon_noText.png",
-                is_admin_user: "is_admin_user",
-                active_view: "analysis",
-                ftp_upload_dir: "ftp_upload_dir",
-                ftp_upload_site: "ftp_upload_site",
-                datatypes_disable_auto: true,
-                allow_user_creation: true,
-                enable_cloud_launch: true,
-                user_requests: true
-            });
-            $("body").append(this.masthead.render().$el);
-        },
-        afterEach: function() {
-            testApp.destroy();
-        }
-    });
+import testApp from "qunit/test-app";
+import Masthead from "layout/masthead";
+import Galaxy from "galaxy";
 
-    QUnit.test("tabs", function(assert) {
-        var tab = this.masthead.collection.findWhere({ id: "analysis" });
-        var $tab = $("#analysis").find(".dropdown");
-        var $toggle = $tab.find(".dropdown-toggle");
-        var $note = $tab.find(".dropdown-note");
-        var $menu = $tab.find("ul");
-        assert.ok(tab && $tab.length == 1, "Found analysis tab");
-        tab.set("title", "Analyze");
-        assert.ok($toggle.html() == "Analyze", "Correct title");
-        assert.ok(tab.get("target") == "_parent", "Correct initial target");
-        tab.set("target", "_target");
-        assert.ok($toggle.attr("target") == "_target", "Correct test target");
-        assert.ok($tab.css("visibility") == "visible", "Tab visible");
-        tab.set("visible", false);
-        assert.ok($tab.css("visibility") == "hidden", "Tab hidden");
-        tab.set("visible", true);
-        assert.ok($tab.css("visibility") == "visible", "Tab visible, again");
-        assert.ok($toggle.attr("href") == Galaxy.root, "Correct initial url");
-        tab.set("url", "_url");
-        assert.ok($toggle.attr("href") == "/_url", "Correct test url");
-        tab.set("url", "http://_url");
-        assert.ok($toggle.attr("href") == "http://_url", "Correct http url");
-        tab.set("tooltip", "_tooltip");
-        $toggle.trigger("mouseover");
-        assert.ok($(".tooltip-inner").html() == "_tooltip", "Correct tooltip");
-        tab.set("tooltip", null);
-        $toggle.trigger("mouseover");
-        assert.ok($(".tooltip-inner").length == 0, "Tooltip removed");
-        tab.set("tooltip", "_tooltip_new");
-        $toggle.trigger("mouseover");
-        assert.ok($(".tooltip-inner").html() == "_tooltip_new", "Correct new tooltip");
-        tab.set("cls", "_cls");
-        assert.ok($toggle.hasClass("_cls"), "Correct extra class");
-        tab.set("cls", "_cls_new");
-        assert.ok($toggle.hasClass("_cls_new") && !$toggle.hasClass("_cls"), "Correct new extra class");
-        assert.ok($note.html() == "", "Correct empty note");
-        tab.set({ note: "_note", show_note: true });
-        assert.ok($note.html() == "_note", "Correct new note");
-        tab.set("toggle", true);
-        assert.ok($toggle.hasClass("toggle"), "Toggled");
-        tab.set("toggle", false);
-        assert.ok(!$toggle.hasClass("toggle"), "Untoggled");
-        tab.set("disabled", true);
-        assert.ok($tab.hasClass("disabled"), "Correctly disabled");
-        tab.set("disabled", false);
-        assert.ok(!$tab.hasClass("disabled"), "Correctly enabled");
-        assert.ok($tab.hasClass("active"), "Highlighted");
-        tab.set("active", false);
-        assert.ok(!$tab.hasClass("active"), "Not highlighted");
-        tab.set("active", true);
-        assert.ok($tab.hasClass("active"), "Highlighted, again");
-        tab.set("menu", [{ title: "_menu_title", url: "_menu_url", target: "_menu_target" }]);
-        assert.ok($menu.hasClass("dropdown-menu"), "Menu has correct class");
-        assert.ok($menu.css("display") == "none", "Menu hidden");
-        $toggle.trigger("click");
-        assert.ok($menu.css("display") == "block", "Menu shown");
-        var $item = $menu.find("a");
-        assert.ok($item.length == 1, "Added one menu item");
-        assert.ok($item.html() == "_menu_title", "Menu item has correct title");
-        assert.ok($item.attr("href") == "/_menu_url", "Menu item has correct url");
-        assert.ok($item.attr("target") == "_menu_target", "Menu item has correct target");
-        tab.set("menu", null);
-        $item = $menu.find("a");
-        assert.ok($item.length == 0, "All menu items removed");
-        tab.set("menu", [
-            { title: "_menu_title_0", url: "_menu_url_0", target: "_menu_target_0" },
-            { title: "_menu_title_1", url: "_menu_url_1", target: "_menu_target_1" }
-        ]);
-        $item = $menu.find("a");
-        assert.ok($item.length == 2, "Two menu items added");
-        tab.set("show_menu", false);
-        assert.ok($menu.css("display", "none"), "Menu manually hidden");
-        tab.set("show_menu", true);
-        assert.ok($menu.css("display", "block"), "Menu manually shown, again");
-        var tab = this.masthead.collection.findWhere({ id: "enable-scratchbook" });
-        var $tab = $("#enable-scratchbook").find(".dropdown");
-        assert.ok(tab && $tab.length == 1, "Found tab to enable scratchbook");
-        var $toggle = $tab.find(".dropdown-toggle");
-        assert.ok(!$toggle.hasClass("toggle"), "Untoggled before click");
-        $toggle.trigger("click");
-        assert.ok($toggle.hasClass("toggle"), "Toggled after click");
-        assert.ok(Galaxy.frame.active, "Scratchbook is active");
-    });
+QUnit.module("Masthead test", {
+    beforeEach: function() {
+        testApp.create();
+        var self = this;
+        this.masthead = new Masthead.View({
+            brand: "brand",
+            use_remote_user: "use_remote_user",
+            remote_user_logout_href: "remote_user_logout_href",
+            lims_doc_url: "lims_doc_url",
+            biostar_url: "biostar_url",
+            biostar_url_redirect: "biostar_url_redirect",
+            support_url: "support_url",
+            search_url: "search_url",
+            mailing_lists: "mailing_lists",
+            screencasts_url: "screencasts_url",
+            wiki_url: "wiki_url",
+            citation_url: "citation_url",
+            terms_url: "terms_url",
+            logo_url: "logo_url",
+            logo_src: "../../../static/images/galaxyIcon_noText.png",
+            is_admin_user: "is_admin_user",
+            active_view: "analysis",
+            ftp_upload_dir: "ftp_upload_dir",
+            ftp_upload_site: "ftp_upload_site",
+            datatypes_disable_auto: true,
+            allow_user_creation: true,
+            enable_cloud_launch: true,
+            user_requests: true
+        });
+        $("body").append(this.masthead.render().$el);
+    },
+    afterEach: function() {
+        testApp.destroy();
+    }
+});
+
+QUnit.test("tabs", function(assert) {
+    var tab = this.masthead.collection.findWhere({ id: "analysis" });
+    var $tab = $("#analysis").find(".dropdown");
+    var $toggle = $tab.find(".dropdown-toggle");
+    var $note = $tab.find(".dropdown-note");
+    var $menu = $tab.find("ul");
+    assert.ok(tab && $tab.length == 1, "Found analysis tab");
+    tab.set("title", "Analyze");
+    assert.ok($toggle.html() == "Analyze", "Correct title");
+    assert.ok(tab.get("target") == "_parent", "Correct initial target");
+    tab.set("target", "_target");
+    assert.ok($toggle.attr("target") == "_target", "Correct test target");
+    assert.ok($tab.css("visibility") == "visible", "Tab visible");
+    tab.set("visible", false);
+    assert.ok($tab.css("visibility") == "hidden", "Tab hidden");
+    tab.set("visible", true);
+    assert.ok($tab.css("visibility") == "visible", "Tab visible, again");
+    // TODO: cleanup global usage so window.Galaxy isn't needed here.
+    assert.ok($toggle.attr("href") == window.Galaxy.root, "Correct initial url");
+    tab.set("url", "_url");
+    assert.ok($toggle.attr("href") == "/_url", "Correct test url");
+    tab.set("url", "http://_url");
+    assert.ok($toggle.attr("href") == "http://_url", "Correct http url");
+    tab.set("tooltip", "_tooltip");
+    $toggle.trigger("mouseover");
+    assert.ok($(".tooltip-inner").html() == "_tooltip", "Correct tooltip");
+    tab.set("tooltip", null);
+    $toggle.trigger("mouseover");
+    assert.ok($(".tooltip-inner").length == 0, "Tooltip removed");
+    tab.set("tooltip", "_tooltip_new");
+    $toggle.trigger("mouseover");
+    assert.ok($(".tooltip-inner").html() == "_tooltip_new", "Correct new tooltip");
+    tab.set("cls", "_cls");
+    assert.ok($toggle.hasClass("_cls"), "Correct extra class");
+    tab.set("cls", "_cls_new");
+    assert.ok($toggle.hasClass("_cls_new") && !$toggle.hasClass("_cls"), "Correct new extra class");
+    assert.ok($note.html() == "", "Correct empty note");
+    tab.set({ note: "_note", show_note: true });
+    assert.ok($note.html() == "_note", "Correct new note");
+    tab.set("toggle", true);
+    assert.ok($toggle.hasClass("toggle"), "Toggled");
+    tab.set("toggle", false);
+    assert.ok(!$toggle.hasClass("toggle"), "Untoggled");
+    tab.set("disabled", true);
+    assert.ok($tab.hasClass("disabled"), "Correctly disabled");
+    tab.set("disabled", false);
+    assert.ok(!$tab.hasClass("disabled"), "Correctly enabled");
+    assert.ok($tab.hasClass("active"), "Highlighted");
+    tab.set("active", false);
+    assert.ok(!$tab.hasClass("active"), "Not highlighted");
+    tab.set("active", true);
+    assert.ok($tab.hasClass("active"), "Highlighted, again");
+    tab.set("menu", [{ title: "_menu_title", url: "_menu_url", target: "_menu_target" }]);
+    assert.ok($menu.hasClass("dropdown-menu"), "Menu has correct class");
+    assert.ok($menu.css("display") == "none", "Menu hidden");
+    $toggle.trigger("click");
+    assert.ok($menu.css("display") == "block", "Menu shown");
+    var $item = $menu.find("a");
+    assert.ok($item.length == 1, "Added one menu item");
+    assert.ok($item.html() == "_menu_title", "Menu item has correct title");
+    assert.ok($item.attr("href") == "/_menu_url", "Menu item has correct url");
+    assert.ok($item.attr("target") == "_menu_target", "Menu item has correct target");
+    tab.set("menu", null);
+    $item = $menu.find("a");
+    assert.ok($item.length == 0, "All menu items removed");
+    tab.set("menu", [
+        { title: "_menu_title_0", url: "_menu_url_0", target: "_menu_target_0" },
+        { title: "_menu_title_1", url: "_menu_url_1", target: "_menu_target_1" }
+    ]);
+    $item = $menu.find("a");
+    assert.ok($item.length == 2, "Two menu items added");
+    tab.set("show_menu", false);
+    assert.ok($menu.css("display", "none"), "Menu manually hidden");
+    tab.set("show_menu", true);
+    assert.ok($menu.css("display", "block"), "Menu manually shown, again");
+    var tab = this.masthead.collection.findWhere({ id: "enable-scratchbook" });
+    var $tab = $("#enable-scratchbook").find(".dropdown");
+    assert.ok(tab && $tab.length == 1, "Found tab to enable scratchbook");
+    var $toggle = $tab.find(".dropdown-toggle");
+    assert.ok(!$toggle.hasClass("toggle"), "Untoggled before click");
+    $toggle.trigger("click");
+    assert.ok($toggle.hasClass("toggle"), "Toggled after click");
+    // TODO: cleanup global usage so window.Galaxy isn't needed here.
+    assert.ok(window.Galaxy.frame.active, "Scratchbook is active");
 });

client/galaxy/scripts/qunit/tests/metrics-logger.js

@@ -1,370 +0,0 @@
-// This file isn't really testing anything useful yet, it is just testing
-// (or demonstrating) qunit+backbone interactions.
-/* global define */
-define(["qunit/test-app", "utils/metrics-logger", "jquery", "sinon"], function(testApp, metrics, $, sinon) {
-    "use strict";
-    metrics = metrics.default;
-
-    var MockConsole = function() {
-        var self = this;
-        self.lastMessage = null;
-        ["log", "debug", "info", "warn", "error"].forEach(function(fnName) {
-            self[fnName] = function() {
-                var args = Array.prototype.slice.call(arguments, 0);
-                self.lastMessage = { level: fnName, args: args };
-            };
-        });
-        return self;
-    };
-
-    (window.bootstrapped = {}).user = {
-        id: "test"
-    };
-
-    QUnit.module("Metrics logger tests");
-    // ======================================================================== MetricsLogger
-    QUnit.test("logger construction/initializiation defaults", function(assert) {
-        var logger = new metrics.MetricsLogger({});
-        assert.equal(logger.consoleLogger, null);
-        assert.equal(logger.options.logLevel, metrics.MetricsLogger.NONE);
-        assert.equal(logger.options.consoleLevel, metrics.MetricsLogger.NONE);
-        assert.equal(logger.options.defaultNamespace, "Galaxy");
-        assert.equal(logger.options.clientPrefix, "client.");
-        assert.equal(logger.options.postSize, 1000);
-        assert.equal(logger.options.maxCacheSize, 3000);
-        assert.equal(logger.options.addTime, true);
-        assert.equal(logger.options.postUrl, "/api/metrics");
-        assert.equal(logger.options.getPingData, undefined);
-        assert.equal(logger.options.onServerResponse, undefined);
-
-        assert.equal(logger._postSize, 1000);
-        assert.equal(logger.cache.constructor, metrics.LoggingCache);
-    });
-
-    QUnit.test("_parseLevel", function(assert) {
-        var logger = new metrics.MetricsLogger({});
-        assert.equal(logger._parseLevel("all"), metrics.MetricsLogger.ALL);
-        assert.equal(logger._parseLevel("debug"), metrics.MetricsLogger.DEBUG);
-        assert.equal(logger._parseLevel("info"), metrics.MetricsLogger.INFO);
-        assert.equal(logger._parseLevel("warn"), metrics.MetricsLogger.WARN);
-        assert.equal(logger._parseLevel("error"), metrics.MetricsLogger.ERROR);
-        assert.equal(logger._parseLevel("metric"), metrics.MetricsLogger.METRIC);
-        assert.equal(logger._parseLevel("none"), metrics.MetricsLogger.NONE);
-        assert.equal(logger._parseLevel(15), 15);
-
-        assert.throws(
-            function() {
-                logger._parseLevel(undefined);
-            },
-            /Unknown log level/,
-            "Unknown log level throws error"
-        );
-        assert.throws(
-            function() {
-                logger._parseLevel("nope");
-            },
-            /Unknown log level/,
-            "Unknown log level throws error"
-        );
-    });
-
-    // ------------------------------------------------------------------------ Emit to cache
-    QUnit.test("emit to cache at level", function(assert) {
-        var logger = new metrics.MetricsLogger({
-            logLevel: "metric"
-        });
-        logger.cache.empty();
-
-        assert.equal(logger.options.logLevel, metrics.MetricsLogger.METRIC);
-        logger.emit("metric", "test", [1, 2, { three: 3 }]);
-        assert.equal(logger.cache.length(), 1);
-
-        var cached = logger.cache.get(1)[0];
-        //console.debug( 'cached:', JSON.stringify( cached ) );
-        assert.equal(cached.level, metrics.MetricsLogger.METRIC);
-        assert.equal(cached.namespace, "client.test");
-        assert.equal(cached.args.length, 3);
-        assert.equal(cached.args[2].three, 3);
-        assert.ok(typeof cached.time === "string");
-        assert.ok(cached.time === new Date(cached.time).toISOString());
-    });
-
-    QUnit.test("emit to cache below does not cache", function(assert) {
-        var logger = new metrics.MetricsLogger({
-            logLevel: "metric"
-        });
-        logger.cache.empty();
-
-        logger.emit("error", "test", [1, 2, { three: 3 }]);
-        assert.equal(logger.cache.length(), 0);
-    });
-
-    QUnit.test("emit to cache (silently) drops non-parsable", function(assert) {
-        var logger = new metrics.MetricsLogger({
-            logLevel: "metric"
-        });
-        logger.cache.empty();
-
-        logger.emit("metric", "test", [{ window: window }]);
-        assert.equal(logger.cache.length(), 0);
-    });
-
-    function metricsFromRequestBody(request) {
-        // assumes 'metrics' is only entry in requestBody
-        return JSON.parse(decodeURIComponent(request.requestBody.replace("metrics=", "")));
-    }
-
-    QUnit.test("_postCache success", function(assert) {
-        var callback = sinon.spy(),
-            logger = new metrics.MetricsLogger({
-                logLevel: "metric",
-                onServerResponse: function(response) {
-                    callback();
-                }
-            });
-        logger.cache.empty();
-
-        var server = sinon.fakeServer.create(),
-            metricsOnServer;
-        server.respondWith("POST", "/api/metrics", function(request) {
-            metricsOnServer = metricsFromRequestBody(request);
-            //console.debug( 'requestBody:', request.requestBody );
-            //console.debug( 'metricsOnServer:', JSON.stringify( metricsOnServer, null, '  ' ) );
-            request.respond(
-                200,
-                { "Content-Type": "application/json" },
-                JSON.stringify({
-                    fakeResponse: "yes"
-                })
-            );
-        });
-
-        logger.emit("metric", "test", [1, 2, { three: 3 }]);
-        logger._postCache();
-        server.respond();
-
-        assert.ok(callback.calledOnce, "onServerResponse was called");
-        assert.equal(logger.cache.length(), 0, "should have emptied cache (on success)");
-        assert.equal(logger._postSize, 1000, "_postSize still at default");
-
-        // metrics were in proper form on server
-        assert.equal(metricsOnServer.length, 1);
-        var metric = metricsOnServer[0];
-        assert.equal(metric.level, metrics.MetricsLogger.METRIC);
-        assert.equal(metric.namespace, "client.test");
-        assert.equal(metric.args.length, 3);
-        assert.equal(metric.args[2].three, 3);
-        assert.ok(typeof metric.time === "string");
-        assert.ok(metric.time === new Date(metric.time).toISOString());
-
-        server.restore();
-    });
-
-    QUnit.test("_postCache failure", function(assert) {
-        var callback = sinon.spy(),
-            logger = new metrics.MetricsLogger({
-                logLevel: "metric",
-                onServerResponse: function(response) {
-                    callback();
-                }
-            });
-        logger.cache.empty();
-
-        var server = sinon.fakeServer.create();
-        server.respondWith("POST", "/api/metrics", function(request) {
-            request.respond(
-                500,
-                { "Content-Type": "application/json" },
-                JSON.stringify({
-                    err_msg: "NoooOPE!"
-                })
-            );
-        });
-
-        logger.emit("metric", "test", [1, 2, { three: 3 }]);
-        logger._postCache();
-        server.respond();
-        //TODO: is the following what we want?
-        assert.ok(!callback.calledOnce, "onServerResponse was NOT called");
-        assert.equal(logger.cache.length(), 1, "should NOT have emptied cache");
-        assert.equal(logger._postSize, logger.options.maxCacheSize, "_postSize changed to max");
-
-        server.restore();
-    });
-
-    // ------------------------------------------------------------------------ Emit to console
-    QUnit.test("emit to console at level", function(assert) {
-        var mockConsole = new MockConsole(),
-            logger = new metrics.MetricsLogger({
-                consoleLevel: "debug",
-                consoleLogger: mockConsole
-            });
-        assert.equal(logger.options.consoleLevel, metrics.MetricsLogger.DEBUG);
-        assert.equal(logger.consoleLogger.constructor, MockConsole);
-
-        logger.emit("debug", "test", [1, 2, { three: 3 }]);
-        assert.equal(logger.cache.length(), 1);
-        //console.debug( JSON.stringify( mockConsole.lastMessage ) );
-        assert.equal(mockConsole.lastMessage.level, "debug");
-        assert.equal(mockConsole.lastMessage.args.length, 4);
-        assert.equal(mockConsole.lastMessage.args[0], "test");
-        assert.equal(mockConsole.lastMessage.args[3].three, 3);
-    });
-
-    QUnit.test("emit to console below does not output", function(assert) {
-        var mockConsole = new MockConsole(),
-            logger = new metrics.MetricsLogger({
-                consoleLevel: "error",
-                consoleLogger: mockConsole
-            });
-        logger.emit("debug", "test", [1, 2, { three: 3 }]);
-        assert.equal(mockConsole.lastMessage, null);
-    });
-
-    // ------------------------------------------------------------------------ Shortcuts
-    QUnit.test("logger shortcuts emit to default namespace properly", function(assert) {
-        var logger = new metrics.MetricsLogger({
-            logLevel: "all"
-        });
-        logger.cache.empty();
-
-        assert.equal(logger.options.logLevel, metrics.MetricsLogger.ALL);
-        logger.log(0);
-        logger.debug(1);
-        logger.info(2);
-        logger.warn(3);
-        logger.error(4);
-        logger.metric(5);
-
-        assert.equal(logger.cache.length(), 6);
-        var cached = logger.cache.remove(6),
-            entry;
-
-        cached.forEach(function(entry) {
-            assert.ok(entry.namespace === logger.options.clientPrefix + logger.options.defaultNamespace);
-            assert.ok(jQuery.type(entry.args) === "array");
-            assert.ok(typeof entry.time === "string");
-        });
-
-        // log is different
-        entry = cached[0];
-        assert.ok(entry.level === 1);
-        assert.ok(entry.args[0] === 0);
-
-        ["debug", "info", "warn", "error", "metric"].forEach(function(level, i) {
-            entry = cached[i + 1];
-            assert.ok(entry.level === logger._parseLevel(level));
-            assert.ok(entry.args[0] === i + 1);
-        });
-    });
-
-    // ======================================================================== LoggingCache
-    QUnit.test("cache construction/initializiation defaults", function(assert) {
-        // use empty to prevent tests stepping on one another due to persistence
-        var cache = new metrics.LoggingCache({ key: "logs-test" }).empty();
-        assert.equal(cache.maxSize, 5000);
-        assert.equal(window.localStorage.getItem("logs-test"), "[]");
-    });
-
-    QUnit.test("cache construction/initializiation failure", function(assert) {
-        ////TODO: doesn't work - readonly
-        //window.localStorage = null;
-        //console.debug( 'localStorage:', window.localStorage );
-        var oldFn = metrics.LoggingCache.prototype._hasStorage;
-        metrics.LoggingCache.prototype._hasStorage = function() {
-            return false;
-        };
-        assert.throws(
-            function() {
-                return new metrics.LoggingCache({ key: "logs-test" });
-            },
-            /LoggingCache needs localStorage/,
-            "lack of localStorage throws error"
-        );
-        metrics.LoggingCache.prototype._hasStorage = oldFn;
-
-        assert.throws(
-            function() {
-                return new metrics.LoggingCache();
-            },
-            /LoggingCache needs key for localStorage/,
-            "lack of key throws error"
-        );
-    });
-
-    QUnit.test("cache construction/initializiation setting max cache size", function(assert) {
-        var cache = new metrics.LoggingCache({
-            key: "logs-test",
-            maxSize: 5
-        }).empty();
-        assert.equal(cache.maxSize, 5);
-    });
-
-    QUnit.test("cache plays well with no data", function(assert) {
-        var cache = new metrics.LoggingCache({ key: "logs-test" }).empty();
-
-        assert.equal(cache.length(), 0);
-        var get = cache.get(10);
-        assert.ok(jQuery.type(get) === "array" && get.length === 0);
-        var remove = cache.remove(10);
-        assert.ok(jQuery.type(remove) === "array" && remove.length === 0);
-        assert.equal(cache.length(), 0);
-    });
-
-    QUnit.test("cache add properly adds and removes data", function(assert) {
-        var cache = new metrics.LoggingCache({
-            key: "logs-test",
-            maxSize: 5
-        }).empty();
-
-        var entry1 = [{ one: 1 }, "two"];
-        cache.add(entry1);
-
-        assert.equal(cache.length(), 1);
-        assert.equal(JSON.stringify(cache.get(1)[0]), JSON.stringify(entry1));
-
-        var entry2 = { blah: { one: 1 }, bler: ["three", { two: 2 }] };
-        cache.add(entry2);
-        assert.equal(cache.length(), 2);
-        assert.equal(cache.stringify(2), "[" + JSON.stringify(entry1) + "," + JSON.stringify(entry2) + "]");
-
-        // FIFO
-        var returned = cache.remove(1);
-        assert.equal(cache.length(), 1);
-        assert.ok(jQuery.type(returned) === "array" && returned.length === 1);
-        var returned0 = returned[0];
-        assert.ok(jQuery.type(returned0) === "array" && JSON.stringify(returned0) === JSON.stringify(entry1));
-    });
-
-    QUnit.test("cache past max loses oldest", function(assert) {
-        var cache = new metrics.LoggingCache({
-            key: "logs-test",
-            maxSize: 5
-        }).empty();
-
-        for (var i = 0; i < 10; i += 1) {
-            cache.add({ index: i });
-        }
-        assert.equal(cache.length(), 5);
-        var get = cache.get(5);
-        assert.ok(get[0].index === 5);
-        assert.ok(get[1].index === 6);
-        assert.ok(get[2].index === 7);
-        assert.ok(get[3].index === 8);
-        assert.ok(get[4].index === 9);
-    });
-
-    QUnit.test("cache is properly persistent", function(assert) {
-        var cache1 = new metrics.LoggingCache({ key: "logs-test" }).empty(),
-            entry = [{ one: 1 }, "two"];
-        cache1.add(entry);
-        assert.equal(cache1.length(), 1);
-
-        var cache2 = new metrics.LoggingCache({ key: "logs-test" });
-        assert.equal(cache2.length(), 1, "old key gets previously stored");
-        assert.equal(JSON.stringify(cache2.get(1)[0]), JSON.stringify(entry));
-
-        var cache3 = new metrics.LoggingCache({ key: "logs-bler" });
-        assert.equal(cache3.length(), 0, "new key causes new storage");
-    });
-});

client/galaxy/scripts/qunit/tests/metrics_logger_tests.js

@@ -0,0 +1,371 @@
+// This file isn't really testing anything useful yet, it is just testing
+// (or demonstrating) qunit+backbone interactions.
+/* global define */
+
+import testApp from "qunit/test-app";
+import metrics from "utils/metrics-logger";
+import $ from "jquery";
+import sinon from "sinon";
+
+var MockConsole = function() {
+    var self = this;
+    self.lastMessage = null;
+    ["log", "debug", "info", "warn", "error"].forEach(function(fnName) {
+        self[fnName] = function() {
+            var args = Array.prototype.slice.call(arguments, 0);
+            self.lastMessage = { level: fnName, args: args };
+        };
+    });
+    return self;
+};
+
+(window.bootstrapped = {}).user = {
+    id: "test"
+};
+
+QUnit.module("Metrics logger tests");
+// ======================================================================== MetricsLogger
+QUnit.test("logger construction/initializiation defaults", function(assert) {
+    var logger = new metrics.MetricsLogger({});
+    assert.equal(logger.consoleLogger, null);
+    assert.equal(logger.options.logLevel, metrics.MetricsLogger.NONE);
+    assert.equal(logger.options.consoleLevel, metrics.MetricsLogger.NONE);
+    assert.equal(logger.options.defaultNamespace, "Galaxy");
+    assert.equal(logger.options.clientPrefix, "client.");
+    assert.equal(logger.options.postSize, 1000);
+    assert.equal(logger.options.maxCacheSize, 3000);
+    assert.equal(logger.options.addTime, true);
+    assert.equal(logger.options.postUrl, "/api/metrics");
+    assert.equal(logger.options.getPingData, undefined);
+    assert.equal(logger.options.onServerResponse, undefined);
+
+    assert.equal(logger._postSize, 1000);
+    assert.equal(logger.cache.constructor, metrics.LoggingCache);
+});
+
+QUnit.test("_parseLevel", function(assert) {
+    var logger = new metrics.MetricsLogger({});
+    assert.equal(logger._parseLevel("all"), metrics.MetricsLogger.ALL);
+    assert.equal(logger._parseLevel("debug"), metrics.MetricsLogger.DEBUG);
+    assert.equal(logger._parseLevel("info"), metrics.MetricsLogger.INFO);
+    assert.equal(logger._parseLevel("warn"), metrics.MetricsLogger.WARN);
+    assert.equal(logger._parseLevel("error"), metrics.MetricsLogger.ERROR);
+    assert.equal(logger._parseLevel("metric"), metrics.MetricsLogger.METRIC);
+    assert.equal(logger._parseLevel("none"), metrics.MetricsLogger.NONE);
+    assert.equal(logger._parseLevel(15), 15);
+
+    assert.throws(
+        function() {
+            logger._parseLevel(undefined);
+        },
+        /Unknown log level/,
+        "Unknown log level throws error"
+    );
+    assert.throws(
+        function() {
+            logger._parseLevel("nope");
+        },
+        /Unknown log level/,
+        "Unknown log level throws error"
+    );
+});
+
+// ------------------------------------------------------------------------ Emit to cache
+QUnit.test("emit to cache at level", function(assert) {
+    var logger = new metrics.MetricsLogger({
+        logLevel: "metric"
+    });
+    logger.cache.empty();
+
+    assert.equal(logger.options.logLevel, metrics.MetricsLogger.METRIC);
+    logger.emit("metric", "test", [1, 2, { three: 3 }]);
+    assert.equal(logger.cache.length(), 1);
+
+    var cached = logger.cache.get(1)[0];
+    //console.debug( 'cached:', JSON.stringify( cached ) );
+    assert.equal(cached.level, metrics.MetricsLogger.METRIC);
+    assert.equal(cached.namespace, "client.test");
+    assert.equal(cached.args.length, 3);
+    assert.equal(cached.args[2].three, 3);
+    assert.ok(typeof cached.time === "string");
+    assert.ok(cached.time === new Date(cached.time).toISOString());
+});
+
+QUnit.test("emit to cache below does not cache", function(assert) {
+    var logger = new metrics.MetricsLogger({
+        logLevel: "metric"
+    });
+    logger.cache.empty();
+
+    logger.emit("error", "test", [1, 2, { three: 3 }]);
+    assert.equal(logger.cache.length(), 0);
+});
+
+QUnit.test("emit to cache (silently) drops non-parsable", function(assert) {
+    var logger = new metrics.MetricsLogger({
+        logLevel: "metric"
+    });
+    logger.cache.empty();
+
+    logger.emit("metric", "test", [{ window: window }]);
+    assert.equal(logger.cache.length(), 0);
+});
+
+function metricsFromRequestBody(request) {
+    // assumes 'metrics' is only entry in requestBody
+    return JSON.parse(decodeURIComponent(request.requestBody.replace("metrics=", "")));
+}
+
+QUnit.test("_postCache success", function(assert) {
+    var callback = sinon.spy(),
+        logger = new metrics.MetricsLogger({
+            logLevel: "metric",
+            onServerResponse: function(response) {
+                callback();
+            }
+        });
+    logger.cache.empty();
+
+    var server = sinon.fakeServer.create(),
+        metricsOnServer;
+    server.respondWith("POST", "/api/metrics", function(request) {
+        metricsOnServer = metricsFromRequestBody(request);
+        //console.debug( 'requestBody:', request.requestBody );
+        //console.debug( 'metricsOnServer:', JSON.stringify( metricsOnServer, null, '  ' ) );
+        request.respond(
+            200,
+            { "Content-Type": "application/json" },
+            JSON.stringify({
+                fakeResponse: "yes"
+            })
+        );
+    });
+
+    logger.emit("metric", "test", [1, 2, { three: 3 }]);
+    logger._postCache();
+    server.respond();
+
+    assert.ok(callback.calledOnce, "onServerResponse was called");
+    assert.equal(logger.cache.length(), 0, "should have emptied cache (on success)");
+    assert.equal(logger._postSize, 1000, "_postSize still at default");
+
+    // metrics were in proper form on server
+    assert.equal(metricsOnServer.length, 1);
+    var metric = metricsOnServer[0];
+    assert.equal(metric.level, metrics.MetricsLogger.METRIC);
+    assert.equal(metric.namespace, "client.test");
+    assert.equal(metric.args.length, 3);
+    assert.equal(metric.args[2].three, 3);
+    assert.ok(typeof metric.time === "string");
+    assert.ok(metric.time === new Date(metric.time).toISOString());
+
+    server.restore();
+});
+
+QUnit.test("_postCache failure", function(assert) {
+    var callback = sinon.spy(),
+        logger = new metrics.MetricsLogger({
+            logLevel: "metric",
+            onServerResponse: function(response) {
+                callback();
+            }
+        });
+    logger.cache.empty();
+
+    var server = sinon.fakeServer.create();
+    server.respondWith("POST", "/api/metrics", function(request) {
+        request.respond(
+            500,
+            { "Content-Type": "application/json" },
+            JSON.stringify({
+                err_msg: "NoooOPE!"
+            })
+        );
+    });
+
+    logger.emit("metric", "test", [1, 2, { three: 3 }]);
+    logger._postCache();
+    server.respond();
+    //TODO: is the following what we want?
+    assert.ok(!callback.calledOnce, "onServerResponse was NOT called");
+    assert.equal(logger.cache.length(), 1, "should NOT have emptied cache");
+    assert.equal(logger._postSize, logger.options.maxCacheSize, "_postSize changed to max");
+
+    server.restore();
+});
+
+// ------------------------------------------------------------------------ Emit to console
+QUnit.test("emit to console at level", function(assert) {
+    var mockConsole = new MockConsole(),
+        logger = new metrics.MetricsLogger({
+            consoleLevel: "debug",
+            consoleLogger: mockConsole
+        });
+    assert.equal(logger.options.consoleLevel, metrics.MetricsLogger.DEBUG);
+    assert.equal(logger.consoleLogger.constructor, MockConsole);
+
+    logger.emit("debug", "test", [1, 2, { three: 3 }]);
+    assert.equal(logger.cache.length(), 1);
+    //console.debug( JSON.stringify( mockConsole.lastMessage ) );
+    assert.equal(mockConsole.lastMessage.level, "debug");
+    assert.equal(mockConsole.lastMessage.args.length, 4);
+    assert.equal(mockConsole.lastMessage.args[0], "test");
+    assert.equal(mockConsole.lastMessage.args[3].three, 3);
+});
+
+QUnit.test("emit to console below does not output", function(assert) {
+    var mockConsole = new MockConsole(),
+        logger = new metrics.MetricsLogger({
+            consoleLevel: "error",
+            consoleLogger: mockConsole
+        });
+    logger.emit("debug", "test", [1, 2, { three: 3 }]);
+    assert.equal(mockConsole.lastMessage, null);
+});
+
+// ------------------------------------------------------------------------ Shortcuts
+QUnit.test("logger shortcuts emit to default namespace properly", function(assert) {
+    var logger = new metrics.MetricsLogger({
+        logLevel: "all"
+    });
+    logger.cache.empty();
+
+    assert.equal(logger.options.logLevel, metrics.MetricsLogger.ALL);
+    logger.log(0);
+    logger.debug(1);
+    logger.info(2);
+    logger.warn(3);
+    logger.error(4);
+    logger.metric(5);
+
+    assert.equal(logger.cache.length(), 6);
+    var cached = logger.cache.remove(6),
+        entry;
+
+    cached.forEach(function(entry) {
+        assert.ok(entry.namespace === logger.options.clientPrefix + logger.options.defaultNamespace);
+        assert.ok(jQuery.type(entry.args) === "array");
+        assert.ok(typeof entry.time === "string");
+    });
+
+    // log is different
+    entry = cached[0];
+    assert.ok(entry.level === 1);
+    assert.ok(entry.args[0] === 0);
+
+    ["debug", "info", "warn", "error", "metric"].forEach(function(level, i) {
+        entry = cached[i + 1];
+        assert.ok(entry.level === logger._parseLevel(level));
+        assert.ok(entry.args[0] === i + 1);
+    });
+});
+
+// ======================================================================== LoggingCache
+QUnit.test("cache construction/initializiation defaults", function(assert) {
+    // use empty to prevent tests stepping on one another due to persistence
+    var cache = new metrics.LoggingCache({ key: "logs-test" }).empty();
+    assert.equal(cache.maxSize, 5000);
+    assert.equal(window.localStorage.getItem("logs-test"), "[]");
+});
+
+QUnit.test("cache construction/initializiation failure", function(assert) {
+    ////TODO: doesn't work - readonly
+    //window.localStorage = null;
+    //console.debug( 'localStorage:', window.localStorage );
+    var oldFn = metrics.LoggingCache.prototype._hasStorage;
+    metrics.LoggingCache.prototype._hasStorage = function() {
+        return false;
+    };
+    assert.throws(
+        function() {
+            return new metrics.LoggingCache({ key: "logs-test" });
+        },
+        /LoggingCache needs localStorage/,
+        "lack of localStorage throws error"
+    );
+    metrics.LoggingCache.prototype._hasStorage = oldFn;
+
+    assert.throws(
+        function() {
+            return new metrics.LoggingCache();
+        },
+        /LoggingCache needs key for localStorage/,
+        "lack of key throws error"
+    );
+});
+
+QUnit.test("cache construction/initializiation setting max cache size", function(assert) {
+    var cache = new metrics.LoggingCache({
+        key: "logs-test",
+        maxSize: 5
+    }).empty();
+    assert.equal(cache.maxSize, 5);
+});
+
+QUnit.test("cache plays well with no data", function(assert) {
+    var cache = new metrics.LoggingCache({ key: "logs-test" }).empty();
+
+    assert.equal(cache.length(), 0);
+    var get = cache.get(10);
+    assert.ok(jQuery.type(get) === "array" && get.length === 0);
+    var remove = cache.remove(10);
+    assert.ok(jQuery.type(remove) === "array" && remove.length === 0);
+    assert.equal(cache.length(), 0);
+});
+
+QUnit.test("cache add properly adds and removes data", function(assert) {
+    var cache = new metrics.LoggingCache({
+        key: "logs-test",
+        maxSize: 5
+    }).empty();
+
+    var entry1 = [{ one: 1 }, "two"];
+    cache.add(entry1);
+
+    assert.equal(cache.length(), 1);
+    assert.equal(JSON.stringify(cache.get(1)[0]), JSON.stringify(entry1));
+
+    var entry2 = { blah: { one: 1 }, bler: ["three", { two: 2 }] };
+    cache.add(entry2);
+    assert.equal(cache.length(), 2);
+    assert.equal(cache.stringify(2), "[" + JSON.stringify(entry1) + "," + JSON.stringify(entry2) + "]");
+
+    // FIFO
+    var returned = cache.remove(1);
+    assert.equal(cache.length(), 1);
+    assert.ok(jQuery.type(returned) === "array" && returned.length === 1);
+    var returned0 = returned[0];
+    assert.ok(jQuery.type(returned0) === "array" && JSON.stringify(returned0) === JSON.stringify(entry1));
+});
+
+QUnit.test("cache past max loses oldest", function(assert) {
+    var cache = new metrics.LoggingCache({
+        key: "logs-test",
+        maxSize: 5
+    }).empty();
+
+    for (var i = 0; i < 10; i += 1) {
+        cache.add({ index: i });
+    }
+    assert.equal(cache.length(), 5);
+    var get = cache.get(5);
+    assert.ok(get[0].index === 5);
+    assert.ok(get[1].index === 6);
+    assert.ok(get[2].index === 7);
+    assert.ok(get[3].index === 8);
+    assert.ok(get[4].index === 9);
+});
+
+QUnit.test("cache is properly persistent", function(assert) {
+    var cache1 = new metrics.LoggingCache({ key: "logs-test" }).empty(),
+        entry = [{ one: 1 }, "two"];
+    cache1.add(entry);
+    assert.equal(cache1.length(), 1);
+
+    var cache2 = new metrics.LoggingCache({ key: "logs-test" });
+    assert.equal(cache2.length(), 1, "old key gets previously stored");
+    assert.equal(JSON.stringify(cache2.get(1)[0]), JSON.stringify(entry));
+
+    var cache3 = new metrics.LoggingCache({ key: "logs-bler" });
+    assert.equal(cache3.length(), 0, "new key causes new storage");
+});

client/galaxy/scripts/qunit/tests/modal_tests.js

@@ -1,92 +1,86 @@
 /* global define */
 
-define(["qunit/test-app", "mvc/ui/ui-modal", "sinon"], function(testApp, GalaxyModal, sinon) {
-    "use strict";
-    GalaxyModal = GalaxyModal.default;
-    QUnit.module("Modal dialog test", {
-        beforeEach: function() {
-            testApp.create();
-            this.clock = sinon.useFakeTimers();
-            var self = this;
-            this.app = new GalaxyModal.View({
-                title: "Test title",
-                body: "Test body",
-                buttons: {
-                    Ok: function() {},
-                    Cancel: function() {
-                        self.app.hide();
-                    }
+import testApp from "qunit/test-app";
+import GalaxyModal from "mvc/ui/ui-modal";
+import $ from "jquery";
+
+QUnit.module("Modal dialog test", {
+    beforeEach: function() {
+        $.fx.off = true;
+        testApp.create();
+        var self = this;
+        this.app = new GalaxyModal.View({
+            title: "Test title",
+            body: "Test body",
+            buttons: {
+                Ok: function() {},
+                Cancel: function() {
+                    self.app.hide();
                 }
-            });
-        },
-        afterEach: function() {
-            testApp.destroy();
-            this.clock.restore();
-        }
-    });
+            }
+        });
+    },
+    afterEach: function() {
+        $.fx.off = false;
+        testApp.destroy();
+    }
+});
 
-    QUnit.test("test dialog attributes", function(assert) {
-        assert.ok(this.app.$header.find(".title").html() == "Test title", "Modal header has correct title.");
-        assert.ok(this.app.$body.html() == "Test body", "Modal header has correct body.");
-    });
+QUnit.test("test dialog attributes", function(assert) {
+    assert.ok(this.app.$header.find(".title").html() == "Test title", "Modal header has correct title.");
+    assert.ok(this.app.$body.html() == "Test body", "Modal header has correct body.");
+});
 
-    QUnit.test("test dialog visibility", function(assert) {
-        assert.ok(this.app.$el.css("display") == "block", "Modal is initially visible");
-        this.app.hide();
-        this.clock.tick(WAIT_FADE);
-        assert.ok(this.app.$el.css("display") == "none", "Modal hidden manually");
-        this.app.show();
-        this.clock.tick(WAIT_FADE);
-        assert.ok(this.app.$el.css("display") == "block", "Modal shown manually");
-        this.app.getButton("Ok").trigger("click");
-        this.clock.tick(WAIT_FADE);
-        assert.ok(this.app.$el.css("display") == "block", "Modal still visible after clicking Ok");
-        this.app.getButton("Cancel").trigger("click");
-        this.clock.tick(WAIT_FADE);
-        assert.ok(this.app.$el.css("display") == "none", "Modal hidden after clicking Cancel");
-        this.app.show();
-        this.clock.tick(WAIT_FADE);
-        assert.ok(this.app.$el.css("display") == "block", "Modal manually shown again");
-        assert.ok(!this.app.$header.hasClass("no-separator"), "Title separator tagged as visible.");
-        this.app.show({ title_separator: false });
-        assert.ok(this.app.$header.hasClass("no-separator"), "Title separator tagged as hidden.");
-        assert.ok(this.app.$backdrop.hasClass("in"), "Backdrop tagged as shown.");
-        this.app.show({ backdrop: false });
-        assert.ok(!this.app.$backdrop.hasClass("in"), "Backdrop tagged as hidden.");
-    });
+QUnit.test("test dialog visibility", function(assert) {
+    this.app.hide();
+    assert.ok(this.app.$el.css("display") == "none", "Modal hidden manually");
+    this.app.show();
+    assert.ok(this.app.$el.css("display") == "block", "Modal shown manually");
+    this.app.getButton("Ok").trigger("click");
+    assert.ok(this.app.$el.css("display") == "block", "Modal still visible after clicking Ok");
+    this.app.getButton("Cancel").trigger("click");
+    assert.ok(this.app.$el.css("display") == "none", "Modal hidden after clicking Cancel");
+    this.app.show();
+    assert.ok(this.app.$el.css("display") == "block", "Modal manually shown again");
+    assert.ok(!this.app.$header.hasClass("no-separator"), "Title separator tagged as visible.");
+    this.app.show({ title_separator: false });
+    assert.ok(this.app.$header.hasClass("no-separator"), "Title separator tagged as hidden.");
+    assert.ok(this.app.$backdrop.hasClass("in"), "Backdrop tagged as shown.");
+    this.app.show({ backdrop: false });
+    assert.ok(!this.app.$backdrop.hasClass("in"), "Backdrop tagged as hidden.");
+});
 
-    QUnit.test("test dialog closing events", function(assert) {
-        this.app.$backdrop.trigger("click");
-        this.clock.tick(WAIT_FADE);
-        assert.ok(this.app.$el.css("display") == "block", "Modal shown after backdrop click");
-        this.app.show({ closing_events: true });
-        assert.ok(this.app.$el.css("display") == "block", "Modal shown with closing events");
-        this.app.$backdrop.trigger("click");
-        this.clock.tick(WAIT_FADE);
-        assert.ok(this.app.$el.css("display") == "none", "Modal hidden after backdrop click");
-    });
+QUnit.test("test dialog closing events", function(assert) {
+    this.app.show();
+    this.app.$backdrop.trigger("click");
+    assert.ok(this.app.$el.css("display") == "block", "Modal shown after backdrop click");
+    this.app.hide();
+    this.app.show({ closing_events: true });
+    assert.ok(this.app.$el.css("display") == "block", "Modal shown with closing events");
+    this.app.$backdrop.trigger("click");
+    assert.ok(this.app.$el.css("display") == "none", "Modal hidden after backdrop click");
+});
 
-    QUnit.test("test dialog rendering", function(assert) {
-        var before = this.app.$el.html();
-        this.app.render();
-        assert.ok(before == this.app.$el.html(), "Re-rendering successful");
-        this.app.options.title = "New Title";
-        this.app.render();
-        assert.ok(this.app.$header.find(".title").html() == "New Title", "Modal header has correct new title.");
-    });
+QUnit.test("test dialog rendering", function(assert) {
+    var before = this.app.$el.html();
+    this.app.render();
+    assert.ok(before == this.app.$el.html(), "Re-rendering successful");
+    this.app.options.title = "New Title";
+    this.app.render();
+    assert.ok(this.app.$header.find(".title").html() == "New Title", "Modal header has correct new title.");
+});
 
-    QUnit.test("test button states", function(assert) {
-        assert.ok(this.app.getButton("Ok").html() === "Ok", "Ok has correct label");
-        assert.ok(!this.app.getButton("Ok").prop("disabled"), "Ok is active");
-        assert.ok(!this.app.getButton("Cancel").prop("disabled"), "Cancel is active");
-        this.app.disableButton("Ok");
-        assert.ok(this.app.getButton("Ok").prop("disabled"), "Ok is disabled");
-        assert.ok(!this.app.getButton("Cancel").prop("disabled"), "Cancel is still active");
-        this.app.disableButton("Cancel");
-        assert.ok(this.app.getButton("Cancel").prop("disabled"), "Cancel is also disabled");
-        assert.ok(this.app.getButton("Ok").prop("disabled"), "Ok is still disabled");
-        this.app.enableButton("Ok");
-        assert.ok(this.app.getButton("Cancel").prop("disabled"), "Cancel is still disabled");
-        assert.ok(!this.app.getButton("Ok").prop("disabled"), "Ok is active again");
-    });
+QUnit.test("test button states", function(assert) {
+    assert.ok(this.app.getButton("Ok").html() === "Ok", "Ok has correct label");
+    assert.ok(!this.app.getButton("Ok").prop("disabled"), "Ok is active");
+    assert.ok(!this.app.getButton("Cancel").prop("disabled"), "Cancel is active");
+    this.app.disableButton("Ok");
+    assert.ok(this.app.getButton("Ok").prop("disabled"), "Ok is disabled");
+    assert.ok(!this.app.getButton("Cancel").prop("disabled"), "Cancel is still active");
+    this.app.disableButton("Cancel");
+    assert.ok(this.app.getButton("Cancel").prop("disabled"), "Cancel is also disabled");
+    assert.ok(this.app.getButton("Ok").prop("disabled"), "Ok is still disabled");
+    this.app.enableButton("Ok");
+    assert.ok(this.app.getButton("Cancel").prop("disabled"), "Cancel is still disabled");
+    assert.ok(!this.app.getButton("Ok").prop("disabled"), "Ok is active again");
 });

client/galaxy/scripts/qunit/tests/page_tests.js

@@ -1,56 +1,55 @@
 /* global define */
 
-define(["qunit/test-app", "layout/page", "layout/panel"], function(testApp, Page, Panel) {
-    "use strict";
-    Page = Page.default;
-    Panel = Panel.default;
-    QUnit.module("Page test", {
-        beforeEach: function() {
-            testApp.create();
-            $("body").append((this.$container = $("<div/>").css("display", "none")));
-        },
-        afterEach: function() {
-            testApp.destroy();
-        }
-    });
+import testApp from "qunit/test-app";
+import Page from "layout/page";
+import Panel from "layout/panel";
 
-    function _check(assert, page, sidePanels) {
-        assert.ok(page.$("#center").length == 1, "Center panel found.");
-        _.each(sidePanels, function(panelVisible, panelId) {
-            assert.ok(
-                page.$("#" + panelId).css("display") == panelVisible,
-                (panelVisible ? "" : "No") + " " + panelId + " panel found."
-            );
-        });
+QUnit.module("Page test", {
+    beforeEach: function() {
+        testApp.create();
+        $("body").append((this.$container = $("<div/>").css("display", "none")));
+    },
+    afterEach: function() {
+        testApp.destroy();
     }
+});
 
-    QUnit.test("test center/right", function(assert) {
-        this.$container.empty();
-        var page = new Page.View({
-            Right: Backbone.View.extend({
-                initialize: function() {
-                    this.setElement($("<div/>"));
-                    this.model = new Backbone.Model({});
-                }
-            })
-        }).render();
-        _check(assert, page, { left: "none", right: "block" });
+function _check(assert, page, sidePanels) {
+    assert.ok(page.$("#center").length == 1, "Center panel found.");
+    _.each(sidePanels, function(panelVisible, panelId) {
+        assert.ok(
+            page.$("#" + panelId).css("display") == panelVisible,
+            (panelVisible ? "" : "No") + " " + panelId + " panel found."
+        );
     });
-    QUnit.test("test center", function(assert) {
-        this.$container.empty();
-        var page = new Page.View({}).render();
-        _check(assert, page, { left: "none", right: "none" });
-    });
-    QUnit.test("test left/center", function(assert) {
-        this.$container.empty();
-        var page = new Page.View({
-            Left: Backbone.View.extend({
-                initialize: function() {
-                    this.setElement($("<div/>"));
-                    this.model = new Backbone.Model({});
-                }
-            })
-        }).render();
-        _check(assert, page, { left: "block", right: "none" });
-    });
+}
+
+QUnit.test("test center/right", function(assert) {
+    this.$container.empty();
+    var page = new Page.View({
+        Right: Backbone.View.extend({
+            initialize: function() {
+                this.setElement($("<div/>"));
+                this.model = new Backbone.Model({});
+            }
+        })
+    }).render();
+    _check(assert, page, { left: "none", right: "block" });
+});
+QUnit.test("test center", function(assert) {
+    this.$container.empty();
+    var page = new Page.View({}).render();
+    _check(assert, page, { left: "none", right: "none" });
+});
+QUnit.test("test left/center", function(assert) {
+    this.$container.empty();
+    var page = new Page.View({
+        Left: Backbone.View.extend({
+            initialize: function() {
+                this.setElement($("<div/>"));
+                this.model = new Backbone.Model({});
+            }
+        })
+    }).render();
+    _check(assert, page, { left: "block", right: "none" });
 });

client/galaxy/scripts/qunit/tests/popover_tests.js

@@ -1,41 +1,40 @@
 /* global define */
-define(["qunit/test-app", "mvc/ui/ui-misc", "mvc/ui/ui-popover"], function(testApp, Ui, Popover) {
-    "use strict";
-    Ui = Ui.default;
-    Popover = Popover.default;
-    QUnit.module("Popover test", {
-        beforeEach: function() {
-            testApp.create();
-            var self = this;
-            this.button = new Ui.Button({
-                title: "Test button",
-                onclick: function() {
-                    self.popover.show();
-                }
-            });
-            this.$parent = $("<div/>").append(this.button.$el);
-            this.popover = new Popover.View({
-                title: "Test Title",
-                body: "Test Body",
-                placement: "bottom",
-                container: this.button.$el
-            });
-            $("body").append(this.$parent);
-        },
-        afterEach: function() {
-            testApp.destroy();
-        }
-    });
+import testApp from "qunit/test-app";
+import Ui from "mvc/ui/ui-misc";
+import Popover from "mvc/ui/ui-popover";
 
-    QUnit.test("test popover visibility", function(assert) {
-        assert.ok(this.popover.$el.css("display") == "none", "Popover is hidden.");
-        this.button.$el.trigger("click");
-        assert.ok(this.popover.$el.css("display") == "block", "Popover is shown.");
-        assert.ok(this.popover.$el.hasClass("bottom"), "Popover at bottom.");
-        this.popover.hide();
-        assert.ok(this.popover.$el.css("display") == "none", "Popover is hidden manually.");
-        assert.ok(this.popover.$title.html() == "Test Title", "Initial title correct.");
-        this.popover.title("New Title");
-        assert.ok(this.popover.$title.html() == "New Title", "New title correct.");
-    });
+QUnit.module("Popover test", {
+    beforeEach: function() {
+        testApp.create();
+        var self = this;
+        this.button = new Ui.Button({
+            title: "Test button",
+            onclick: function() {
+                self.popover.show();
+            }
+        });
+        this.$parent = $("<div/>").append(this.button.$el);
+        this.popover = new Popover.View({
+            title: "Test Title",
+            body: "Test Body",
+            placement: "bottom",
+            container: this.button.$el
+        });
+        $("body").append(this.$parent);
+    },
+    afterEach: function() {
+        testApp.destroy();
+    }
+});
+
+QUnit.test("test popover visibility", function(assert) {
+    assert.ok(this.popover.$el.css("display") == "none", "Popover is hidden.");
+    this.button.$el.trigger("click");
+    assert.ok(this.popover.$el.css("display") == "block", "Popover is shown.");
+    assert.ok(this.popover.$el.hasClass("bottom"), "Popover at bottom.");
+    this.popover.hide();
+    assert.ok(this.popover.$el.css("display") == "none", "Popover is hidden manually.");
+    assert.ok(this.popover.$title.html() == "Test Title", "Initial title correct.");
+    this.popover.title("New Title");
+    assert.ok(this.popover.$title.html() == "New Title", "New title correct.");
 });

client/galaxy/scripts/qunit/tests/upload_dialog_tests.js

@@ -1,87 +1,83 @@
 /* global define */
-define(["qunit/test-app", "mvc/upload/upload-view"], function(testApp, GalaxyUpload) {
-    "use strict";
-    GalaxyUpload = GalaxyUpload.default;
-    QUnit.module("Upload dialog test", {
-        beforeEach: function() {
-            testApp.create();
-            this.app = new GalaxyUpload();
-        },
-        afterEach: function() {
-            testApp.destroy();
-        }
-    });
+import testApp from "qunit/test-app";
+import GalaxyUpload from "mvc/upload/upload-view";
 
-    QUnit.test("test initial dialog state", function(assert) {
-        $(this.app.ui_button.$el).trigger("click");
-        assert.ok(this.app.default_view.collection.length == 0, "Invalid initial upload item collection.");
-        assert.ok($("#btn-start").hasClass("disabled"), "Start button should be disabled.");
-        assert.ok($("#btn-stop").hasClass("disabled"), "Stop button should be disabled.");
-        assert.ok($("#btn-reset").hasClass("disabled"), "Reset button should be disabled.");
-    });
+QUnit.module("Upload dialog test", {
+    beforeEach: function() {
+        testApp.create();
+        this.app = new GalaxyUpload();
+    },
+    afterEach: function() {
+        testApp.destroy();
+    }
+});
 
-    QUnit.test("test adding/removing paste/fetch upload item", function(assert) {
-        $(this.app.ui_button.$el).trigger("click");
-        $("#btn-new").trigger("click");
-        assert.ok(!$("#btn-start").hasClass("disabled"), "Start button should be enabled.");
-        assert.ok($("#btn-stop").hasClass("disabled"), "Stop button should (still) be disabled.");
-        assert.ok(!$("#btn-reset").hasClass("disabled"), "Reset button should be enabled.");
-        assert.ok(
-            this.app.default_view.collection.length == 1,
-            "Invalid upload item collection length after adding item."
-        );
-        assert.ok(
-            $("#btn-new")
-                .find("i")
-                .hasClass("fa-edit"),
-            "Paste/fetch icon changed"
-        );
-        assert.ok($("#btn-start").hasClass("btn-primary"), "Start button should be enabled/highlighted.");
-        assert.ok(
-            $("#upload-row-0")
-                .find(".upload-symbol")
-                .hasClass("fa-trash-o"),
-            "Should show regular trash icon."
-        );
-        $("#upload-row-0")
-            .find(".upload-settings")
-            .trigger("click");
-        assert.ok(
-            $("#upload-row-0")
-                .find(".upload-settings-cover")
-                .css("display") == "none",
-            "Settings should be enabled."
-        );
-        $("#upload-row-0")
-            .find(".popover-close")
-            .trigger("click");
-        $("#btn-start").trigger("click");
-        assert.ok(
-            $("#upload-row-0")
-                .find(".upload-symbol")
-                .hasClass("fa-exclamation-triangle"),
-            "Upload attempt should have failed."
-        );
-        assert.ok(
-            $("#upload-row-0")
-                .find(".upload-settings")
-                .trigger("click")
-        );
-        assert.ok(
-            $("#upload-row-0")
-                .find(".upload-settings-cover")
-                .css("display") == "block",
-            "Settings should be disabled."
-        );
+QUnit.test("test initial dialog state", function(assert) {
+    $(this.app.ui_button.$el).trigger("click");
+    assert.ok(this.app.default_view.collection.length == 0, "Invalid initial upload item collection.");
+    assert.ok($("#btn-start").hasClass("disabled"), "Start button should be disabled.");
+    assert.ok($("#btn-stop").hasClass("disabled"), "Stop button should be disabled.");
+    assert.ok($("#btn-reset").hasClass("disabled"), "Reset button should be disabled.");
+});
+
+QUnit.test("test adding/removing paste/fetch upload item", function(assert) {
+    $(this.app.ui_button.$el).trigger("click");
+    $("#btn-new").trigger("click");
+    assert.ok(!$("#btn-start").hasClass("disabled"), "Start button should be enabled.");
+    assert.ok($("#btn-stop").hasClass("disabled"), "Stop button should (still) be disabled.");
+    assert.ok(!$("#btn-reset").hasClass("disabled"), "Reset button should be enabled.");
+    assert.ok(this.app.default_view.collection.length == 1, "Invalid upload item collection length after adding item.");
+    assert.ok(
+        $("#btn-new")
+            .find("i")
+            .hasClass("fa-edit"),
+        "Paste/fetch icon changed"
+    );
+    assert.ok($("#btn-start").hasClass("btn-primary"), "Start button should be enabled/highlighted.");
+    assert.ok(
         $("#upload-row-0")
             .find(".upload-symbol")
-            .trigger("click");
-        assert.ok(this.app.default_view.collection.length == 0, "Removing item from collection failed.");
-    });
+            .hasClass("fa-trash-o"),
+        "Should show regular trash icon."
+    );
+    $("#upload-row-0")
+        .find(".upload-settings")
+        .trigger("click");
+    assert.ok(
+        $("#upload-row-0")
+            .find(".upload-settings-cover")
+            .css("display") == "none",
+        "Settings should be enabled."
+    );
+    $("#upload-row-0")
+        .find(".popover-close")
+        .trigger("click");
+    $("#btn-start").trigger("click");
+    assert.ok(
+        $("#upload-row-0")
+            .find(".upload-symbol")
+            .hasClass("fa-exclamation-triangle"),
+        "Upload attempt should have failed."
+    );
+    assert.ok(
+        $("#upload-row-0")
+            .find(".upload-settings")
+            .trigger("click")
+    );
+    assert.ok(
+        $("#upload-row-0")
+            .find(".upload-settings-cover")
+            .css("display") == "block",
+        "Settings should be disabled."
+    );
+    $("#upload-row-0")
+        .find(".upload-symbol")
+        .trigger("click");
+    assert.ok(this.app.default_view.collection.length == 0, "Removing item from collection failed.");
+});
 
-    QUnit.test("test ftp popup", function(assert) {
-        $(this.app.ui_button.$el).trigger("click");
-        $("#btn-ftp").trigger("click");
-        assert.ok($(".upload-ftp").length == 1, "Should show ftp popover.");
-    });
+QUnit.test("test ftp popup", function(assert) {
+    $(this.app.ui_button.$el).trigger("click");
+    $("#btn-ftp").trigger("click");
+    assert.ok($(".upload-ftp").length == 1, "Should show ftp popover.");
 });

client/galaxy/scripts/qunit/tests/utils_test.js

@@ -1,20 +0,0 @@
-/* global define */
-define(["qunit/test-app", "utils/utils"], function(testApp, Utils) {
-    "use strict";
-    Utils = Utils.default;
-    QUnit.module("Utils test", {});
-
-    QUnit.test("isEmpty", function(assert) {
-        assert.ok(Utils.isEmpty([]), "Empty array");
-        assert.ok(Utils.isEmpty(["data", undefined]), "Array contains `undefined`");
-        assert.ok(Utils.isEmpty(["data", null]), "Array contains `null`");
-        assert.ok(Utils.isEmpty(["data", "__null__"]), "Array contains `__null__`");
-        assert.ok(Utils.isEmpty(["data", "__undefined__"]), "Array contains `__undefined__`");
-        assert.ok(Utils.isEmpty(null), "Array is null");
-        assert.ok(Utils.isEmpty("__null__"), "Array is __null__");
-        assert.ok(Utils.isEmpty("__undefined__"), "Array is __undefined__");
-        assert.ok(!Utils.isEmpty(["data"]), "Array contains `data`");
-        assert.ok(!Utils.isEmpty(1), "Value is int");
-        assert.ok(!Utils.isEmpty(0), "Value is zero");
-    });
-});

client/galaxy/scripts/qunit/tests/utils_tests.js

@@ -0,0 +1,19 @@
+/* global define */
+import testApp from "qunit/test-app";
+import Utils from "utils/utils";
+
+QUnit.module("Utils test", {});
+
+QUnit.test("isEmpty", function(assert) {
+    assert.ok(Utils.isEmpty([]), "Empty array");
+    assert.ok(Utils.isEmpty(["data", undefined]), "Array contains `undefined`");
+    assert.ok(Utils.isEmpty(["data", null]), "Array contains `null`");
+    assert.ok(Utils.isEmpty(["data", "__null__"]), "Array contains `__null__`");
+    assert.ok(Utils.isEmpty(["data", "__undefined__"]), "Array contains `__undefined__`");
+    assert.ok(Utils.isEmpty(null), "Array is null");
+    assert.ok(Utils.isEmpty("__null__"), "Array is __null__");
+    assert.ok(Utils.isEmpty("__undefined__"), "Array is __undefined__");
+    assert.ok(!Utils.isEmpty(["data"]), "Array contains `data`");
+    assert.ok(!Utils.isEmpty(1), "Value is int");
+    assert.ok(!Utils.isEmpty(0), "Value is zero");
+});

client/galaxy/scripts/viz/trackster.js

@@ -572,7 +572,7 @@ var TracksterView = Backbone.View.extend({
         // close selection/finalize template
         html += `</select></div><div style="clear: both;"></div></div><div class="form-row">Is the build not listed here? <a href="${
             Galaxy.root
-        }user/dbkeys?use_panels=True">Add a Custom Build</a></div></form>`;
+        }custom_builds">Add a Custom Build</a></div></form>`;
 
         // return
         return html;

client/galaxy/style/less/base.less

@@ -1108,6 +1108,37 @@ ul.manage-table-actions li {
     margin-left: 0.5em;
 }
 
+.state-progress {
+    border: 1px solid gray;
+    position: relative;
+
+    margin-top: 2px;
+    margin-bottom: 1px;
+
+    .info {
+        color: @black;
+        background: @white;
+    }
+
+    .new {
+        background: @state-default-bg;
+    }
+
+    .running {
+        background: @state-running-bg;
+    }
+
+    .ok {
+        background: @state-success-bg;
+    }
+
+    .note {
+        margin-left: 1em;
+        position: absolute;
+    }
+
+}
+
 // State colors
 
 .state-color-new {

client/galaxy/style/less/dataset.less

@@ -176,6 +176,22 @@
             }
         }
     }
+
+    &.state-loading {
+        background: @state-default-bg;
+        .state-icon {
+            .state-icon-running;
+        }
+    }
+
+}
+
+.blinking {
+  animation: blinker 500ms linear infinite;
+}
+
+@keyframes blinker {
+  50% { opacity: 0; }
 }
 
 // ---------------------------------------------------------------------------- datasets as list-items

client/galaxy/style/less/list-item.less

@@ -86,6 +86,13 @@
                 color: inherit;
             }
         }
+        .state-description {
+            color: #777;
+            font-size: 90%;
+            a {
+                color: inherit;
+            }
+        }
     }
 
     .primary-actions {

client/gulpfile.js

@@ -40,20 +40,24 @@ var paths = {
     libs: ['galaxy/scripts/libs/**/*.js']
 };
 
-var nopack_mode = function(){
-    return process.env.GXY_NOPACK || false;
+var dev_mode = function(){
+    return process.env.NODE_ENV != "production";
+};
+
+var source_maps = function(){
+    return dev_mode() || process.env.GXY_BUILD_SOURCEMAPS !== undefined;
 };
 
 gulp.task('scripts', function() {
   return gulp.src(paths.scripts)
     .pipe(plumber())
     .pipe(cached('scripts'))
-    //.pipe(sourcemaps.init())
+    .pipe(gulpif(source_maps, sourcemaps.init()))
     .pipe(babel({
         plugins: ['transform-es2015-modules-amd']
     }))
-    .pipe(gulpif(nopack_mode, beautify(), uglify()))
-    //.pipe(sourcemaps.write('../maps/'))
+    .pipe(gulpif(dev_mode, beautify(), uglify()))
+    .pipe(gulpif(source_maps, sourcemaps.write('../maps/')))
     .pipe(gulp.dest('../static/scripts/'));
 });
 

client/karma.config.js

@@ -15,29 +15,30 @@ var single_pack_mode = function(){
 };
 
 var TESTS_SEPARATE_PACKS = [
-    {pattern: 'galaxy/scripts/qunit/tests/list-of-pairs-collection-creator.js', watched: false},
-    {pattern: 'galaxy/scripts/qunit/tests/galaxy-app-base.js', watched: false},
-    {pattern: 'galaxy/scripts/qunit/tests/graph.js', watched: false},
-    {pattern: 'galaxy/scripts/qunit/tests/hda-base.js', watched: false},
+    {pattern: 'galaxy/scripts/qunit/tests/galaxy_app_base_tests.js', watched: false},
+    // Something is funky with form_tests.js - needs to come before all other tests.
+    {pattern: 'galaxy/scripts/qunit/tests/form_tests.js', watched: false},
+    {pattern: 'galaxy/scripts/qunit/tests/upload_dialog_tests.js', watched: false},
+    // Something is funky with masthead_tests - needs to come before one of the other
+    // tests - not sure which though...
+    {pattern: 'galaxy/scripts/qunit/tests/masthead_tests.js', watched: false},
+    //{pattern: 'galaxy/scripts/qunit/tests/list_of_pairs_collection_creator_tests.js', watched: false},
+    {pattern: 'galaxy/scripts/qunit/tests/graph_tests.js', watched: false},
+    {pattern: 'galaxy/scripts/qunit/tests/hda_base_tests.js', watched: false},
     {pattern: 'galaxy/scripts/qunit/tests/history_contents_model_tests.js', watched: false},
-    {pattern: 'galaxy/scripts/qunit/tests/job-dag.js', watched: false},
-    {pattern: 'galaxy/scripts/qunit/tests/metrics-logger.js', watched: false},
+    {pattern: 'galaxy/scripts/qunit/tests/job_dag_tests.js', watched: false},
+    {pattern: 'galaxy/scripts/qunit/tests/metrics_logger_tests.js', watched: false},
     {pattern: 'galaxy/scripts/qunit/tests/popover_tests.js', watched: false},
-    {pattern: 'galaxy/scripts/qunit/tests/utils_test.js', watched: false},
+    {pattern: 'galaxy/scripts/qunit/tests/utils_tests.js', watched: false},
     {pattern: 'galaxy/scripts/qunit/tests/page_tests.js', watched: false},
     {pattern: 'galaxy/scripts/qunit/tests/workflow_editor_tests.js', watched: false},
+    {pattern: 'galaxy/scripts/qunit/tests/modal_tests.js', watched: false},
     // The following tests don't work for state reasons:
 
     // Error: Following test works on its own or with rest but not with 
-    //     list-of-pairs-collection-creator in the same suite. Not as much isolation
+    //     list_of_pairs_collection_creator in the same suite. Not as much isolation
     //     as seperate page setup of previous runner.
-    // 'galaxy/scripts/qunit/tests/ui_tests.js',
-    // Error: things displayed wrong I guess cause assertions fail on CSS stuff
-    // 'galaxy/scripts/qunit/tests/modal_tests.js',
-    // 'galaxy/scripts/qunit/tests/upload_dialog_tests.js',
-    // Error: Cannot find module "libs/bibtexParse"
-    // 'galaxy/scripts/qunit/tests/form_tests.js',
-    // 'galaxy/scripts/qunit/tests/masthead_tests.js',
+    // {pattern: 'galaxy/scripts/qunit/tests/ui_tests.js', watched: false},
 ];
 
 var TESTS_AS_SINGLE_PACK = [

client/package.json

@@ -30,16 +30,20 @@
   },
   "scripts": {
     "watch": "gulp stage-libs && concurrently \"yarn run webpack-watch\" \"yarn run gulp watch\" \"yarn run style-watch\"",
-    "build": "gulp stage-libs && concurrently \"yarn run style\" \"yarn run webpack\" \"yarn run gulp clean && yarn run gulp\"",
-    "webpack": "webpack -p",
-    "webpack-watch": "webpack -p --watch",
-    "webpack-dev": "webpack -d",
-    "webpack-dev-watch": "webpack -d --watch",
+    "build": "NODE_ENV=development gulp stage-libs && concurrently \"yarn run style\" \"yarn run webpack\" \"yarn run gulp clean && yarn run gulp\"",
+    "build-production": "NODE_ENV=production gulp stage-libs && concurrently \"yarn run style\" \"yarn run webpack-production\" \"yarn run gulp clean && yarn run gulp-production\"",
+    "build-production-maps": "NODE_ENV=production gulp stage-libs && concurrently \"yarn run style\" \"yarn run webpack-production-maps\" \"yarn run gulp clean && yarn run gulp-production-maps\"",
+    "webpack": "webpack -d",
+    "webpack-watch": "webpack -d --watch",
+    "webpack-production": "webpack -p",
+    "webpack-production-maps": "GXY_BUILD_SOURCEMAPS=1 webpack -p",
     "style": "grunt style",
     "style-watch": "grunt watch-style",
-    "test": "karma start --verbose --single-run --browsers PhantomJS karma.config.js",
-    "test-watch": "karma start --verbose --browsers PhantomJS karma.config.js",
+    "test": "GALAXY_TEST_AS_SINGLE_PACK=true karma start --verbose --single-run --browsers PhantomJS karma.config.js",
+    "test-watch": "GALAXY_TEST_AS_SINGLE_PACK=true karma start --verbose --browsers PhantomJS karma.config.js",
     "gulp": "gulp",
+    "gulp-production": "NODE_ENV=production gulp",
+    "gulp-production-maps": "GXY_BUILD_SOURCEMAPS=1 NODE_ENV=production gulp",
     "build-toolshed": "grunt --app=toolshed",
     "jshint": "jshint --exclude='galaxy/scripts/libs/**' galaxy/scripts/**/*.js",
     "build-charts": "webpack -p --config ../config/plugins/visualizations/charts/webpack.config.js",

client/webpack.config.js

@@ -37,8 +37,7 @@ var commonLibs = [
     "onload"
 ];
 
-module.exports = {
-    //devtool: "source-map",
+let buildconfig = {
     entry: {
         libs: commonLibs,
         login: "./galaxy/scripts/apps/login.js",
@@ -104,6 +103,9 @@ module.exports = {
             name: "libs",
             filename: "libs.bundled.js"
         }),
+        new webpack.SourceMapDevToolPlugin({
+            filename: '[name].js.map',
+        }),
         // this plugin allows using the following keys/globals in scripts (w/o req'ing them first)
         // and webpack will automagically require them in the bundle for you
         new webpack.ProvidePlugin({
@@ -116,3 +118,9 @@ module.exports = {
         // new webpack.optimize.LimitChunkCountPlugin({ maxChunks: 1 })
     ]
 };
+
+if (process.env.GXY_BUILD_SOURCEMAPS){
+    buildconfig.devtool = 'source-map';
+}
+
+module.exports = buildconfig;

config/datatypes_conf.xml.sample

@@ -39,7 +39,7 @@
     <datatype extension="len" type="galaxy.datatypes.chrominfo:ChromInfo" display_in_upload="true">
       <converter file="len_to_linecount.xml" target_datatype="linecount" />
     </datatype>
-    <datatype extension="dmnd" type="galaxy.datatypes.binary:Binary:DMND" display_in_upload="false"/>
+    <datatype extension="dmnd" type="galaxy.datatypes.binary:DMND" display_in_upload="false"/>
     <datatype extension="bigbed" type="galaxy.datatypes.binary:BigBed" mimetype="application/octet-stream" display_in_upload="true">
       <display file="ucsc/bigbed.xml" />
       <display file="igb/bb.xml" />
@@ -146,7 +146,7 @@
     <datatype extension="toolshed.gz" type="galaxy.datatypes.binary:Binary" mimetype="multipart/x-gzip" subclass="true" />
     <datatype extension="h5" type="galaxy.datatypes.binary:H5" mimetype="application/octet-stream" display_in_upload="true"/>
     <datatype extension="mz5" type="galaxy.datatypes.binary:H5" subclass="true" mimetype="application/octet-stream" display_in_upload="true"/>
-    <datatype extension="cool" type="galaxy.datatypes.binary:Cool" subclass="true" mimetype="application/octet-stream" display_in_upload="true"/>
+    <datatype extension="cool" type="galaxy.datatypes.binary:Cool" mimetype="application/octet-stream" display_in_upload="true"/>
     <datatype extension="html" type="galaxy.datatypes.text:Html" mimetype="text/html"/>
     <datatype extension="interval" type="galaxy.datatypes.interval:Interval" display_in_upload="true" description="File must start with definition line in the following format (columns may be in any order)." >
       <converter file="interval_to_bed_converter.xml" target_datatype="bed"/>
@@ -237,7 +237,7 @@
     <datatype extension="sf3" type="galaxy.datatypes.proteomics:Sf3" display_in_upload="true" />
     <datatype extension="cps" type="galaxy.datatypes.binary:Binary" subclass="true" display_in_upload="true" />
     <datatype extension="ct" type="galaxy.datatypes.tabular:ConnectivityTable" display_in_upload="true"/>
-    <datatype extension="postgresql" type="galaxy.datatypes.binary:PostgresqlArchive" subclass="True" display_in_upload="True"/>
+    <datatype extension="postgresql" type="galaxy.datatypes.binary:PostgresqlArchive" display_in_upload="True"/>
     <datatype extension="searchgui_archive" type="galaxy.datatypes.binary:SearchGuiArchive" display_in_upload="true"/>
     <datatype extension="fast5.tar" type="galaxy.datatypes.binary:Fast5Archive" display_in_upload="true"/>
     <datatype extension="fast5.tar.gz" type="galaxy.datatypes.binary:Fast5ArchiveGz" display_in_upload="true"/>
@@ -282,13 +282,13 @@
     <datatype extension="qualillumina" type="galaxy.datatypes.qualityscore:QualityScoreIllumina" display_in_upload="true"/>
     <datatype extension="qualsolid" type="galaxy.datatypes.qualityscore:QualityScoreSOLiD" display_in_upload="true"/>
     <datatype extension="qual454" type="galaxy.datatypes.qualityscore:QualityScore454" display_in_upload="true"/>
-    <datatype extension="Roadmaps" type="galaxy.datatypes.assembly:Roadmaps" display_in_upload="false"/>
+    <datatype extension="roadmaps" type="galaxy.datatypes.assembly:Roadmaps" display_in_upload="false"/>
     <datatype extension="sam" type="galaxy.datatypes.tabular:Sam" display_in_upload="true">
         <converter file="sam_to_bam.xml" target_datatype="bam"/>
         <converter file="sam_to_bigwig_converter.xml" target_datatype="bigwig"/>
     </datatype>
     <datatype extension="scf" type="galaxy.datatypes.binary:Scf" mimetype="application/octet-stream" display_in_upload="true" description="A binary sequence file in 'scf' format with a '.scf' file extension.  You must manually select this 'File Format' when uploading the file." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Scf"/>
-    <datatype extension="Sequences" type="galaxy.datatypes.assembly:Sequences" display_in_upload="false"/>
+    <datatype extension="sequences" type="galaxy.datatypes.assembly:Sequences" display_in_upload="false"/>
     <datatype extension="snpeffdb" type="galaxy.datatypes.text:SnpEffDb" display_in_upload="true"/>
     <datatype extension="snpsiftdbnsfp" type="galaxy.datatypes.text:SnpSiftDbNSFP" display_in_upload="true"/>
     <datatype extension="dbnsfp.tabular" type="galaxy.datatypes.tabular:Tabular" subclass="true" display_in_upload="true">
@@ -518,7 +518,7 @@
     <datatype extension="vtkascii" type="galaxy.datatypes.constructive_solid_geometry:VtkAscii" display_in_upload="true" />
     <datatype extension="vtkbinary" type="galaxy.datatypes.constructive_solid_geometry:VtkBinary" display_in_upload="true" />
     <!-- Metagenomic Datatypes -->
-    <datatype extension="biom1" type="galaxy.datatypes.text:Biom1" display_in_upload="true" subclass="true" mimetype="application/json">
+    <datatype extension="biom1" type="galaxy.datatypes.text:Biom1" display_in_upload="true" mimetype="application/json">
         <display file="biom/biom_simple.xml" />
         <converter file="biom1_to_biom2.xml" target_datatype="biom2"/>
     </datatype>

config/galaxy.ini.sample

@@ -1122,6 +1122,14 @@ use_interactive = True
 # invocation to schedule indefinitely. The default corresponds to 1 month.
 #maximum_workflow_invocation_duration = 2678400
 
+# Specify a maximum number of jobs that any given workflow scheduling iteration can create.
+# Set this to a positive integer to prevent large collection jobs in a workflow from
+# preventing other jobs from executing. This may also mitigate memory issues associated with
+# scheduling workflows at the expense of increased total DB traffic because model objects
+# are expunged from the SQL alchemy session between workflow invocation scheduling iterations.
+# Set to -1 to disable any such maximum (the default).
+#maximum_workflow_jobs_per_scheduling_iteration = -1
+
 # Force serial scheduling of workflows within the context of a particular history
 #history_local_serial_workflow_scheduling=False
 

config/plugins/interactive_environments/jupyter/config/allowed_images.yml.sample

@@ -6,7 +6,7 @@
 # appropriate `apt-get/pip install` statements.
 ---
 -
-    image: quay.io/bgruening/docker-jupyter-notebook:17.05
+    image: quay.io/bgruening/docker-jupyter-notebook:17.09
     description: |
         The Jupyter notebook is the next iteration of IPython, allowing
         analysis in many different languages. This image features the Python,

lib/galaxy/config.py

@@ -385,6 +385,7 @@ class Configuration(object):
         self.history_local_serial_workflow_scheduling = string_as_bool(kwargs.get('history_local_serial_workflow_scheduling', 'False'))
         self.parallelize_workflow_scheduling_within_histories = string_as_bool(kwargs.get('parallelize_workflow_scheduling_within_histories', 'False'))
         self.maximum_workflow_invocation_duration = int(kwargs.get("maximum_workflow_invocation_duration", 2678400))
+        self.maximum_workflow_jobs_per_scheduling_iteration = int(kwargs.get("maximum_workflow_jobs_per_scheduling_iteration", -1))
 
         self.cache_user_job_count = string_as_bool(kwargs.get('cache_user_job_count', False))
         self.pbs_application_server = kwargs.get('pbs_application_server', "")

lib/galaxy/dataset_collections/builder.py

@@ -24,6 +24,7 @@ def set_collection_elements(dataset_collection, type, dataset_instances):
         element_index += 1
 
     dataset_collection.elements = elements
+    dataset_collection.element_count = element_index
     return dataset_collection
 
 

lib/galaxy/dataset_collections/matching.py

@@ -44,27 +44,35 @@ class MatchingCollections(object):
         self.linked_structure = None
         self.unlinked_structures = []
         self.collections = {}
+        self.subcollection_types = {}
 
     def __attempt_add_to_linked_match(self, input_name, hdca, collection_type_description, subcollection_type):
         structure = get_structure(hdca, collection_type_description, leaf_subcollection_type=subcollection_type)
         if not self.linked_structure:
             self.linked_structure = structure
             self.collections[input_name] = hdca
+            self.subcollection_types[input_name] = subcollection_type
         else:
             if not self.linked_structure.can_match(structure):
                 raise exceptions.MessageException(CANNOT_MATCH_ERROR_MESSAGE)
             self.collections[input_name] = hdca
+            self.subcollection_types[input_name] = subcollection_type
 
     def slice_collections(self):
         return self.linked_structure.walk_collections(self.collections)
 
+    def subcollection_mapping_type(self, input_name):
+        return self.subcollection_types[input_name]
+
     @property
     def structure(self):
-        """Yield cross product of all unlinked datasets to linked dataset."""
+        """Yield cross product of all unlinked collections structures to linked collection structure."""
         effective_structure = leaf
         for unlinked_structure in self.unlinked_structures:
             effective_structure = effective_structure.multiply(unlinked_structure)
-        linked_structure = self.linked_structure or leaf
+        linked_structure = self.linked_structure
+        if linked_structure is None:
+            linked_structure = leaf
         effective_structure = effective_structure.multiply(linked_structure)
         return None if effective_structure.is_leaf else effective_structure
 

lib/galaxy/dataset_collections/structure.py

@@ -1,12 +1,17 @@
 """ Module for reasoning about structure of and matching hierarchical collections of data.
 """
 import logging
-log = logging.getLogger(__name__)
+
+import six
 
 from .type_description import map_over_collection_type
 
+log = logging.getLogger(__name__)
 
+
+@six.python_2_unicode_compatible
 class Leaf(object):
+    children_known = True
 
     def __len__(self):
         return 1
@@ -18,18 +23,60 @@ class Leaf(object):
     def clone(self):
         return self
 
-    def multiply(self, other_structure):
-        return other_structure.clone()
+    def multiply(self, other_structure, uninitialized=False):
+        if not uninitialized:
+            return other_structure.clone()
+        else:
+            return UnitializedTree(other_structure.collection_type_description)
+
+    def sliced_collection_type(self, collection):
+        return input
+
+    def __str__(self):
+        return "Leaf[]"
 
 
 leaf = Leaf()
 
 
-class Tree(object):
+class BaseTree(object):
+
+    def __init__(self, collection_type_description):
+        self.collection_type_description = collection_type_description
+
+
+@six.python_2_unicode_compatible
+class UnitializedTree(BaseTree):
+    children_known = False
+
+    def clone(self):
+        return self
+
+    @property
+    def is_leaf(self):
+        return False
+
+    def __len__(self):
+        raise Exception("Unknown length")
+
+    def multiply(self, other_structure, uninitialized=False):
+        if other_structure.is_leaf:
+            return self.clone()
+
+        new_collection_type = self.collection_type_description.multiply(other_structure.collection_type_description)
+        return UnitializedTree(new_collection_type)
+
+    def __str__(self):
+        return "UnitializedTree[collection_type=%s]" % self.collection_type_description
+
+
+@six.python_2_unicode_compatible
+class Tree(BaseTree):
+    children_known = True
 
     def __init__(self, children, collection_type_description):
+        super(Tree, self).__init__(collection_type_description)
         self.children = children
-        self.collection_type_description = collection_type_description
 
     @staticmethod
     def for_dataset_collection(dataset_collection, collection_type_description):
@@ -107,14 +154,14 @@ class Tree(object):
             element_identifiers=element_identifiers,
         )
 
-    def multiply(self, other_structure):
+    def multiply(self, other_structure, uninitialized=False):
         if other_structure.is_leaf:
             return self.clone()
 
         new_collection_type = self.collection_type_description.multiply(other_structure.collection_type_description)
         new_children = []
         for (identifier, structure) in self.children:
-            new_children.append((identifier, structure.multiply(other_structure)))
+            new_children.append((identifier, structure.multiply(other_structure, uninitialized=uninitialized)))
 
         return Tree(new_children, new_collection_type)
 
@@ -122,6 +169,30 @@ class Tree(object):
         cloned_children = [(_[0], _[1].clone()) for _ in self.children]
         return Tree(cloned_children, self.collection_type_description)
 
+    def __str__(self):
+        return "Tree[collection_type=%s,children=%s]" % (self.collection_type_description, ",".join(map(lambda identifier_and_element: "%s=%s" % (identifier_and_element[0], identifier_and_element[1]), self.children)))
+
+
+def tool_output_to_structure(get_sliced_input_collection_type, tool_output, collections_manager):
+    if not tool_output.collection:
+        tree = leaf
+    else:
+        collection_type_descriptions = collections_manager.collection_type_descriptions
+        # Okay this is ToolCollectionOutputStructure not a Structure - different
+        # concepts of structure.
+        if tool_output.dynamic_structure:
+            # Two cases collection_type_source and collection_type right?
+            tree = UnitializedTree(collection_type_descriptions.for_type_description("list"))  # list is obviously wrong...
+        else:
+            structured_like = tool_output.structure.structured_like
+            if structured_like:
+                collection_type = get_sliced_input_collection_type(structured_like)
+            else:
+                collection_type = tool_output.structure.collection_type
+            tree = UnitializedTree(collection_type)
+
+    return tree
+
 
 def dict_map(func, input_dict):
     return dict((k, func(v)) for k, v in input_dict.items())
@@ -131,4 +202,5 @@ def get_structure(dataset_collection_instance, collection_type_description, leaf
     if leaf_subcollection_type:
         collection_type_description = collection_type_description.effective_collection_type_description(leaf_subcollection_type)
 
-    return Tree.for_dataset_collection(dataset_collection_instance.collection, collection_type_description)
+    collection = dataset_collection_instance.collection
+    return Tree.for_dataset_collection(collection, collection_type_description)

lib/galaxy/dataset_collections/type_description.py

@@ -8,6 +8,7 @@ class CollectionTypeDescriptionFactory(object):
         self.type_registry = type_registry
 
     def for_collection_type(self, collection_type):
+        assert collection_type is not None
         return CollectionTypeDescription(collection_type, self)
 
 

lib/galaxy/datatypes/annotation.py

@@ -1,7 +1,7 @@
 import logging
 import tarfile
 
-from galaxy.datatypes.binary import Binary, CompressedArchive
+from galaxy.datatypes.binary import CompressedArchive
 from galaxy.datatypes.data import get_file_peek, Text
 from galaxy.util import nice_size
 
@@ -82,6 +82,3 @@ class Augustus(CompressedArchive):
         except Exception as e:
             log.warning('%s, sniff Exception: %s', self, e)
         return False
-
-
-Binary.register_sniffable_binary_format("augustus", "augustus", Augustus)

lib/galaxy/datatypes/assembly.py

@@ -71,6 +71,7 @@ class Amos(data.Text):
 class Sequences(sequence.Fasta):
     """Class describing the Sequences file generated by velveth """
     edam_data = "data_0925"
+    file_ext = 'sequences'
 
     def sniff(self, filename):
         """
@@ -110,6 +111,7 @@ class Sequences(sequence.Fasta):
 class Roadmaps(data.Text):
     """Class describing the Sequences file generated by velveth """
     edam_format = "format_2561"
+    file_ext = 'roadmaps'
 
     def sniff(self, filename):
         """

lib/galaxy/datatypes/binary.py

@@ -33,33 +33,17 @@ log = logging.getLogger(__name__)
 class Binary(data.Data):
     """Binary data"""
     edam_format = "format_2333"
-    sniffable_binary_formats = []
     unsniffable_binary_formats = []
 
     @staticmethod
     def register_sniffable_binary_format(data_type, ext, type_class):
-        Binary.sniffable_binary_formats.append({"type": data_type, "ext": ext.lower(), "class": type_class})
+        """Deprecated method."""
+        pass
 
     @staticmethod
     def register_unsniffable_binary_ext(ext):
         Binary.unsniffable_binary_formats.append(ext.lower())
 
-    @staticmethod
-    def is_sniffable_binary(filename):
-        format_information = None
-        for format in Binary.sniffable_binary_formats:
-            format_instance = format["class"]()
-            try:
-                if format_instance.sniff(filename):
-                    format_information = (format["type"], format["ext"])
-                    break
-            except Exception:
-                # Sniffer raised exception, could be any number of
-                # reasons for this so there is not much to do besides
-                # trying next sniffer.
-                pass
-        return format_information
-
     @staticmethod
     def is_ext_unsniffable(ext):
         return ext in Binary.unsniffable_binary_formats
@@ -128,9 +112,6 @@ class Idat(Binary):
             return False
 
 
-Binary.register_sniffable_binary_format("idat", "idat", Idat)
-
-
 class Cel(Binary):
 
     """Binary data in CEL format."""
@@ -160,9 +141,6 @@ class Cel(Binary):
             return False
 
 
-Binary.register_sniffable_binary_format("cel", "cel", Cel)
-
-
 class CompressedArchive(Binary):
     """
         Class describing an compressed binary file
@@ -595,9 +573,6 @@ class Bam(Binary):
         return dataproviders.dataset.SamtoolsDataProvider(dataset_source, **settings)
 
 
-Binary.register_sniffable_binary_format("bam", "bam", Bam)
-
-
 class CRAM(Binary):
     file_ext = "cram"
     edam_format = "format_3462"
@@ -667,13 +642,9 @@ class CRAM(Binary):
             return False
 
 
-Binary.register_sniffable_binary_format('cram', 'cram', CRAM)
-
-
 class BaseBcf(Binary):
     edam_format = "format_3020"
     edam_data = "data_3498"
-    file_ext = "bcf"
 
 
 class Bcf(BaseBcf):
@@ -688,6 +659,7 @@ class Bcf(BaseBcf):
     >>> Bcf().sniff(fname)
     False
     """
+    file_ext = "bcf"
 
     MetadataElement(name="bcf_index", desc="BCF Index File", param=metadata.FileParameter, file_ext="csi", readonly=True, no_value=None, visible=False, optional=True)
 
@@ -731,9 +703,6 @@ class Bcf(BaseBcf):
         dataset.metadata.bcf_index = index_file
 
 
-Binary.register_sniffable_binary_format("bcf", "bcf", Bcf)
-
-
 class BcfUncompressed(Bcf):
     """
     Class describing an uncompressed BCF file
@@ -759,9 +728,6 @@ class BcfUncompressed(Bcf):
             return False
 
 
-Binary.register_sniffable_binary_format("bcf_uncompressed", "bcf_uncompressed", BcfUncompressed)
-
-
 class H5(Binary):
     """
     Class describing an HDF5 file
@@ -942,11 +908,6 @@ class Cool(H5):
             return "Cool (HDF5) file (%s)." % (nice_size(dataset.get_size()))
 
 
-Binary.register_sniffable_binary_format("cool", "cool", Cool)
-Binary.register_sniffable_binary_format("biom2", "biom2", Biom2)
-Binary.register_sniffable_binary_format("h5", "h5", H5)
-
-
 class Scf(Binary):
     """Class describing an scf binary sequence file"""
     edam_format = "format_1632"
@@ -1003,9 +964,6 @@ class Sff(Binary):
             return "Binary sff file (%s)" % (nice_size(dataset.get_size()))
 
 
-Binary.register_sniffable_binary_format("sff", "sff", Sff)
-
-
 class BigWig(Binary):
     """
     Accessing binary BigWig files from UCSC.
@@ -1014,6 +972,7 @@ class BigWig(Binary):
     """
     edam_format = "format_3006"
     edam_data = "data_3002"
+    file_ext = "bigwig"
     track_type = "LineTrack"
     data_sources = {"data_standalone": "bigwig"}
 
@@ -1047,13 +1006,11 @@ class BigWig(Binary):
             return "Binary UCSC %s file (%s)" % (self._name, nice_size(dataset.get_size()))
 
 
-Binary.register_sniffable_binary_format("bigwig", "bigwig", BigWig)
-
-
 class BigBed(BigWig):
     """BigBed support from UCSC."""
     edam_format = "format_3004"
     edam_data = "data_3002"
+    file_ext = "bigbed"
     data_sources = {"data_standalone": "bigbed"}
 
     def __init__(self, **kwd):
@@ -1062,9 +1019,6 @@ class BigBed(BigWig):
         self._name = "BigBed"
 
 
-Binary.register_sniffable_binary_format("bigbed", "bigbed", BigBed)
-
-
 class TwoBit(Binary):
     """Class describing a TwoBit format nucleotide file"""
     edam_format = "format_3009"
@@ -1097,9 +1051,6 @@ class TwoBit(Binary):
             return "Binary TwoBit format nucleotide file (%s)" % (nice_size(dataset.get_size()))
 
 
-Binary.register_sniffable_binary_format("twobit", "twobit", TwoBit)
-
-
 @dataproviders.decorators.has_dataproviders
 class SQlite(Binary):
     """Class describing a Sqlite database """
@@ -1191,9 +1142,6 @@ class SQlite(Binary):
         return dataproviders.dataset.SQliteDataDictProvider(dataset_source, **settings)
 
 
-# Binary.register_sniffable_binary_format("sqlite", "sqlite", SQlite)
-
-
 class GeminiSQLite(SQlite):
     """Class describing a Gemini Sqlite database """
     MetadataElement(name="gemini_version", default='0.10.0', param=MetadataParameter, desc="Gemini Version",
@@ -1322,14 +1270,6 @@ class IdpDB(SQlite):
             return "IDPickerDB SQLite file (%s)" % (nice_size(dataset.get_size()))
 
 
-Binary.register_sniffable_binary_format("gemini.sqlite", "gemini.sqlite", GeminiSQLite)
-Binary.register_sniffable_binary_format("idpdb", "idpdb", IdpDB)
-Binary.register_sniffable_binary_format("mz.sqlite", "mz.sqlite", MzSQlite)
-# FIXME: We need to register specialized sqlite formats before sqlite, since register_sniffable_binary_format and is_sniffable_binary called in upload.py
-# ignores sniff order declared in datatypes_conf.xml
-Binary.register_sniffable_binary_format("sqlite", "sqlite", SQlite)
-
-
 class Xlsx(Binary):
     """Class for Excel 2007 (xlsx) files"""
     file_ext = "xlsx"
@@ -1346,9 +1286,6 @@ class Xlsx(Binary):
             return False
 
 
-Binary.register_sniffable_binary_format("xlsx", "xlsx", Xlsx)
-
-
 class ExcelXls(Binary):
     """Class describing an Excel (xls) file"""
     file_ext = "excel.xls"
@@ -1377,9 +1314,6 @@ class ExcelXls(Binary):
             return "Microsoft Excel XLS file (%s)" % (data.nice_size(dataset.get_size()))
 
 
-Binary.register_sniffable_binary_format("excel.xls", "excel.xls", ExcelXls)
-
-
 class Sra(Binary):
     """ Sequence Read Archive (SRA) datatype originally from mdshw5/sra-tools-galaxy"""
     file_ext = 'sra'
@@ -1412,9 +1346,6 @@ class Sra(Binary):
             return 'Binary sra file (%s)' % (nice_size(dataset.get_size()))
 
 
-Binary.register_sniffable_binary_format('sra', 'sra', Sra)
-
-
 class RData(Binary):
     """Generic R Data file datatype implementation"""
     file_ext = 'rdata'
@@ -1433,9 +1364,6 @@ class RData(Binary):
             return False
 
 
-Binary.register_sniffable_binary_format('RData', 'RData', RData)
-
-
 class OxliBinary(Binary):
 
     @staticmethod
@@ -1472,15 +1400,12 @@ class OxliCountGraph(OxliBinary):
     >>> OxliCountGraph().sniff( fname )
     True
     """
+    file_ext = 'oxlicg'
 
     def sniff(self, filename):
         return OxliBinary._sniff(filename, b"01")
 
 
-Binary.register_sniffable_binary_format("oxli.countgraph", "oxlicg",
-                                        OxliCountGraph)
-
-
 class OxliNodeGraph(OxliBinary):
     """
     OxliNodeGraph starts with "OXLI" + one byte version number +
@@ -1500,15 +1425,12 @@ class OxliNodeGraph(OxliBinary):
     >>> OxliNodeGraph().sniff( fname )
     True
     """
+    file_ext = 'oxling'
 
     def sniff(self, filename):
         return OxliBinary._sniff(filename, b"02")
 
 
-Binary.register_sniffable_binary_format("oxli.nodegraph", "oxling",
-                                        OxliNodeGraph)
-
-
 class OxliTagSet(OxliBinary):
     """
     OxliTagSet starts with "OXLI" + one byte version number +
@@ -1529,14 +1451,12 @@ class OxliTagSet(OxliBinary):
     >>> OxliTagSet().sniff( fname )
     True
     """
+    file_ext = 'oxlits'
 
     def sniff(self, filename):
         return OxliBinary._sniff(filename, b"03")
 
 
-Binary.register_sniffable_binary_format("oxli.tagset", "oxlits", OxliTagSet)
-
-
 class OxliStopTags(OxliBinary):
     """
     OxliStopTags starts with "OXLI" + one byte version number +
@@ -1552,15 +1472,12 @@ class OxliStopTags(OxliBinary):
     >>> OxliStopTags().sniff( fname )
     True
     """
+    file_ext = 'oxlist'
 
     def sniff(self, filename):
         return OxliBinary._sniff(filename, b"04")
 
 
-Binary.register_sniffable_binary_format("oxli.stoptags", "oxlist",
-                                        OxliStopTags)
-
-
 class OxliSubset(OxliBinary):
     """
     OxliSubset starts with "OXLI" + one byte version number +
@@ -1581,14 +1498,12 @@ class OxliSubset(OxliBinary):
     >>> OxliSubset().sniff( fname )
     True
     """
+    file_ext = 'oxliss'
 
     def sniff(self, filename):
         return OxliBinary._sniff(filename, b"05")
 
 
-Binary.register_sniffable_binary_format("oxli.subset", "oxliss", OxliSubset)
-
-
 class OxliGraphLabels(OxliBinary):
     """
     OxliGraphLabels starts with "OXLI" + one byte version number +
@@ -1610,15 +1525,12 @@ class OxliGraphLabels(OxliBinary):
     >>> OxliGraphLabels().sniff( fname )
     True
     """
+    file_ext = 'oxligl'
 
     def sniff(self, filename):
         return OxliBinary._sniff(filename, b"06")
 
 
-Binary.register_sniffable_binary_format("oxli.graphlabels", "oxligl",
-                                        OxliGraphLabels)
-
-
 class PostgresqlArchive(CompressedArchive):
     """
     Class describing a Postgresql database packed into a tar archive
@@ -1669,9 +1581,6 @@ class PostgresqlArchive(CompressedArchive):
             return "PostgreSQL Archive (%s)" % (nice_size(dataset.get_size()))
 
 
-Binary.register_sniffable_binary_format("postgresql_archiv", "postgresql", PostgresqlArchive)
-
-
 class Fast5Archive(CompressedArchive):
     """
     Class describing a FAST5 archive
@@ -1772,11 +1681,6 @@ class Fast5ArchiveBz2(Fast5Archive):
         return Fast5Archive.sniff(self, filename)
 
 
-Binary.register_sniffable_binary_format("fast5_archive_bz2", "fast5.tar.bz2", Fast5ArchiveBz2)
-Binary.register_sniffable_binary_format("fast5_archive_gz", "fast5.tar.gz", Fast5ArchiveGz)
-Binary.register_sniffable_binary_format("fast5_archive", "fast5.tar", Fast5Archive)
-
-
 class SearchGuiArchive(CompressedArchive):
     """Class describing a SearchGUI archive """
     MetadataElement(name="searchgui_version", default='1.28.0', param=MetadataParameter, desc="SearchGui Version",
@@ -1828,9 +1732,6 @@ class SearchGuiArchive(CompressedArchive):
             return "SearchGUI Archive, version %s" % (dataset.metadata.searchgui_version or 'unknown')
 
 
-Binary.register_sniffable_binary_format("searchgui_archive", "searchgui_archive", SearchGuiArchive)
-
-
 class NetCDF(Binary):
     """Binary data in netCDF format"""
     file_ext = "netcdf"
@@ -1862,9 +1763,6 @@ class NetCDF(Binary):
             return False
 
 
-Binary.register_sniffable_binary_format("netcdf", "netcdf", NetCDF)
-
-
 class DMND(Binary):
     """
     Class describing an DMND file
@@ -1895,9 +1793,6 @@ class DMND(Binary):
             return False
 
 
-Binary.register_sniffable_binary_format("dmnd", "dmnd", DMND)
-
-
 if __name__ == '__main__':
     import doctest
     doctest.testmod(sys.modules[__name__])

lib/galaxy/datatypes/constructive_solid_geometry.py

@@ -131,9 +131,6 @@ class PlyBinary(Ply, Binary):
         Binary.__init__(self, **kwd)
 
 
-Binary.register_sniffable_binary_format("plybinary", "plybinary", PlyBinary)
-
-
 class Vtk(object):
     r"""
     The Visualization Toolkit provides a number of source and writer objects to
@@ -458,9 +455,6 @@ class VtkBinary(Vtk, Binary):
         Binary.__init__(self, **kwd)
 
 
-Binary.register_sniffable_binary_format("vtkbinary", "vtkbinary", VtkBinary)
-
-
 class STL(data.Data):
     file_ext = "stl"
 

lib/galaxy/datatypes/images.py

@@ -6,7 +6,6 @@ import zipfile
 
 from six.moves.urllib.parse import quote_plus
 
-from galaxy.datatypes.binary import Binary
 from galaxy.datatypes.text import Html as HtmlFromText
 from galaxy.util import nice_size
 from galaxy.util.image_util import check_image_type
@@ -163,9 +162,6 @@ class Pdf(Image):
             return fh.read(4) == b"%PDF"
 
 
-Binary.register_sniffable_binary_format("pdf", "pdf", Pdf)
-
-
 def create_applet_tag_peek(class_name, archive, params):
     text = """
 <object classid="java:%s"

lib/galaxy/datatypes/proteomics.py

@@ -52,9 +52,6 @@ class Wiff(Binary):
         return "\n".join(rval)
 
 
-Binary.register_sniffable_binary_format("wiff", "wiff", Wiff)
-
-
 class PepXmlReport(Tabular):
     """pepxml converted to tabular report"""
     edam_data = "data_2536"
@@ -303,9 +300,6 @@ class ThermoRAW(Binary):
             return "Thermo Finnigan RAW file (%s)" % (nice_size(dataset.get_size()))
 
 
-Binary.register_sniffable_binary_format("thermo.raw", "raw", ThermoRAW)
-
-
 class Msp(Text):
     """ Output of NIST MS Search Program chemdata.nist.gov/mass-spc/ftp/mass-spc/PepLib.pdf """
     file_ext = "msp"

lib/galaxy/datatypes/registry.py

@@ -249,6 +249,7 @@ class Registry(object):
                                         datatype_class.edam_format = edam_format
                                     if edam_data:
                                         datatype_class.edam_data = edam_data
+                                datatype_class.is_subclass = make_subclass
                                 self.datatypes_by_extension[extension] = datatype_class()
                                 if mimetype is None:
                                     # Use default mimetype per datatype specification.
@@ -312,16 +313,14 @@ class Registry(object):
         self.set_default_values()
 
         def append_to_sniff_order():
-            # Just in case any supported data types are not included in the config's sniff_order section.
-            for ext, datatype in self.datatypes_by_extension.items():
-                if hasattr(datatype, 'sniff'):
-                    included = False
-                    for atype in self.sniff_order:
-                        if isinstance(atype, datatype.__class__):
-                            included = True
-                            break
-                    if not included:
-                        self.sniff_order.append(datatype)
+            sniff_order_classes = set(type(_) for _ in self.sniff_order)
+            for datatype in self.datatypes_by_extension.values():
+                # Add a datatype only if it is not already in sniff_order, it
+                # has a sniff() method and was not defined with subclass="true"
+                if type(datatype) not in sniff_order_classes and \
+                        hasattr(datatype, 'sniff') and not datatype.is_subclass:
+                    self.sniff_order.append(datatype)
+
         append_to_sniff_order()
 
     def _load_build_sites(self, root):

lib/galaxy/datatypes/sequence.py

@@ -734,7 +734,7 @@ class FastqCSSanger(Fastq):
     file_ext = "fastqcssanger"
 
 
-class FastqGz (BaseFastq, Binary):
+class FastqGz(BaseFastq, Binary):
     """Class representing a generic compressed FASTQ sequence"""
     edam_format = "format_1930"
     file_ext = "fastq.gz"
@@ -742,6 +742,8 @@ class FastqGz (BaseFastq, Binary):
 
     def sniff(self, filename):
         """Determines whether the file is in gzip-compressed FASTQ format"""
+        if not SNIFF_COMPRESSED_FASTQS:
+            return False
         if not is_gzip(filename):
             return False
         return BaseFastq.sniff(self, filename)
@@ -759,11 +761,6 @@ class FastqSolexaGz(FastqGz):
     file_ext = "fastqsolexa.gz"
 
 
-if SNIFF_COMPRESSED_FASTQS:
-    Binary.register_sniffable_binary_format("fastqsanger.gz", "fastqsanger.gz", FastqSangerGz)
-    Binary.register_sniffable_binary_format("fastq.gz", "fastq.gz", FastqGz)
-
-
 class FastqIlluminaGz(FastqGz):
     """Class representing a compressed FASTQ sequence ( the Illumina 1.3+ variant )"""
     edam_format = "format_1931"
@@ -783,6 +780,8 @@ class FastqBz2 (BaseFastq, Binary):
 
     def sniff(self, filename):
         """Determine whether the file is in bzip2-compressed FASTQ format"""
+        if not SNIFF_COMPRESSED_FASTQS:
+            return False
         if not is_bz2(filename):
             return False
         return BaseFastq.sniff(self, filename)
@@ -794,11 +793,6 @@ class FastqSangerBz2(FastqBz2):
     file_ext = "fastqsanger.bz2"
 
 
-if SNIFF_COMPRESSED_FASTQS:
-    Binary.register_sniffable_binary_format("fastqsanger.bz2", "fastqsanger.bz2", FastqSangerBz2)
-    Binary.register_sniffable_binary_format("fastq.bz2", "fastq.bz2", FastqBz2)
-
-
 class FastqSolexaBz2(FastqBz2):
     """Class representing a compressed FASTQ sequence ( the Solexa variant )"""
     edam_format = "format_1933"

lib/galaxy/datatypes/sniff.py

@@ -17,10 +17,7 @@ from six import text_type
 
 from galaxy import util
 from galaxy.datatypes.binary import Binary
-from galaxy.util import (
-    compression_utils,
-    multi_byte,
-)
+from galaxy.util import compression_utils
 from galaxy.util.checkers import (
     check_binary,
     check_html,
@@ -44,13 +41,12 @@ def get_test_fname(fname):
 
 
 def stream_to_open_named_file(stream, fd, filename, source_encoding=None, source_error='strict', target_encoding=None, target_error='strict'):
-    """Writes a stream to the provided file descriptor, returns the file's name and bool( is_multi_byte ). Closes file descriptor"""
+    """Writes a stream to the provided file descriptor, returns the file name. Closes file descriptor"""
     # signature and behavor is somewhat odd, due to backwards compatibility, but this can/should be done better
     CHUNK_SIZE = 1048576
     data_checked = False
     is_compressed = False
     is_binary = False
-    is_multi_byte = False
     try:
         codecs.lookup(target_encoding)
     except Exception:
@@ -72,11 +68,7 @@ def stream_to_open_named_file(stream, fd, filename, source_encoding=None, source
                 except Exception:
                     pass
             if not is_compressed:
-                # See if we have a multi-byte character file
-                chars = chunk[:100]
-                is_multi_byte = multi_byte.is_multi_byte(chars)
-                if not is_multi_byte:
-                    is_binary = util.is_binary(chunk)
+                is_binary = util.is_binary(chunk)
             data_checked = True
         if not is_compressed and not is_binary:
             if not isinstance(chunk, text_type):
@@ -87,7 +79,7 @@ def stream_to_open_named_file(stream, fd, filename, source_encoding=None, source
             # while binary files should not be encoded at all.
             os.write(fd, chunk)
     os.close(fd)
-    return filename, is_multi_byte
+    return filename
 
 
 def stream_to_file(stream, suffix='', prefix='', dir=None, text=False, **kwd):
@@ -396,9 +388,9 @@ def guess_ext(fname, sniff_order):
     file_ext = None
     for datatype in sniff_order:
         """
-        Some classes may not have a sniff function, which is ok.  In fact, the
-        Tabular and Text classes are 2 examples of classes that should never have
-        a sniff function.  Since these classes are default classes, they contain
+        Some classes may not have a sniff function, which is ok.  In fact,
+        Binary, Data, Tabular and Text are examples of classes that should never
+        have a sniff function. Since these classes are default classes, they contain
         few rules to filter out data of other formats, so they should be called
         from this function after all other datatypes in sniff_order have not been
         successfully discovered.

lib/galaxy/datatypes/triples.py

@@ -186,6 +186,3 @@ class HDT(binary.Binary, Triples):
         else:
             dataset.peek = 'file does not exist'
             dataset.blurb = 'file purged from disk'
-
-
-binary.Binary.register_sniffable_binary_format('HDT', 'HDT', HDT)

lib/galaxy/dependencies/conda-environment.txt

@@ -34,7 +34,6 @@ pycrypto
 Paste
 PasteDeploy
 docutils
-#wchartype
 repoze.lru
 Routes
 WebOb

lib/galaxy/dependencies/pinned-hashed-requirements.txt

@@ -84,7 +84,6 @@ pycrypto==2.6.1 --hash:sha256=44c20785a1057b8d5e190d6a1847f7c62a8180b9b27a3aca7c
 Paste==2.0.2 --hash:sha256=4e7bb2d8cbdc47ab46e4af0cb61d2af91fccc620a08415d9c4da9e499f0455fd
 PasteDeploy==1.5.2 --hash:sha256=3922127d3acc6e274a800978b9293c874b3ef4ac2eb8bf485e2c85b22d1f260a
 docutils==0.12 --hash:sha256=732dfc2d706ea390c264bc69110d3d6c7c3a520628a052ccc5998466f95d5c29
-wchartype==0.1 --hash:sha256=2932471fe3a4e5cac4539c1b49bbc2bbd24ec3f00532c7f27c80b7756fdc177d
 repoze.lru==0.6 --hash:sha256=731cc0b0a184c9fd270a4f29d9635099bb0398823e9a1a39bf4f488d1ead57b3
 Routes==2.4.1 --hash:sha256=cc3a4b9a34bfe6b64b1d10d18b845bb2b6cb3ee1fbf6107295dfafe629bb84aa
 WebOb==1.4.1 --hash:sha256=dc3c45ac0b56a3c65f47f1da6c23760dfe0139e8a84caaf1cfc9276b3e10875a

lib/galaxy/dependencies/pinned-requirements.txt

@@ -24,7 +24,6 @@ boltons==17.1.0
 Paste==2.0.2
 PasteDeploy==1.5.2
 docutils==0.14
-wchartype==0.1
 repoze.lru==0.6
 Routes==2.4.1
 WebOb==1.4.1

lib/galaxy/dependencies/requirements.txt

@@ -21,7 +21,6 @@ boltons
 Paste
 PasteDeploy
 docutils
-wchartype
 repoze.lru
 Routes
 WebOb

lib/galaxy/jobs/__init__.py

@@ -1040,7 +1040,7 @@ class JobWrapper(object, HasResourceParameters):
         destination_params = job.destination_params
         if "__resubmit_delay_seconds" in destination_params:
             delay = float(destination_params["__resubmit_delay_seconds"])
-            if job.seconds_since_update < delay:
+            if job.seconds_since_updated < delay:
                 return False
 
         return True

lib/galaxy/managers/collections.py

@@ -46,6 +46,40 @@ class DatasetCollectionManager(object):
         self.tag_manager = tags.GalaxyTagManager(app.model.context)
         self.ldda_manager = lddas.LDDAManager(app)
 
+    def precreate_dataset_collection_instance(self, trans, parent, name, implicit_inputs, implicit_output_name, structure):
+        # TODO: prebuild all required HIDs and send them in so no need to flush in between.
+        dataset_collection = self.precreate_dataset_collection(structure)
+        instance = self._create_instance_for_collection(
+            trans, parent, name, dataset_collection, implicit_inputs=implicit_inputs, implicit_output_name=implicit_output_name, flush=False
+        )
+        return instance
+
+    def precreate_dataset_collection(self, structure):
+        if structure.is_leaf or not structure.children_known:
+            return model.DatasetCollectionElement.UNINITIALIZED_ELEMENT
+        else:
+            collection_type_description = structure.collection_type_description
+            dataset_collection = model.DatasetCollection(populated=False)
+            dataset_collection.collection_type = collection_type_description.collection_type
+            elements = []
+            for index, (identifier, substructure) in enumerate(structure.children):
+                # TODO: Open question - populate these now or later?
+                if substructure.is_leaf:
+                    element = model.DatasetCollectionElement.UNINITIALIZED_ELEMENT
+                else:
+                    element = self.precreate_dataset_collection(substructure)
+
+                element = model.DatasetCollectionElement(
+                    element=element,
+                    element_identifier=identifier,
+                    element_index=index,
+                )
+                elements.append(element)
+            dataset_collection.elements = elements
+            dataset_collection.element_count = len(elements)
+
+            return dataset_collection
+
     def create(self, trans, parent, name, collection_type, element_identifiers=None,
                elements=None, implicit_collection_info=None, trusted_identifiers=None,
                hide_source_items=False, tags=None):
@@ -68,27 +102,30 @@ class DatasetCollectionManager(object):
             hide_source_items=hide_source_items,
         )
 
+        implicit_inputs = []
+        if implicit_collection_info:
+            implicit_inputs = implicit_collection_info.get('implicit_inputs', [])
+
+        implicit_output_name = None
+        if implicit_collection_info:
+            implicit_output_name = implicit_collection_info["implicit_output_name"]
+
+        return self._create_instance_for_collection(
+            trans, parent, name, dataset_collection, implicit_inputs=implicit_inputs, implicit_output_name=implicit_output_name, tags=tags
+        )
+
+    def _create_instance_for_collection(self, trans, parent, name, dataset_collection, implicit_output_name=None, implicit_inputs=None, tags=None, flush=True):
         if isinstance(parent, model.History):
             dataset_collection_instance = self.model.HistoryDatasetCollectionAssociation(
                 collection=dataset_collection,
                 name=name,
             )
-            if implicit_collection_info:
-                for input_name, input_collection in implicit_collection_info["implicit_inputs"]:
+            if implicit_inputs:
+                for input_name, input_collection in implicit_inputs:
                     dataset_collection_instance.add_implicit_input_collection(input_name, input_collection)
-                for output_dataset in implicit_collection_info.get("outputs"):
-                    if output_dataset not in trans.sa_session:
-                        output_dataset = trans.sa_session.query(type(output_dataset)).get(output_dataset.id)
-                    if isinstance(output_dataset, model.HistoryDatasetAssociation):
-                        output_dataset.hidden_beneath_collection_instance = dataset_collection_instance
-                    elif isinstance(output_dataset, model.HistoryDatasetCollectionAssociation):
-                        dataset_collection_instance.add_implicit_input_collection(input_name, input_collection)
-                    else:
-                        # dataset collection, don't need to do anything...
-                        pass
-                    trans.sa_session.add(output_dataset)
 
-                dataset_collection_instance.implicit_output_name = implicit_collection_info["implicit_output_name"]
+            if implicit_output_name:
+                dataset_collection_instance.implicit_output_name = implicit_output_name
 
             log.debug("Created collection with %d elements" % (len(dataset_collection_instance.collection.elements)))
             # Handle setting hid
@@ -105,37 +142,26 @@ class DatasetCollectionManager(object):
             message = "Internal logic error - create called with unknown parent type %s" % type(parent)
             log.exception(message)
             raise MessageException(message)
-        tags = tags or {}
-        if implicit_collection_info:
-            for _, v in implicit_collection_info.get('implicit_inputs', []):
-                for tag in [t for t in v.tags if t.user_tname == 'name']:
-                    tags[tag.value] = tag
-        for _, tag in tags.items():
-            dataset_collection_instance.tags.append(tag.copy(cls=model.HistoryDatasetCollectionTagAssociation))
 
-        return self.__persist(dataset_collection_instance)
+        tags = self._append_tags(dataset_collection_instance, implicit_inputs, tags)
+        return self.__persist(dataset_collection_instance, flush=flush)
 
     def create_dataset_collection(self, trans, collection_type, element_identifiers=None, elements=None,
                                   hide_source_items=None):
+        # Make sure at least one of these is None.
+        assert element_identifiers is None or elements is None
+
         if element_identifiers is None and elements is None:
             raise RequestParameterInvalidException(ERROR_INVALID_ELEMENTS_SPECIFICATION)
         if not collection_type:
             raise RequestParameterInvalidException(ERROR_NO_COLLECTION_TYPE)
+
         collection_type_description = self.collection_type_descriptions.for_collection_type(collection_type)
+
         # If we have elements, this is an internal request, don't need to load
         # objects from identifiers.
         if elements is None:
-            if collection_type_description.has_subcollections():
-                # Nested collection - recursively create collections and update identifiers.
-                self.__recursively_create_collections(trans, element_identifiers)
-            new_collection = False
-            for element_identifier in element_identifiers:
-                if element_identifier.get("src") == "new_collection" and element_identifier.get('collection_type') == '':
-                    new_collection = True
-                    elements = self.__load_elements(trans, element_identifier['element_identifiers'])
-            if not new_collection:
-                elements = self.__load_elements(trans, element_identifiers)
-
+            elements = self._element_identifiers_to_elements(trans, collection_type_description, element_identifiers)
         # else if elements is set, it better be an ordered dict!
 
         if elements is not self.ELEMENTS_UNINITIALIZED:
@@ -150,6 +176,28 @@ class DatasetCollectionManager(object):
         dataset_collection.collection_type = collection_type
         return dataset_collection
 
+    def _element_identifiers_to_elements(self, trans, collection_type_description, element_identifiers):
+        if collection_type_description.has_subcollections():
+            # Nested collection - recursively create collections and update identifiers.
+            self.__recursively_create_collections(trans, element_identifiers)
+        new_collection = False
+        for element_identifier in element_identifiers:
+            if element_identifier.get("src") == "new_collection" and element_identifier.get('collection_type') == '':
+                new_collection = True
+                elements = self.__load_elements(trans, element_identifier['element_identifiers'])
+        if not new_collection:
+            elements = self.__load_elements(trans, element_identifiers)
+        return elements
+
+    def _append_tags(self, dataset_collection_instance, implicit_inputs=None, tags=None):
+        tags = tags or {}
+        implicit_inputs = implicit_inputs or []
+        for _, v in implicit_inputs:
+            for tag in [t for t in v.tags if t.user_tname == 'name']:
+                tags[tag.value] = tag
+        for _, tag in tags.items():
+            dataset_collection_instance.tags.append(tag.copy(cls=model.HistoryDatasetCollectionTagAssociation))
+
     def set_collection_elements(self, dataset_collection, dataset_instances):
         if dataset_collection.populated:
             raise Exception("Cannot reset elements of an already populated dataset collection.")
@@ -240,10 +288,11 @@ class DatasetCollectionManager(object):
         collections = list(filter(query.direct_match, collections))
         return collections
 
-    def __persist(self, dataset_collection_instance):
+    def __persist(self, dataset_collection_instance, flush=True):
         context = self.model.context
         context.add(dataset_collection_instance)
-        context.flush()
+        if flush:
+            context.flush()
         return dataset_collection_instance
 
     def __recursively_create_collections(self, trans, element_identifiers):

lib/galaxy/managers/collections_util.py

@@ -1,4 +1,5 @@
 import logging
+import math
 
 from galaxy import exceptions, model, web
 from galaxy.util import string_as_bool
@@ -101,8 +102,9 @@ def get_collection_elements(collection, name=""):
     return names, hdas
 
 
-def dictify_dataset_collection_instance(dataset_collection_instance, parent, security, view="element"):
-    dict_value = dataset_collection_instance.to_dict(view=view)
+def dictify_dataset_collection_instance(dataset_collection_instance, parent, security, view="element", fuzzy_count=None):
+    hdca_view = "element" if view in ["element", "element-reference"] else "collection"
+    dict_value = dataset_collection_instance.to_dict(view=hdca_view)
     encoded_id = security.encode_id(dataset_collection_instance.id)
     if isinstance(parent, model.History):
         encoded_history_id = security.encode_id(parent.id)
@@ -112,25 +114,160 @@ def dictify_dataset_collection_instance(dataset_collection_instance, parent, sec
         encoded_folder_id = security.encode_id(parent.id)
         # TODO: Work in progress - this end-point is not right yet...
         dict_value['url'] = web.url_for('library_content', library_id=encoded_library_id, id=encoded_id, folder_id=encoded_folder_id)
-    if view == "element":
+
+    if view in ["element", "element-reference"]:
         collection = dataset_collection_instance.collection
-        dict_value['elements'] = [dictify_element(_) for _ in collection.elements]
-        dict_value['populated'] = collection.populated
+        rank_fuzzy_counts = gen_rank_fuzzy_counts(collection.collection_type, fuzzy_count)
+        elements, rest_fuzzy_counts = get_fuzzy_count_elements(collection, rank_fuzzy_counts)
+        if view == "element":
+            dict_value['populated'] = collection.populated
+            element_func = dictify_element
+        else:
+            element_func = dictify_element_reference
+        dict_value['elements'] = [element_func(_, rank_fuzzy_counts=rest_fuzzy_counts) for _ in elements]
+
     security.encode_all_ids(dict_value, recursive=True)  # TODO: Use Kyle's recursive formulation of this.
     return dict_value
 
 
-def dictify_element(element):
+def dictify_element_reference(element, rank_fuzzy_counts=None):
+    """Load minimal details of elements required to show outline of contents in history panel.
+
+    History panel can use this reference to expand to full details if individual dataset elements
+    are clicked.
+    """
     dictified = element.to_dict(view="element")
-    object_detials = element.element_object.to_dict()
-    if element.child_collection:
-        # Recursively yield elements for each nested collection...
-        child_collection = element.child_collection
-        object_detials["elements"] = [dictify_element(_) for _ in child_collection.elements]
-        object_detials["populated"] = child_collection.populated
+    element_object = element.element_object
+    if element_object is not None:
+        object_detials = dict(
+            id=element_object.id,
+            model_class=element_object.__class__.__name__,
+        )
+        if element.child_collection:
+            object_detials["collection_type"] = element_object.collection_type
+            child_collection = element.child_collection
+            elements, rest_fuzzy_counts = get_fuzzy_count_elements(child_collection, rank_fuzzy_counts)
+            # Recursively yield elements for each nested collection...
+            object_detials["elements"] = [dictify_element_reference(_, rank_fuzzy_counts=rest_fuzzy_counts) for _ in elements]
+            object_detials["element_count"] = child_collection.element_count
+        else:
+            object_detials["state"] = element_object.state
+            object_detials["hda_ldda"] = 'hda'
+            object_detials["history_id"] = element_object.history_id
+
+    else:
+        object_detials = None
 
     dictified["object"] = object_detials
     return dictified
 
 
+def dictify_element(element, rank_fuzzy_counts=None):
+    dictified = element.to_dict(view="element")
+    element_object = element.element_object
+    if element_object is not None:
+        object_detials = element.element_object.to_dict()
+        if element.child_collection:
+            child_collection = element.child_collection
+            elements, rest_fuzzy_counts = get_fuzzy_count_elements(child_collection, rank_fuzzy_counts)
+
+            # Recursively yield elements for each nested collection...
+            child_collection = element.child_collection
+            object_detials["elements"] = [dictify_element(_, rank_fuzzy_counts=rest_fuzzy_counts) for _ in elements]
+            object_detials["populated"] = child_collection.populated
+            object_detials["element_count"] = child_collection.element_count
+    else:
+        object_detials = None
+
+    dictified["object"] = object_detials
+    return dictified
+
+
+def get_fuzzy_count_elements(collection, rank_fuzzy_counts):
+    if rank_fuzzy_counts and rank_fuzzy_counts[0]:
+        rank_fuzzy_count = rank_fuzzy_counts[0]
+        elements = collection.elements[0:rank_fuzzy_count]
+    else:
+        elements = collection.elements
+
+    if rank_fuzzy_counts is not None:
+        rest_fuzzy_counts = rank_fuzzy_counts[1:]
+    else:
+        rest_fuzzy_counts = None
+
+    return elements, rest_fuzzy_counts
+
+
+def gen_rank_fuzzy_counts(collection_type, fuzzy_count=None):
+    """Turn a global estimate on elements to return to per nested level based on collection type.
+
+    This takes an arbitrary constant and generates an arbitrary constant and is quite messy.
+    None of this should be relied on as a stable API - it is more of a general guideline to
+    restrict within broad ranges the amount of objects returned.
+
+    >>> def is_around(x, y):
+    ...     return y - 1 < x and y + 1 > y
+    ...
+    >>> gen_rank_fuzzy_counts("list", None)
+    [None]
+    >>> gen_rank_fuzzy_counts("list", 500)
+    [500]
+    >>> gen_rank_fuzzy_counts("paired", 500)
+    [2]
+    >>> gen_rank_fuzzy_counts("list:paired", None)
+    [None, None]
+    >>> gen_rank_fuzzy_counts("list:list", 101)  # 100 would be edge case at 10 so bump to ensure 11
+    [11, 11]
+    >>> ll, pl = gen_rank_fuzzy_counts("list:paired", 100)
+    >>> pl
+    2
+    >>> is_around(ll, 50)
+    True
+    >>> pl, ll = gen_rank_fuzzy_counts("paired:list", 100)
+    >>> pl
+    2
+    >>> is_around(ll, 50)
+    True
+    >>> gen_rank_fuzzy_counts("list:list:list", 1001)
+    [11, 11, 11]
+    >>> l1l, l2l, l3l, pl = gen_rank_fuzzy_counts("list:list:list:paired", 2000)
+    >>> pl
+    2
+    >>> is_around(10, l1l)
+    True
+    >>> gen_rank_fuzzy_counts("list:list:list", 1)
+    [1, 1, 1]
+    >>> gen_rank_fuzzy_counts("list:list:list", 2)
+    [2, 2, 2]
+    >>> gen_rank_fuzzy_counts("paired:paired", 400)
+    [2, 2]
+    >>> gen_rank_fuzzy_counts("paired:paired", 5)
+    [2, 2]
+    >>> gen_rank_fuzzy_counts("paired:paired", 3)
+    [2, 2]
+    >>> gen_rank_fuzzy_counts("paired:paired", 1)
+    [1, 1]
+    >>> gen_rank_fuzzy_counts("paired:paired", 2)
+    [2, 2]
+    """
+    rank_collection_types = collection_type.split(":")
+    if fuzzy_count is None:
+        return [None for rt in rank_collection_types]
+    else:
+        # This is a list...
+        paired_count = sum([1 if rt == "paired" else 0 for rt in rank_collection_types])
+        list_count = len(rank_collection_types) - paired_count
+        paired_fuzzy_count_mult = 1 if paired_count == 0 else 2 << (paired_count - 1)
+        list_fuzzy_count_mult = math.floor((fuzzy_count * 1.0) / paired_fuzzy_count_mult)
+        list_rank_fuzzy_count = int(math.floor(math.pow(list_fuzzy_count_mult, 1.0 / list_count)) + 1) if list_count > 0 else 1.0
+        pair_rank_fuzzy_count = 2
+        if list_rank_fuzzy_count > fuzzy_count:
+            list_rank_fuzzy_count = fuzzy_count
+        if pair_rank_fuzzy_count > fuzzy_count:
+            pair_rank_fuzzy_count = fuzzy_count
+        rank_fuzzy_counts = [pair_rank_fuzzy_count if rt == "paired" else list_rank_fuzzy_count for rt in rank_collection_types]
+
+        return rank_fuzzy_counts
+
+
 __all__ = ('api_payload_to_create_params', 'dictify_dataset_collection_instance')

lib/galaxy/managers/hdcas.py

@@ -117,12 +117,13 @@ class DCSerializer(base.ModelSerializer):
             'create_time',
             'update_time',
             'collection_type',
-            'populated',
             'populated_state',
             'populated_state_message',
+            'element_count',
         ])
         self.add_view('detailed', [
-            'elements'
+            'populated',
+            'elements',
         ], include_keys_from='summary')
 
     def add_serializers(self):
@@ -130,7 +131,6 @@ class DCSerializer(base.ModelSerializer):
         self.serializers.update({
             'model_class'   : lambda *a, **c: 'DatasetCollection',
             'elements'      : self.serialize_elements,
-            'element_count' : self.serialize_element_count
         })
 
     def serialize_elements(self, item, key, **context):
@@ -140,14 +140,6 @@ class DCSerializer(base.ModelSerializer):
             returned.append(serialized)
         return returned
 
-    def serialize_element_count(self, item, key, **context):
-        """Return the count of elements for this collection."""
-        # TODO: app.model.context -> session
-        # TODO: to the container interface (dataset_collection_contents)
-        return (self.app.model.context.query(model.DatasetCollectionElement)
-            .filter(model.DatasetCollectionElement.dataset_collection_id == item.id)
-            .count())
-
 
 class DCASerializer(base.ModelSerializer):
     """
@@ -163,12 +155,13 @@ class DCASerializer(base.ModelSerializer):
             'id',
             'create_time', 'update_time',
             'collection_type',
-            'populated',
             'populated_state',
             'populated_state_message',
+            'element_count',
         ])
         self.add_view('detailed', [
-            'elements'
+            'populated',
+            'elements',
         ], include_keys_from='summary')
 
     def add_serializers(self):
@@ -184,7 +177,7 @@ class DCASerializer(base.ModelSerializer):
             'populated_state',
             'populated_state_message',
             'elements',
-            'element_count'
+            'element_count',
         ]
         for key in collection_keys:
             self.serializers[key] = self._proxy_to_dataset_collection(key=key)
@@ -221,15 +214,15 @@ class HDCASerializer(
             'history_content_type',
 
             'collection_type',
-            'populated',
             'populated_state',
             'populated_state_message',
+            'element_count',
+
+            'job_source_id',
+            'job_source_type',
 
             'name',
             'type_id',
-            'history_id',
-            'hid',
-            'history_content_type',
             'deleted',
             # 'purged',
             'visible',
@@ -238,6 +231,7 @@ class HDCASerializer(
             'tags',  # TODO: detail view only (maybe)
         ])
         self.add_view('detailed', [
+            'populated',
             'elements'
         ], include_keys_from='summary')
 
@@ -254,6 +248,7 @@ class HDCASerializer(
             'history_id'                : self.serialize_id,
             'history_content_type'      : lambda *a, **c: self.hdca_manager.model_class.content_type,
             'type_id'                   : self.serialize_type_id,
+            'job_source_id'             : self.serialize_id,
 
             'url'   : lambda i, k, **c: self.url_for('history_content_typed',
                                                      history_id=self.app.security.encode_id(i.history_id),

lib/galaxy/managers/jobs.py

@@ -3,10 +3,12 @@ import logging
 
 from boltons.iterutils import remap
 from six import string_types
-from sqlalchemy import and_, false, or_
+from sqlalchemy import and_, false, func, or_
 from sqlalchemy.orm import aliased
+from sqlalchemy.sql import select
 
 from galaxy import model
+from galaxy.exceptions import RequestParameterInvalidException
 from galaxy.managers.collections import DatasetCollectionManager
 from galaxy.managers.hdas import HDAManager
 from galaxy.managers.lddas import LDDAManager
@@ -243,3 +245,75 @@ class JobSearch(object):
                 log.info("Searching jobs finished %s", search_timer)
                 return job
         return None
+
+
+def fetch_job_states(app, sa_session, job_source_ids, job_source_types):
+    decode = app.security.decode_id
+    assert len(job_source_ids) == len(job_source_types)
+    job_ids = set()
+    implicit_collection_job_ids = set()
+
+    for job_source_id, job_source_type in zip(job_source_ids, job_source_types):
+        if job_source_type == "Job":
+            job_ids.add(job_source_id)
+        elif job_source_type == "ImplicitCollectionJobs":
+            implicit_collection_job_ids.add(job_source_id)
+        else:
+            raise RequestParameterInvalidException("Invalid job source type %s found." % job_source_type)
+
+    # TODO: use above sets and optimize queries on second pass.
+    rval = []
+    for job_source_id, job_source_type in zip(job_source_ids, job_source_types):
+        if job_source_type == "Job":
+            rval.append(summarize_jobs_to_dict(sa_session, sa_session.query(model.Job).get(decode(job_source_id))))
+        else:
+            rval.append(summarize_jobs_to_dict(sa_session, sa_session.query(model.ImplicitCollectionJobs).get(decode(job_source_id))))
+
+    return rval
+
+
+def summarize_jobs_to_dict(sa_session, jobs_source):
+    """Proudce a summary of jobs for job summary endpoints.
+
+    :type   jobs_source: a Job or ImplicitCollectionJobs or None
+    :param  jobs_source: the object to summarize
+
+    :rtype:     dict
+    :returns:   dictionary containing job summary information
+    """
+    rval = None
+    if jobs_source is None:
+        pass
+    elif isinstance(jobs_source, model.Job):
+        rval = {
+            "populated_state": "ok",
+            "states": {jobs_source.state: 1},
+            "model": "Job",
+            "id": jobs_source.id,
+        }
+    else:
+        populated_state = jobs_source.populated_state
+        rval = {
+            "id": jobs_source.id,
+            "populated_state": populated_state,
+            "model": "ImplicitCollectionJobs",
+        }
+        if populated_state == "ok":
+            # produce state summary...
+            states = {}
+            join = model.ImplicitCollectionJobs.table.join(
+                model.ImplicitCollectionJobsJobAssociation.table.join(model.Job)
+            )
+            statement = select(
+                [model.Job.state, func.count("*")]
+            ).select_from(
+                join
+            ).where(
+                model.ImplicitCollectionJobs.id == jobs_source.id
+            ).group_by(
+                model.Job.state
+            )
+            for row in sa_session.execute(statement):
+                states[row[0]] = row[1]
+            rval["states"] = states
+    return rval

lib/galaxy/model/__init__.py

@@ -4,7 +4,6 @@ Galaxy data model classes
 Naming: try to use class names that have a distinct plural form so that
 the relationship cardinalities are obvious (e.g. prefer Dataset to Data)
 """
-import codecs
 import errno
 import json
 import logging
@@ -36,7 +35,6 @@ from galaxy.util import (directory_hash_id, Params, ready_name_for_url,
 from galaxy.util.bunch import Bunch
 from galaxy.util.dictifiable import Dictifiable
 from galaxy.util.hash_util import new_secure_hash
-from galaxy.util.multi_byte import is_multi_byte
 from galaxy.util.sanitize_html import sanitize_html
 from galaxy.web.form_builder import (AddressField, CheckboxField, HistoryField,
                                      PasswordField, SelectField, TextArea, TextField, WorkflowField,
@@ -116,6 +114,19 @@ class HasName:
         return name
 
 
+class UsesCreateAndUpdateTime:
+
+    @property
+    def seconds_since_updated(self):
+        update_time = self.update_time or galaxy.model.orm.now.now()  # In case not yet flushed
+        return (galaxy.model.orm.now.now() - update_time).total_seconds()
+
+    @property
+    def seconds_since_created(self):
+        create_time = self.create_time or galaxy.model.orm.now.now()  # In case not yet flushed
+        return (galaxy.model.orm.now.now() - create_time).total_seconds()
+
+
 class JobLike:
 
     def _init_metrics(self):
@@ -423,7 +434,7 @@ class TaskMetricNumeric(BaseJobMetric):
     pass
 
 
-class Job(object, JobLike, Dictifiable):
+class Job(object, JobLike, UsesCreateAndUpdateTime, Dictifiable):
     dict_collection_visible_keys = ['id', 'state', 'exit_code', 'update_time', 'create_time']
     dict_element_visible_keys = ['id', 'state', 'exit_code', 'update_time', 'create_time']
 
@@ -804,10 +815,6 @@ class Job(object, JobLike, Dictifiable):
             config_value = default
         return config_value
 
-    @property
-    def seconds_since_update(self):
-        return (galaxy.model.orm.now.now() - self.update_time).total_seconds()
-
 
 class Task(object, JobLike):
     """
@@ -1043,6 +1050,33 @@ class ImplicitlyCreatedDatasetCollectionInput(object):
         self.input_dataset_collection = input_dataset_collection
 
 
+class ImplicitCollectionJobs(object):
+
+    populated_states = Bunch(
+        NEW='new',  # New implicit jobs object, unpopulated job associations
+        OK='ok',  # Job associations are set and fixed.
+        FAILED='failed',  # There were issues populating job associations, object is in error.
+    )
+
+    def __init__(
+        self,
+        id=None,
+        populated_state=None,
+    ):
+        self.id = id
+        self.populated_state = populated_state or ImplicitCollectionJobs.populated_states.NEW
+
+    @property
+    def job_list(self):
+        return [icjja.job for icjja in self.jobs]
+
+
+class ImplicitCollectionJobsJobAssociation(object):
+
+    def __init__(self):
+        pass
+
+
 class PostJobAction(object):
     def __init__(self, action_type, workflow_step, output_name=None, action_arguments=None):
         self.action_type = action_type
@@ -1857,15 +1891,7 @@ class Dataset(StorableObject):
     def mark_deleted(self):
         self.deleted = True
 
-    def is_multi_byte(self):
-        if not self.has_data():
-            return False
-        try:
-            return is_multi_byte(codecs.open(self.file_name, 'r', 'utf-8').read(100))
-        except UnicodeDecodeError:
-            return False
     # FIXME: sqlalchemy will replace this
-
     def _delete(self):
         """Remove the file that corresponds to this data"""
         self.object_store.delete(self)
@@ -2066,10 +2092,6 @@ class DatasetInstance(object):
             # extension is None
             return 'data'
 
-    def is_multi_byte(self):
-        """Data consists of multi-byte characters"""
-        return self.dataset.is_multi_byte()
-
     def set_peek(self):
         return self.datatype.set_peek(self)
 
@@ -3207,6 +3229,15 @@ class DatasetCollection(object, Dictifiable, UsesAnnotations):
         self.populated_state = DatasetCollection.populated_states.FAILED
         self.populated_state_message = message
 
+    def finalize(self):
+        # All jobs have written out their elements - everything should be populated
+        # but might not be - check that second case! (TODO)
+        self.mark_as_populated()
+        if self.has_subcollections:
+            # THIS IS WRONG - SHOULD ONLY BE TO THE DEPTH OF THE MAP OVER.
+            for element in self.elements:
+                element.child_collection.finalize()
+
     @property
     def dataset_instances(self):
         instances = []
@@ -3307,6 +3338,7 @@ class DatasetCollectionInstance(object, HasName):
             populated=self.populated,
             populated_state=self.collection.populated_state,
             populated_state_message=self.collection.populated_state_message,
+            element_count=self.collection.element_count,
             type="collection",  # contents type (distinguished from file or folder (in case of library))
         )
 
@@ -3382,6 +3414,19 @@ class HistoryDatasetCollectionAssociation(DatasetCollectionInstance,
         return ((type_coerce(cls.content_type, types.Unicode) + u'-' +
                  type_coerce(cls.id, types.Unicode)).label('type_id'))
 
+    @property
+    def job_source_type(self):
+        if self.implicit_collection_jobs_id:
+            return "ImplicitCollectionJobs"
+        elif self.job_id:
+            return "Job"
+        else:
+            return None
+
+    @property
+    def job_source_id(self):
+        return self.implicit_collection_jobs_id or self.job_id
+
     def to_hda_representative(self, multiple=False):
         rval = []
         for dataset in self.collection.dataset_elements:
@@ -3399,6 +3444,8 @@ class HistoryDatasetCollectionAssociation(DatasetCollectionInstance,
             history_content_type=self.history_content_type,
             visible=self.visible,
             deleted=self.deleted,
+            job_source_id=self.job_source_id,
+            job_source_type=self.job_source_type,
             **self._base_to_dict(view=view)
         )
 
@@ -3430,6 +3477,11 @@ class HistoryDatasetCollectionAssociation(DatasetCollectionInstance,
             name=self.name,
             copied_from_history_dataset_collection_association=self,
         )
+        if self.implicit_collection_jobs_id:
+            hdca.implicit_collection_jobs_id = self.implicit_collection_jobs_id
+        elif self.job_id:
+            hdca.job_id = self.job_id
+
         collection_copy = self.collection.copy(
             destination=hdca,
             element_destination=element_destination,
@@ -3474,6 +3526,8 @@ class DatasetCollectionElement(object, Dictifiable):
     dict_collection_visible_keys = ['id', 'element_type', 'element_index', 'element_identifier']
     dict_element_visible_keys = ['id', 'element_type', 'element_index', 'element_identifier']
 
+    UNINITIALIZED_ELEMENT = object()
+
     def __init__(
         self,
         id=None,
@@ -3488,7 +3542,7 @@ class DatasetCollectionElement(object, Dictifiable):
             self.ldda = element
         elif isinstance(element, DatasetCollection):
             self.child_collection = element
-        else:
+        elif element != self.UNINITIALIZED_ELEMENT:
             raise AttributeError('Unknown element type provided: %s' % type(element))
 
         self.id = id
@@ -3506,7 +3560,7 @@ class DatasetCollectionElement(object, Dictifiable):
             # TOOD: Rename element_type to element_type.
             return "dataset_collection"
         else:
-            raise Exception("Unknown element instance type")
+            return None
 
     @property
     def is_collection(self):
@@ -3521,7 +3575,7 @@ class DatasetCollectionElement(object, Dictifiable):
         elif self.child_collection:
             return self.child_collection
         else:
-            raise Exception("Unknown element instance type")
+            return None
 
     @property
     def dataset_instance(self):
@@ -3951,7 +4005,7 @@ class StoredWorkflowMenuEntry(object):
         self.order_index = None
 
 
-class WorkflowInvocation(object, Dictifiable):
+class WorkflowInvocation(object, UsesCreateAndUpdateTime, Dictifiable):
     dict_collection_visible_keys = ['id', 'update_time', 'workflow_id', 'history_id', 'uuid', 'state']
     dict_element_visible_keys = ['id', 'update_time', 'workflow_id', 'history_id', 'uuid', 'state']
     states = Bunch(
@@ -4026,17 +4080,16 @@ class WorkflowInvocation(object, Dictifiable):
         step_invocations = {}
         for invocation_step in self.steps:
             step_id = invocation_step.workflow_step_id
-            if step_id not in step_invocations:
-                step_invocations[step_id] = []
-            step_invocations[step_id].append(invocation_step)
+            assert step_id not in step_invocations
+            step_invocations[step_id] = invocation_step
         return step_invocations
 
-    def step_invocations_for_step_id(self, step_id):
-        step_invocations = []
+    def step_invocation_for_step_id(self, step_id):
+        target_invocation_step = None
         for invocation_step in self.steps:
             if step_id == invocation_step.workflow_step_id:
-                step_invocations.append(invocation_step)
-        return step_invocations
+                target_invocation_step = invocation_step
+        return target_invocation_step
 
     @staticmethod
     def poll_active_workflow_ids(
@@ -4062,6 +4115,24 @@ class WorkflowInvocation(object, Dictifiable):
         # is relatively intutitive.
         return [wid for wid in query.all()]
 
+    def add_output(self, workflow_output, step, output_object):
+        if output_object.history_content_type == "dataset":
+            output_assoc = WorkflowInvocationOutputDatasetAssociation()
+            output_assoc.workflow_invocation = self
+            output_assoc.workflow_output = workflow_output
+            output_assoc.workflow_step = step
+            output_assoc.dataset = output_object
+            self.output_datasets.append(output_assoc)
+        elif output_object.history_content_type == "dataset_collection":
+            output_assoc = WorkflowInvocationOutputDatasetCollectionAssociation()
+            output_assoc.workflow_invocation = self
+            output_assoc.workflow_output = workflow_output
+            output_assoc.workflow_step = step
+            output_assoc.dataset_collection = output_object
+            self.output_dataset_collections.append(output_assoc)
+        else:
+            raise Exception("Uknown output type encountered")
+
     def to_dict(self, view='collection', value_mapper=None, step_details=False):
         rval = super(WorkflowInvocation, self).to_dict(view=view, value_mapper=value_mapper)
         if view == 'element':
@@ -4077,17 +4148,43 @@ class WorkflowInvocation(object, Dictifiable):
             inputs = {}
             for step in self.steps:
                 if step.workflow_step.type == 'tool':
-                    for step_input in step.workflow_step.input_connections:
-                        output_step_type = step_input.output_step.type
-                        if output_step_type in ['data_input', 'data_collection_input']:
-                            src = "hda" if output_step_type == 'data_input' else 'hdca'
-                            for job_input in step.job.input_datasets:
-                                if job_input.name == step_input.input_name:
-                                    inputs[str(step_input.output_step.order_index)] = {
-                                        "id": job_input.dataset_id, "src": src,
-                                        "uuid" : str(job_input.dataset.dataset.uuid) if job_input.dataset.dataset.uuid is not None else None
-                                    }
+                    for job in step.jobs:
+                        for step_input in step.workflow_step.input_connections:
+                            output_step_type = step_input.output_step.type
+                            if output_step_type in ['data_input', 'data_collection_input']:
+                                src = "hda" if output_step_type == 'data_input' else 'hdca'
+                                for job_input in job.input_datasets:
+                                    if job_input.name == step_input.input_name:
+                                        inputs[str(step_input.output_step.order_index)] = {
+                                            "id": job_input.dataset_id, "src": src,
+                                            "uuid" : str(job_input.dataset.dataset.uuid) if job_input.dataset.dataset.uuid is not None else None
+                                        }
             rval['inputs'] = inputs
+
+            outputs = {}
+            for output_assoc in self.output_datasets:
+                label = output_assoc.workflow_output.label
+                if not label:
+                    continue
+
+                outputs[label] = {
+                    'src': 'hda',
+                    'id': output_assoc.dataset_id,
+                }
+
+            output_collections = {}
+            for output_assoc in self.output_dataset_collections:
+                label = output_assoc.workflow_output.label
+                if not label:
+                    continue
+
+                output_collections[label] = {
+                    'src': 'hdca',
+                    'id': output_assoc.dataset_collection_id,
+                }
+
+            rval['outputs'] = outputs
+            rval['output_collections'] = output_collections
         return rval
 
     def update(self):
@@ -4120,11 +4217,6 @@ class WorkflowInvocation(object, Dictifiable):
                 return True
         return False
 
-    @property
-    def seconds_since_created(self):
-        create_time = self.create_time or galaxy.model.orm.now.now()  # In case not flushed yet
-        return (galaxy.model.orm.now.now() - create_time).total_seconds()
-
 
 class WorkflowInvocationToSubworkflowInvocationAssociation(object, Dictifiable):
     dict_collection_visible_keys = ['id', 'workflow_step_id', 'workflow_invocation_id', 'subworkflow_invocation_id']
@@ -4132,36 +4224,83 @@ class WorkflowInvocationToSubworkflowInvocationAssociation(object, Dictifiable):
 
 
 class WorkflowInvocationStep(object, Dictifiable):
-    dict_collection_visible_keys = ['id', 'update_time', 'job_id', 'workflow_step_id', 'action']
-    dict_element_visible_keys = ['id', 'update_time', 'job_id', 'workflow_step_id', 'action']
+    dict_collection_visible_keys = ['id', 'update_time', 'job_id', 'workflow_step_id', 'state', 'action']
+    dict_element_visible_keys = ['id', 'update_time', 'job_id', 'workflow_step_id', 'state', 'action']
+    states = Bunch(
+        NEW='new',  # Brand new workflow invocation step
+        READY='ready',  # Workflow invocation step ready for another iteration of scheduling.
+        SCHEDULED='scheduled',  # Workflow invocation step has been scheduled.
+        # CANCELLED='cancelled',  TODO: implement and expose
+        # FAILED='failed',  TODO: implement and expose
+    )
 
     def update(self):
         self.workflow_invocation.update()
 
+    @property
+    def is_new(self):
+        return self.state == self.states.NEW
+
+    def add_output(self, output_name, output_object):
+        if output_object.history_content_type == "dataset":
+            output_assoc = WorkflowInvocationStepOutputDatasetAssociation()
+            output_assoc.workflow_invocation_step = self
+            output_assoc.dataset = output_object
+            output_assoc.output_name = output_name
+            self.output_datasets.append(output_assoc)
+        elif output_object.history_content_type == "dataset_collection":
+            output_assoc = WorkflowInvocationStepOutputDatasetCollectionAssociation()
+            output_assoc.workflow_invocation_step = self
+            output_assoc.dataset_collection = output_object
+            output_assoc.output_name = output_name
+            self.output_dataset_collections.append(output_assoc)
+        else:
+            raise Exception("Uknown output type encountered")
+
+    @property
+    def jobs(self):
+        if self.job:
+            return [self.job]
+        elif self.implicit_collection_jobs:
+            return self.implicit_collection_jobs.job_list
+        else:
+            return []
+
     def to_dict(self, view='collection', value_mapper=None):
         rval = super(WorkflowInvocationStep, self).to_dict(view=view, value_mapper=value_mapper)
         rval['order_index'] = self.workflow_step.order_index
         rval['workflow_step_label'] = self.workflow_step.label
         rval['workflow_step_uuid'] = str(self.workflow_step.uuid)
-        rval['state'] = self.job.state if self.job is not None else None
-        if self.job is not None and view == 'element':
-            output_dict = {}
-            for i in self.job.output_datasets:
-                if i.dataset is not None:
-                    output_dict[i.name] = {
-                        "id" : i.dataset.id, "src" : "hda",
-                        "uuid" : str(i.dataset.dataset.uuid) if i.dataset.dataset.uuid is not None else None
-                    }
-            for i in self.job.output_library_datasets:
-                if i.dataset is not None:
-                    output_dict[i.name] = {
-                        "id" : i.dataset.id, "src" : "ldda",
-                        "uuid" : str(i.dataset.dataset.uuid) if i.dataset.dataset.uuid is not None else None
-                    }
-            rval['outputs'] = output_dict
+        # Following no longer makes sense...
+        # rval['state'] = self.job.state if self.job is not None else None
+        if view == 'element':
+            outputs = {}
+            for output_assoc in self.output_datasets:
+                name = output_assoc.output_name
+                outputs[name] = {
+                    'src': 'hda',
+                    'id': output_assoc.dataset.id,
+                    'uuid': str(output_assoc.dataset.dataset.uuid) if output_assoc.dataset.dataset.uuid is not None else None
+                }
+
+            output_collections = {}
+            for output_assoc in self.output_dataset_collections:
+                name = output_assoc.output_name
+                output_collections[name] = {
+                    'src': 'hdca',
+                    'id': output_assoc.dataset_collection.id,
+                }
+
+            rval['outputs'] = outputs
+            rval['output_collections'] = output_collections
         return rval
 
 
+class WorkflowInvocationStepJobAssociation(object, Dictifiable):
+    dict_collection_visible_keys = ('id', 'job_id', 'workflow_invocation_step_id')
+    dict_element_visible_keys = ('id', 'job_id', 'workflow_invocation_step_id')
+
+
 class WorkflowRequest(object, Dictifiable):
     dict_collection_visible_keys = ['id', 'name', 'type', 'state', 'history_id', 'workflow_id']
     dict_element_visible_keys = ['id', 'name', 'type', 'state', 'history_id', 'workflow_id']
@@ -4216,6 +4355,26 @@ class WorkflowRequestInputStepParmeter(object, Dictifiable):
     dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'parameter_value']
 
 
+class WorkflowInvocationOutputDatasetAssociation(object, Dictifiable):
+    """Represents links to output datasets for the workflow."""
+    dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'dataset_id', 'name']
+
+
+class WorkflowInvocationOutputDatasetCollectionAssociation(object, Dictifiable):
+    """Represents links to output dataset collections for the workflow."""
+    dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'dataset_collection_id', 'name']
+
+
+class WorkflowInvocationStepOutputDatasetAssociation(object, Dictifiable):
+    """Represents links to output datasets for the workflow."""
+    dict_collection_visible_keys = ['id', 'workflow_invocation_step_id', 'dataset_id', 'output_name']
+
+
+class WorkflowInvocationStepOutputDatasetCollectionAssociation(object, Dictifiable):
+    """Represents links to output dataset collections for the workflow."""
+    dict_collection_visible_keys = ['id', 'workflow_invocation_step_id', 'dataset_collection_id', 'output_name']
+
+
 class MetadataFile(StorableObject):
 
     def __init__(self, dataset=None, name=None):

lib/galaxy/model/mapping.py

@@ -557,6 +557,20 @@ model.ImplicitlyCreatedDatasetCollectionInput.table = Table(
         ForeignKey("history_dataset_collection_association.id"), index=True),
     Column("name", Unicode(255)))
 
+model.ImplicitCollectionJobs.table = Table(
+    "implicit_collection_jobs", metadata,
+    Column("id", Integer, primary_key=True),
+    Column("populated_state", TrimmedString(64), default='new', nullable=False),
+)
+
+model.ImplicitCollectionJobsJobAssociation.table = Table(
+    "implicit_collection_jobs_job_association", metadata,
+    Column("id", Integer, primary_key=True),
+    Column("implicit_collection_jobs_id", Integer, ForeignKey("implicit_collection_jobs.id"), index=True),
+    Column("job_id", Integer, ForeignKey("job.id"), index=True),  # Consider making this nullable...
+    Column("order_index", Integer, nullable=False),
+)
+
 model.JobExternalOutputMetadata.table = Table(
     "job_external_output_metadata", metadata,
     Column("id", Integer, primary_key=True),
@@ -702,6 +716,7 @@ model.DatasetCollection.table = Table(
     Column("collection_type", Unicode(255), nullable=False),
     Column("populated_state", TrimmedString(64), default='ok', nullable=False),
     Column("populated_state_message", TEXT),
+    Column("element_count", Integer, nullable=True),
     Column("create_time", DateTime, default=now),
     Column("update_time", DateTime, default=now, onupdate=now))
 
@@ -716,7 +731,10 @@ model.HistoryDatasetCollectionAssociation.table = Table(
     Column("deleted", Boolean, default=False),
     Column("copied_from_history_dataset_collection_association_id", Integer,
         ForeignKey("history_dataset_collection_association.id"), nullable=True),
-    Column("implicit_output_name", Unicode(255), nullable=True))
+    Column("implicit_output_name", Unicode(255), nullable=True),
+    Column("job_id", ForeignKey("job.id"), index=True, nullable=True),
+    Column("implicit_collection_jobs_id", ForeignKey("implicit_collection_jobs.id"), index=True, nullable=True),
+)
 
 model.LibraryDatasetCollectionAssociation.table = Table(
     "library_dataset_collection_association", metadata,
@@ -901,9 +919,46 @@ model.WorkflowInvocationStep.table = Table(
     Column("update_time", DateTime, default=now, onupdate=now),
     Column("workflow_invocation_id", Integer, ForeignKey("workflow_invocation.id"), index=True, nullable=False),
     Column("workflow_step_id", Integer, ForeignKey("workflow_step.id"), index=True, nullable=False),
+    Column("state", TrimmedString(64), index=True),
     Column("job_id", Integer, ForeignKey("job.id"), index=True, nullable=True),
+    Column("implicit_collection_jobs_id", Integer, ForeignKey("implicit_collection_jobs.id"), index=True, nullable=True),
     Column("action", JSONType, nullable=True))
 
+model.WorkflowInvocationOutputDatasetAssociation.table = Table(
+    "workflow_invocation_output_dataset_association", metadata,
+    Column("id", Integer, primary_key=True),
+    Column("workflow_invocation_id", Integer, ForeignKey("workflow_invocation.id"), index=True),
+    Column("workflow_step_id", Integer, ForeignKey("workflow_step.id"), index=True),
+    Column("dataset_id", Integer, ForeignKey("history_dataset_association.id"), index=True),
+    Column("workflow_output_id", Integer, ForeignKey("workflow_output.id"), index=True),
+)
+
+model.WorkflowInvocationOutputDatasetCollectionAssociation.table = Table(
+    "workflow_invocation_output_dataset_collection_association", metadata,
+    Column("id", Integer, primary_key=True),
+    Column("workflow_invocation_id", Integer, ForeignKey("workflow_invocation.id"), index=True),
+    Column("workflow_step_id", Integer, ForeignKey("workflow_step.id"), index=True),
+    Column("dataset_collection_id", Integer, ForeignKey("history_dataset_collection_association.id"), index=True),
+    Column("workflow_output_id", Integer, ForeignKey("workflow_output.id"), index=True),
+)
+
+model.WorkflowInvocationStepOutputDatasetAssociation.table = Table(
+    "workflow_invocation_step_output_dataset_association", metadata,
+    Column("id", Integer, primary_key=True),
+    Column("workflow_invocation_step_id", Integer, ForeignKey("workflow_invocation_step.id"), index=True),
+    Column("dataset_id", Integer, ForeignKey("history_dataset_association.id"), index=True),
+    Column("output_name", String(255), nullable=True),
+)
+
+model.WorkflowInvocationStepOutputDatasetCollectionAssociation.table = Table(
+    "workflow_invocation_step_output_dataset_collection_association", metadata,
+    Column("id", Integer, primary_key=True),
+    Column("workflow_invocation_step_id", Integer, ForeignKey("workflow_invocation_step.id"), index=True),
+    Column("workflow_step_id", Integer, ForeignKey("workflow_step.id"), index=True),
+    Column("dataset_collection_id", Integer, ForeignKey("history_dataset_collection_association.id"), index=True),
+    Column("output_name", String(255), nullable=True),
+)
+
 model.WorkflowInvocationToSubworkflowInvocationAssociation.table = Table(
     "workflow_invocation_to_subworkflow_invocation_association", metadata,
     Column("id", Integer, primary_key=True),
@@ -1837,6 +1892,31 @@ simple_mapping(model.ImplicitlyCreatedDatasetCollectionInput,
     ),
 )
 
+simple_mapping(model.ImplicitCollectionJobs)
+
+# simple_mapping(
+#     model.ImplicitCollectionJobsHistoryDatasetCollectionAssociation,
+#     history_dataset_collection_associations=relation(
+#         model.HistoryDatasetCollectionAssociation,
+#         backref=backref("implicit_collection_jobs_association", uselist=False),
+#         uselist=True,
+#     ),
+# )
+
+simple_mapping(
+    model.ImplicitCollectionJobsJobAssociation,
+    implicit_collection_jobs=relation(
+        model.ImplicitCollectionJobs,
+        backref=backref("jobs", uselist=True),
+        uselist=False,
+    ),
+    job=relation(
+        model.Job,
+        backref=backref("implicit_collection_jobs_association", uselist=False),
+        uselist=False,
+    ),
+)
+
 mapper(model.JobParameter, model.JobParameter.table)
 
 mapper(model.JobExternalOutputMetadata, model.JobExternalOutputMetadata.table, properties=dict(
@@ -1928,6 +2008,16 @@ simple_mapping(model.HistoryDatasetCollectionAssociation,
                       model.ImplicitlyCreatedDatasetCollectionInput.table.c.dataset_collection_id)),
         backref="dataset_collection",
     ),
+    implicit_collection_jobs=relation(
+        model.ImplicitCollectionJobs,
+        backref=backref("history_dataset_collection_associations", uselist=True),
+        uselist=False,
+    ),
+    job=relation(
+        model.Job,
+        backref=backref("history_dataset_collection_associations", uselist=True),
+        uselist=False,
+    ),
     tags=relation(model.HistoryDatasetCollectionTagAssociation,
         order_by=model.HistoryDatasetCollectionTagAssociation.table.c.id,
         backref='dataset_collections'),
@@ -2081,7 +2171,7 @@ mapper(model.WorkflowInvocation, model.WorkflowInvocation.table, properties=dict
         uselist=True,
     ),
     steps=relation(model.WorkflowInvocationStep,
-        backref='workflow_invocation'),
+        backref="workflow_invocation"),
     workflow=relation(model.Workflow)
 ))
 
@@ -2094,12 +2184,11 @@ mapper(model.WorkflowInvocationToSubworkflowInvocationAssociation, model.Workflo
     workflow_step=relation(model.WorkflowStep),
 ))
 
-mapper(model.WorkflowInvocationStep, model.WorkflowInvocationStep.table, properties=dict(
+simple_mapping(model.WorkflowInvocationStep,
     workflow_step=relation(model.WorkflowStep),
-    job=relation(model.Job,
-        backref=backref('workflow_invocation_step',
-            uselist=False))
-))
+    job=relation(model.Job, backref=backref('workflow_invocation_step', uselist=False), uselist=False),
+    implicit_collection_jobs=relation(model.ImplicitCollectionJobs, backref=backref('workflow_invocation_step', uselist=False), uselist=False),)
+
 
 simple_mapping(model.WorkflowRequestInputParameter,
     workflow_invocation=relation(model.WorkflowInvocation))
@@ -2129,6 +2218,39 @@ mapper(model.MetadataFile, model.MetadataFile.table, properties=dict(
     library_dataset=relation(model.LibraryDatasetDatasetAssociation)
 ))
 
+
+simple_mapping(
+    model.WorkflowInvocationOutputDatasetAssociation,
+    workflow_invocation=relation(model.WorkflowInvocation, backref="output_datasets"),
+    workflow_step=relation(model.WorkflowStep),
+    dataset=relation(model.HistoryDatasetAssociation),
+    workflow_output=relation(model.WorkflowOutput),
+)
+
+
+simple_mapping(
+    model.WorkflowInvocationOutputDatasetCollectionAssociation,
+    workflow_invocation=relation(model.WorkflowInvocation, backref="output_dataset_collections"),
+    workflow_step=relation(model.WorkflowStep),
+    dataset_collection=relation(model.HistoryDatasetCollectionAssociation),
+    workflow_output=relation(model.WorkflowOutput),
+)
+
+
+simple_mapping(
+    model.WorkflowInvocationStepOutputDatasetAssociation,
+    workflow_invocation_step=relation(model.WorkflowInvocationStep, backref="output_datasets"),
+    dataset=relation(model.HistoryDatasetAssociation),
+)
+
+
+simple_mapping(
+    model.WorkflowInvocationStepOutputDatasetCollectionAssociation,
+    workflow_invocation_step=relation(model.WorkflowInvocationStep, backref="output_dataset_collections"),
+    dataset_collection=relation(model.HistoryDatasetCollectionAssociation),
+)
+
+
 mapper(model.PageRevision, model.PageRevision.table)
 
 mapper(model.Page, model.Page.table, properties=dict(

lib/galaxy/model/migrate/versions/0136_collection_and_workflow_state.py

@@ -0,0 +1,172 @@
+"""
+Migration script for collections and workflows connections.
+"""
+from __future__ import print_function
+
+import datetime
+import logging
+
+from collections import OrderedDict
+
+from sqlalchemy import Column, ForeignKey, Integer, MetaData, String, Table
+
+from galaxy.model.custom_types import TrimmedString
+
+
+now = datetime.datetime.utcnow
+
+log = logging.getLogger(__name__)
+metadata = MetaData()
+
+
+workflow_invocation_output_dataset_association_table = Table(
+    "workflow_invocation_output_dataset_association", metadata,
+    Column("id", Integer, primary_key=True),
+    Column("workflow_invocation_id", Integer, ForeignKey("workflow_invocation.id"), index=True),
+    Column("workflow_step_id", Integer, ForeignKey("workflow_step.id")),
+    Column("dataset_id", Integer, ForeignKey("history_dataset_association.id"), index=True),
+    Column("workflow_output_id", Integer, ForeignKey("workflow_output.id")),
+)
+
+workflow_invocation_output_dataset_collection_association_table = Table(
+    "workflow_invocation_output_dataset_collection_association", metadata,
+    Column("id", Integer, primary_key=True),
+    Column("workflow_invocation_id", Integer, ForeignKey("workflow_invocation.id"), index=True),
+    Column("workflow_step_id", Integer, ForeignKey("workflow_step.id")),
+    Column("dataset_collection_id", Integer, ForeignKey("history_dataset_collection_association.id"), index=True),
+    Column("workflow_output_id", Integer, ForeignKey("workflow_output.id")),
+)
+
+workflow_invocation_step_output_dataset_association_table = Table(
+    "workflow_invocation_step_output_dataset_association", metadata,
+    Column("id", Integer, primary_key=True),
+    Column("workflow_invocation_step_id", Integer, ForeignKey("workflow_invocation_step.id"), index=True),
+    Column("dataset_id", Integer, ForeignKey("history_dataset_association.id"), index=True),
+    Column("output_name", String(255), nullable=True),
+)
+
+workflow_invocation_step_output_dataset_collection_association_table = Table(
+    "workflow_invocation_step_output_dataset_collection_association", metadata,
+    Column("id", Integer, primary_key=True),
+    Column("workflow_invocation_step_id", Integer, ForeignKey("workflow_invocation_step.id"), index=True),
+    Column("workflow_step_id", Integer, ForeignKey("workflow_step.id")),
+    Column("dataset_collection_id", Integer, ForeignKey("history_dataset_collection_association.id"), index=True),
+    Column("output_name", String(255), nullable=True),
+)
+
+implicit_collection_jobs_table = Table(
+    "implicit_collection_jobs", metadata,
+    Column("id", Integer, primary_key=True),
+    Column("populated_state", TrimmedString(64), default='new', nullable=False),
+)
+
+implicit_collection_jobs_job_association_table = Table(
+    "implicit_collection_jobs_job_association", metadata,
+    Column("implicit_collection_jobs_id", Integer, ForeignKey("implicit_collection_jobs.id"), index=True),
+    Column("id", Integer, primary_key=True),
+    Column("job_id", Integer, ForeignKey("job.id"), index=True),  # Consider making this nullable...
+    Column("order_index", Integer, nullable=False),
+)
+
+
+def get_new_tables():
+    # Normally we define this globally in the file, but we need to delay the
+    # reading of existing tables because an existing workflow_invocation_step
+    # table exists that we want to recreate.
+
+    tables = OrderedDict()
+    tables["workflow_invocation_output_dataset_association"] = workflow_invocation_output_dataset_association_table
+    tables["workflow_invocation_output_dataset_collection_association"] = workflow_invocation_output_dataset_collection_association_table
+    tables["workflow_invocation_step_output_dataset_association"] = workflow_invocation_step_output_dataset_association_table
+    tables["workflow_invocation_step_output_dataset_collection_association"] = workflow_invocation_step_output_dataset_collection_association_table
+    tables["implicit_collection_jobs"] = implicit_collection_jobs_table
+    tables["implicit_collection_jobs_job_association"] = implicit_collection_jobs_job_association_table
+
+    return tables
+
+
+def upgrade(migrate_engine):
+    metadata.bind = migrate_engine
+    print(__doc__)
+
+    metadata.reflect()
+    tables = get_new_tables()
+    for table in tables.values():
+        __create(table)
+
+    def nextval(table, col='id'):
+        if migrate_engine.name in ['postgres', 'postgresql']:
+            return "nextval('%s_%s_seq')" % (table, col)
+        elif migrate_engine.name in ['mysql', 'sqlite']:
+            return "null"
+        else:
+            raise Exception("Unhandled database type")
+
+    # Set default for creation to scheduled, actual mapping has new as default.
+    workflow_invocation_step_state_column = Column("state", TrimmedString(64), default="scheduled")
+    if migrate_engine.name in ['postgres', 'postgresql']:
+        implicit_collection_jobs_id_column = Column("implicit_collection_jobs_id", Integer, ForeignKey("implicit_collection_jobs.id"), nullable=True)
+        job_id_column = Column("job_id", Integer, ForeignKey("job.id"), nullable=True)
+    else:
+        implicit_collection_jobs_id_column = Column("implicit_collection_jobs_id", Integer, nullable=True)
+        job_id_column = Column("job_id", Integer, nullable=True)
+    dataset_collection_element_count_column = Column("element_count", Integer, nullable=True)
+
+    __add_column(implicit_collection_jobs_id_column, "history_dataset_collection_association", metadata)
+    __add_column(job_id_column, "history_dataset_collection_association", metadata)
+    __add_column(dataset_collection_element_count_column, "dataset_collection", metadata)
+
+    implicit_collection_jobs_id_column = Column("implicit_collection_jobs_id", Integer, ForeignKey("implicit_collection_jobs.id"), nullable=True)
+    __add_column(implicit_collection_jobs_id_column, "workflow_invocation_step", metadata)
+    __add_column(workflow_invocation_step_state_column, "workflow_invocation_step", metadata)
+
+    cmd = \
+        "UPDATE dataset_collection SET element_count = " + \
+        "(SELECT (CASE WHEN count(*) > 0 THEN count(*) ELSE 0 END) FROM dataset_collection_element WHERE " + \
+        "dataset_collection_element.dataset_collection_id = dataset_collection.id)"
+    migrate_engine.execute(cmd)
+
+
+def __add_column(column, table_name, metadata, **kwds):
+    try:
+        table = Table(table_name, metadata, autoload=True)
+        column.create(table, **kwds)
+    except Exception:
+        log.exception("Adding column %s failed.", column)
+
+
+def __drop_column(column_name, table_name, metadata):
+    try:
+        table = Table(table_name, metadata, autoload=True)
+        getattr(table.c, column_name).drop()
+    except Exception:
+        log.exception("Dropping column %s failed.", column_name)
+
+
+def downgrade(migrate_engine):
+    metadata.bind = migrate_engine
+    metadata.reflect()
+
+    __drop_column("implicit_collection_jobs_id", "history_dataset_collection_association", metadata)
+    __drop_column("job_id", "history_dataset_collection_association", metadata)
+    __drop_column("implicit_collection_jobs_id", "workflow_invocation_step", metadata)
+    __drop_column("state", "workflow_invocation_step", metadata)
+    __drop_column("element_count", "dataset_collection", metadata)
+
+    tables = get_new_tables()
+    for table in reversed(tables.values()):
+        __drop(table)
+
+
+def __create(table):
+    try:
+        table.create()
+    except Exception:
+        log.exception("Creating %s table failed.", table.name)
+
+
+def __drop(table):
+    try:
+        table.drop()
+    except Exception:
+        log.exception("Dropping %s table failed.", table.name)

lib/galaxy/tools/__init__.py

@@ -84,7 +84,10 @@ from galaxy.web import url_for
 from galaxy.web.form_builder import SelectField
 from galaxy.work.context import WorkRequestContext
 from tool_shed.util import common_util
-from .execute import execute as execute_job
+from .execute import (
+    execute as execute_job,
+    MappingParameters,
+)
 from .loader import (
     imported_macro_paths,
     raw_tool_xml_tree,
@@ -1244,8 +1247,6 @@ class Tool(object, Dictifiable):
         # Fixed set of input parameters may correspond to any number of jobs.
         # Expand these out to individual parameters for given jobs (tool executions).
         expanded_incomings, collection_info = expand_meta_parameters(trans, self, incoming)
-        if not expanded_incomings:
-            raise exceptions.MessageException('Tool execution failed, trying to run a tool over an empty collection.')
 
         # Remapping a single job to many jobs doesn't make sense, so disable
         # remap if multi-runs of tools are being used.
@@ -1295,24 +1296,38 @@ class Tool(object, Dictifiable):
             err_data = {key: value for d in all_errors for (key, value) in d.items()}
             raise exceptions.MessageException(', '.join(msg for msg in err_data.values()), err_data=err_data)
         else:
-            execution_tracker = execute_job(trans, self, all_params, history=request_context.history, rerun_remap_job_id=rerun_remap_job_id, collection_info=collection_info)
-            if execution_tracker.successful_jobs:
-                return dict(out_data=execution_tracker.output_datasets,
-                            num_jobs=len(execution_tracker.successful_jobs),
-                            job_errors=execution_tracker.execution_errors,
-                            jobs=execution_tracker.successful_jobs,
-                            output_collections=execution_tracker.output_collections,
-                            implicit_collections=execution_tracker.implicit_collections)
-            else:
+            mapping_params = MappingParameters(incoming, all_params)
+            execution_tracker = execute_job(trans, self, mapping_params, history=request_context.history, rerun_remap_job_id=rerun_remap_job_id, collection_info=collection_info)
+            # Raise an exception if there were jobs to execute and none of them were submitted,
+            # if at least one is submitted or there are no jobs to execute - return aggregate
+            # information including per-job errors. Arguably we should just always return the
+            # aggregate information - we just haven't done that historically.
+            raise_execution_exception = not execution_tracker.successful_jobs and len(all_params) > 0
+
+            if raise_execution_exception:
                 raise exceptions.MessageException(execution_tracker.execution_errors[0])
 
-    def handle_single_execution(self, trans, rerun_remap_job_id, params, history, mapping_over_collection, execution_cache=None):
+            return dict(out_data=execution_tracker.output_datasets,
+                        num_jobs=len(execution_tracker.successful_jobs),
+                        job_errors=execution_tracker.execution_errors,
+                        jobs=execution_tracker.successful_jobs,
+                        output_collections=execution_tracker.output_collections,
+                        implicit_collections=execution_tracker.implicit_collections)
+
+    def handle_single_execution(self, trans, rerun_remap_job_id, execution_slice, history, execution_cache=None):
         """
         Return a pair with whether execution is successful as well as either
         resulting output data or an error message indicating the problem.
         """
         try:
-            job, out_data = self.execute(trans, incoming=params, history=history, rerun_remap_job_id=rerun_remap_job_id, mapping_over_collection=mapping_over_collection, execution_cache=execution_cache)
+            job, out_data = self.execute(
+                trans,
+                incoming=execution_slice.param_combination,
+                history=history,
+                rerun_remap_job_id=rerun_remap_job_id,
+                execution_cache=execution_cache,
+                dataset_collection_elements=execution_slice.dataset_collection_elements,
+            )
         except httpexceptions.HTTPFound as e:
             # if it's a paste redirect exception, pass it up the stack
             raise e
@@ -2255,7 +2270,7 @@ class DatabaseOperationTool(Tool):
     def check_inputs_ready(self, input_datasets, input_dataset_collections):
         def check_dataset_instance(input_dataset):
             if input_dataset.is_pending:
-                raise ToolInputsNotReadyException()
+                raise ToolInputsNotReadyException("An input dataset is pending.")
 
             if self.require_dataset_ok:
                 if input_dataset.state != input_dataset.dataset.states.OK:
@@ -2267,7 +2282,7 @@ class DatabaseOperationTool(Tool):
         for input_dataset_collection_pairs in input_dataset_collections.values():
             for input_dataset_collection, is_mapped in input_dataset_collection_pairs:
                 if not input_dataset_collection.collection.populated:
-                    raise ToolInputsNotReadyException()
+                    raise ToolInputsNotReadyException("An input collection is not populated.")
 
             map(check_dataset_instance, input_dataset_collection.dataset_instances)
 

lib/galaxy/tools/actions/__init__.py

@@ -195,7 +195,7 @@ class DefaultToolAction(object):
 
         return history, inp_data, inp_dataset_collections
 
-    def execute(self, tool, trans, incoming={}, return_job=False, set_output_hid=True, history=None, job_params=None, rerun_remap_job_id=None, mapping_over_collection=False, execution_cache=None):
+    def execute(self, tool, trans, incoming={}, return_job=False, set_output_hid=True, history=None, job_params=None, rerun_remap_job_id=None, execution_cache=None, dataset_collection_elements=None):
         """
         Executes a tool, creating job and tool outputs, associating them, and
         submitting the job to the job queue. If history is not specified, use
@@ -267,7 +267,7 @@ class DefaultToolAction(object):
             tool=tool,
             tool_action=self,
             input_collections=input_collections,
-            mapping_over_collection=mapping_over_collection,
+            dataset_collection_elements=dataset_collection_elements,
             on_text=on_text,
             incoming=incoming,
             params=wrapped_params.params,
@@ -304,8 +304,12 @@ class DefaultToolAction(object):
                 data = app.model.HistoryDatasetAssociation(extension=ext, create_dataset=True, flush=False)
                 if hidden is None:
                     hidden = output.hidden
+                if not hidden and dataset_collection_elements is not None:  # Mapping over a collection - hide datasets
+                    hidden = True
                 if hidden:
                     data.visible = False
+                if dataset_collection_elements is not None and name in dataset_collection_elements:
+                    dataset_collection_elements[name].hda = data
                 trans.sa_session.add(data)
                 trans.app.security_agent.set_all_dataset_permissions(data.dataset, output_permissions, new=True)
             for _, tag in preserved_tags.items():
@@ -351,6 +355,7 @@ class DefaultToolAction(object):
 
         for name, output in tool.outputs.items():
             if not filter_output(output, incoming):
+                handle_output_timer = ExecutionTimer()
                 if output.collection:
                     collections_manager = app.dataset_collections_service
                     element_identifiers = []
@@ -402,15 +407,14 @@ class DefaultToolAction(object):
                         element_kwds = dict(elements=collections_manager.ELEMENTS_UNINITIALIZED)
                     else:
                         element_kwds = dict(element_identifiers=element_identifiers)
-
                     output_collections.create_collection(
                         output=output,
                         name=name,
                         tags=preserved_tags,
                         **element_kwds
                     )
+                    log.info("Handled collection output named %s for tool %s %s" % (name, tool.id, handle_output_timer))
                 else:
-                    handle_output_timer = ExecutionTimer()
                     handle_output(name, output)
                     log.info("Handled output named %s for tool %s %s" % (name, tool.id, handle_output_timer))
 
@@ -594,6 +598,7 @@ class DefaultToolAction(object):
             job.add_implicit_output_dataset_collection(name, dataset_collection)
         for name, dataset_collection_instance in out_collection_instances.items():
             job.add_output_dataset_collection(name, dataset_collection_instance)
+            dataset_collection_instance.job = job
 
     def _check_input_data_access(self, trans, job, inp_data, current_user_roles):
         access_timer = ExecutionTimer()
@@ -670,13 +675,13 @@ class OutputCollections(object):
     parameter).
     """
 
-    def __init__(self, trans, history, tool, tool_action, input_collections, mapping_over_collection, on_text, incoming, params, job_params):
+    def __init__(self, trans, history, tool, tool_action, input_collections, dataset_collection_elements, on_text, incoming, params, job_params):
         self.trans = trans
         self.history = history
         self.tool = tool
         self.tool_action = tool_action
         self.input_collections = input_collections
-        self.mapping_over_collection = mapping_over_collection
+        self.dataset_collection_elements = dataset_collection_elements
         self.on_text = on_text
         self.incoming = incoming
         self.params = params
@@ -710,12 +715,15 @@ class OutputCollections(object):
                         for dataset in value.dataset_instances:
                             assert dataset.history is not None
 
-        if self.mapping_over_collection:
+        if self.dataset_collection_elements is not None:
             dc = collections_manager.create_dataset_collection(
                 self.trans,
                 collection_type=collection_type,
                 **element_kwds
             )
+            if name in self.dataset_collection_elements:
+                self.dataset_collection_elements[name].child_collection = dc
+                # self.trans.sa_session.add(self.dataset_collection_elements[name])
             self.out_collections[name] = dc
         else:
             hdca_name = self.tool_action.get_output_name(

lib/galaxy/tools/actions/model_operations.py

@@ -21,7 +21,7 @@ class ModelOperationToolAction(DefaultToolAction):
 
         tool.check_inputs_ready(inp_data, inp_dataset_collections)
 
-    def execute(self, tool, trans, incoming={}, set_output_hid=False, overwrite=True, history=None, job_params=None, mapping_over_collection=False, execution_cache=None, **kwargs):
+    def execute(self, tool, trans, incoming={}, set_output_hid=False, overwrite=True, history=None, job_params=None, execution_cache=None, **kwargs):
         if execution_cache is None:
             execution_cache = ToolExecutionCache(trans)
 
@@ -42,7 +42,7 @@ class ModelOperationToolAction(DefaultToolAction):
             tool=tool,
             tool_action=self,
             input_collections=input_collections,
-            mapping_over_collection=mapping_over_collection,
+            dataset_collection_elements=kwargs.get("dataset_collection_elements", None),
             on_text=on_text,
             incoming=incoming,
             params=wrapped_params.params,

lib/galaxy/tools/actions/upload_common.py

@@ -124,7 +124,7 @@ def persist_uploads(params, trans):
             elif type(f) == dict and 'local_filename' not in f:
                 raise Exception('Uploaded file was encoded in a way not understood by Galaxy.')
             if upload_dataset['url_paste'] and upload_dataset['url_paste'].strip() != '':
-                upload_dataset['url_paste'], is_multi_byte = datatypes.sniff.stream_to_file(
+                upload_dataset['url_paste'] = datatypes.sniff.stream_to_file(
                     StringIO(validate_url(upload_dataset['url_paste'], trans.app.config.fetch_url_whitelist_ips)),
                     prefix="strio_url_paste_"
                 )

lib/galaxy/tools/execute.py

@@ -4,12 +4,17 @@ from various states, tracking results, and building implicit dataset
 collections from matched collections.
 """
 import collections
+import itertools
 import logging
 from threading import Thread
 
+import six
+
 from six.moves.queue import Queue
 
-from galaxy.tools.actions import on_text_for_names, ToolExecutionCache
+from galaxy import model
+from galaxy.dataset_collections.structure import tool_output_to_structure
+from galaxy.tools.actions import filter_output, on_text_for_names, ToolExecutionCache
 from galaxy.tools.parser import ToolOutputCollectionPart
 from galaxy.util import ExecutionTimer
 
@@ -18,36 +23,51 @@ log = logging.getLogger(__name__)
 EXECUTION_SUCCESS_MESSAGE = "Tool [%s] created job [%s] %s"
 
 
-def execute(trans, tool, param_combinations, history, rerun_remap_job_id=None, collection_info=None, workflow_invocation_uuid=None):
+class PartialJobExecution(Exception):
+
+    def __init__(self, execution_tracker):
+        self.execution_tracker = execution_tracker
+
+
+MappingParameters = collections.namedtuple("MappingParameters", ["param_template", "param_combinations"])
+
+
+def execute(trans, tool, mapping_params, history, rerun_remap_job_id=None, collection_info=None, workflow_invocation_uuid=None, invocation_step=None, max_num_jobs=None, job_callback=None):
     """
     Execute a tool and return object containing summary (output data, number of
     failures, etc...).
     """
+    if max_num_jobs:
+        assert invocation_step is not None
+    if rerun_remap_job_id:
+        assert invocation_step is None
+
     all_jobs_timer = ExecutionTimer()
-    execution_tracker = ToolExecutionTracker(tool, param_combinations, collection_info)
+    if invocation_step is None:
+        execution_tracker = ToolExecutionTracker(trans, tool, mapping_params, collection_info)
+    else:
+        execution_tracker = WorkflowStepExecutionTracker(trans, tool, mapping_params, collection_info, invocation_step, job_callback=job_callback)
     app = trans.app
     execution_cache = ToolExecutionCache(trans)
 
-    def execute_single_job(params):
+    def execute_single_job(execution_slice):
         job_timer = ExecutionTimer()
+        params = execution_slice.param_combination
         if workflow_invocation_uuid:
             params['__workflow_invocation_uuid__'] = workflow_invocation_uuid
         elif '__workflow_invocation_uuid__' in params:
             # Only workflow invocation code gets to set this, ignore user supplied
             # values or rerun parameters.
             del params['__workflow_invocation_uuid__']
-        job, result = tool.handle_single_execution(trans, rerun_remap_job_id, params, history, collection_info, execution_cache)
+
+        job, result = tool.handle_single_execution(trans, rerun_remap_job_id, execution_slice, history, execution_cache)
         if job:
             message = EXECUTION_SUCCESS_MESSAGE % (tool.id, job.id, job_timer)
             log.debug(message)
-            execution_tracker.record_success(job, result)
+            execution_tracker.record_success(execution_slice, job, result)
         else:
             execution_tracker.record_error(result)
 
-    config = app.config
-    burst_at = getattr(config, 'tool_submission_burst_at', 10)
-    burst_threads = getattr(config, 'tool_submission_burst_threads', 1)
-
     tool_action = tool.tool_action
     if hasattr(tool_action, "check_inputs_ready"):
         for params in execution_tracker.param_combinations:
@@ -59,11 +79,26 @@ def execute(trans, tool, param_combinations, history, rerun_remap_job_id=None, c
                 history
             )
 
+    execution_tracker.ensure_implicit_collections_populated(history, mapping_params.param_template)
+    config = app.config
+    burst_at = getattr(config, 'tool_submission_burst_at', 10)
+    burst_threads = getattr(config, 'tool_submission_burst_threads', 1)
+
     job_count = len(execution_tracker.param_combinations)
-    if job_count < burst_at or burst_threads < 2:
-        for params in execution_tracker.param_combinations:
-            execute_single_job(params)
+
+    jobs_executed = 0
+    has_remaining_jobs = False
+
+    if (job_count < burst_at or burst_threads < 2):
+        for execution_slice in execution_tracker.new_execution_slices():
+            if max_num_jobs and jobs_executed >= max_num_jobs:
+                has_remaining_jobs = True
+                break
+            else:
+                execute_single_job(execution_slice)
+                jobs_executed += 1
     else:
+        # TODO: re-record success...
         q = Queue()
 
         def worker():
@@ -77,42 +112,270 @@ def execute(trans, tool, param_combinations, history, rerun_remap_job_id=None, c
             t.daemon = True
             t.start()
 
-        for params in execution_tracker.param_combinations:
-            q.put(params)
+        for execution_slice in execution_tracker.new_execution_slices():
+            if max_num_jobs and jobs_executed >= max_num_jobs:
+                has_remaining_jobs = True
+                break
+            else:
+                q.put(execution_slice)
+                jobs_executed += 1
 
         q.join()
 
-    log.debug("Executed %d job(s) for tool %s request: %s" % (job_count, tool.id, all_jobs_timer))
-    if collection_info:
-        history = history or tool.get_default_history_by_trans(trans)
-        if len(param_combinations) == 0:
-            template = "Attempting to map over an empty collection, this is not yet implemented. collection_info is [%s]"
-            message = template % collection_info
-            log.warning(message)
-            raise Exception(message)
-        params = param_combinations[0]
-        execution_tracker.create_output_collections(trans, history, params)
+    if has_remaining_jobs:
+        raise PartialJobExecution(execution_tracker)
+    else:
+        execution_tracker.finalize_dataset_collections(trans)
 
+    log.debug("Executed %d job(s) for tool %s request: %s" % (job_count, tool.id, all_jobs_timer))
     return execution_tracker
 
 
-class ToolExecutionTracker(object):
+class ExecutionSlice(object):
 
-    def __init__(self, tool, param_combinations, collection_info):
+    def __init__(self, job_index, param_combination, dataset_collection_elements=None):
+        self.job_index = job_index
+        self.param_combination = param_combination
+        self.dataset_collection_elements = dataset_collection_elements
+
+
+class ExecutionTracker(object):
+
+    def __init__(self, trans, tool, mapping_params, collection_info):
+        # Known ahead of time...
+        self.trans = trans
         self.tool = tool
-        self.param_combinations = param_combinations
+        self.mapping_params = mapping_params
         self.collection_info = collection_info
-        self.successful_jobs = []
+
+        self._on_text = None
+
+        # Populated as we go...
         self.failed_jobs = 0
         self.execution_errors = []
+
+        self.successful_jobs = []
         self.output_datasets = []
         self.output_collections = []
-        self.outputs_by_output_name = collections.defaultdict(list)
+
         self.implicit_collections = {}
 
-    def record_success(self, job, outputs):
+    @property
+    def param_combinations(self):
+        return self.mapping_params.param_combinations
+
+    @property
+    def example_params(self):
+        if self.mapping_params.param_combinations:
+            return self.mapping_params.param_combinations[0]
+        else:
+            # TODO: This isn't quite right - what we want is something like param_template wrapped,
+            # need a test case with an output filter applied to an empty list, still this is
+            # an improvement over not allowing mapping of empty lists.
+            return self.mapping_params.param_template
+
+    @property
+    def job_count(self):
+        return len(self.param_combinations)
+
+    def record_error(self, error):
+        self.failed_jobs += 1
+        message = "There was a failure executing a job for tool [%s] - %s"
+        log.warning(message, self.tool.id, error)
+        self.execution_errors.append(error)
+
+    @property
+    def on_text(self):
+        if self._on_text is None:
+            collection_names = ["collection %d" % c.hid for c in self.collection_info.collections.values()]
+            self._on_text = on_text_for_names(collection_names)
+
+        return self._on_text
+
+    def output_name(self, trans, history, params, output):
+        on_text = self.on_text
+
+        try:
+            output_collection_name = self.tool.tool_action.get_output_name(
+                output,
+                dataset=None,
+                tool=self.tool,
+                on_text=on_text,
+                trans=trans,
+                history=history,
+                params=params,
+                incoming=None,
+                job_params=None,
+            )
+        except Exception:
+            output_collection_name = "%s across %s" % (self.tool.name, on_text)
+
+        return output_collection_name
+
+    def sliced_input_collection_type(self, input_name):
+        if self.is_implicit_input(input_name):
+            subcollection_mapping_type = self.collection_info.subcollection_mapping_type(input_name)
+            return subcollection_mapping_type
+            # return self.collection_info.structure.sliced_input_collection_type(self.implicit_inputs[input_name])
+        else:
+            return self.mapping_params.param_template[input_name].collection.collection_type
+
+    def _structure_for_output(self, trans, tool_output):
+        structure = self.collection_info.structure
+        if hasattr(tool_output, "default_identifier_source"):
+            # Switch the structure for outputs if the output specified a default_identifier_source
+            collection_type_descriptions = trans.app.dataset_collections_service.collection_type_descriptions
+
+            source_collection = self.collection_info.collections.get(tool_output.default_identifier_source)
+            if source_collection:
+                collection_type_description = collection_type_descriptions.for_collection_type(source_collection.collection.collection_type)
+                _structure = structure.for_dataset_collection(source_collection.collection, collection_type_description=collection_type_description)
+                if structure.can_match(_structure):
+                    structure = _structure
+
+        return structure
+
+    def _element_identifiers_for_output(self, trans, tool_output, outputs):
+        output_structure = self._structure_for_output(trans, tool_output)
+        element_identifiers = output_structure.element_identifiers_for_outputs(trans, outputs)
+        return element_identifiers
+
+    def _mapped_output_structure(self, trans, tool_output):
+        collections_manager = trans.app.dataset_collections_service
+        output_structure = tool_output_to_structure(self.sliced_input_collection_type, tool_output, collections_manager)
+        mapping_structure = self._structure_for_output(trans, tool_output)
+        # Output structure may not be known, but input structure must be,
+        # otherwise this step of the workflow shouldn't have been scheduled
+        # or the tool should not have been executable on this input.
+        mapped_output_structure = mapping_structure.multiply(output_structure, uninitialized=True)
+        return mapped_output_structure
+
+    def ensure_implicit_collections_populated(self, history, params):
+        if not self.collection_info:
+            return
+
+        history = history or self.tool.get_default_history_by_trans(self.trans)
+        # params = param_combinations[0] if param_combinations else mapping_params.param_template
+        self.precreate_output_collections(history, params)
+
+    def precreate_output_collections(self, history, params):
+        # params is just one sample tool param execution with parallelized
+        # collection replaced with a specific dataset. Need to replace this
+        # with the collection and wrap everything up so can evaluate output
+        # label.
+        trans = self.trans
+        params.update(self.collection_info.collections)  # Replace datasets with source collections for labelling outputs.
+
+        collection_instances = {}
+        implicit_inputs = self.implicit_inputs
+
+        implicit_collection_jobs = model.ImplicitCollectionJobs()
+        for output_name, output in self.tool.outputs.items():
+            if filter_output(output, self.example_params):
+                continue
+            output_collection_name = self.output_name(trans, history, params, output)
+            effective_structure = self._mapped_output_structure(trans, output)
+            collection_instance = trans.app.dataset_collections_service.precreate_dataset_collection_instance(
+                trans=trans,
+                parent=history,
+                name=output_collection_name,
+                implicit_inputs=implicit_inputs,
+                implicit_output_name=output_name,
+                structure=effective_structure,
+            )
+            collection_instance.implicit_collection_jobs = implicit_collection_jobs
+            collection_instances[output_name] = collection_instance
+            trans.sa_session.add(collection_instance)
+        # Needed to flush the association created just above with
+        # job.add_output_dataset_collection.
+        trans.sa_session.flush()
+        self.implicit_collections = collection_instances
+
+    @property
+    def implicit_collection_jobs(self):
+        # TODO: refactor to track this properly maybe?
+        if self.implicit_collections:
+            return six.next(six.itervalues(self.implicit_collections)).implicit_collection_jobs
+        else:
+            return None
+
+    def finalize_dataset_collections(self, trans):
+        # TODO: this probably needs to be reworked some, we should have the collection methods
+        # return a list of changed objects to add to the session and flush and we should only
+        # be finalizing collections to a depth of self.collection_info.structure. So for instance
+        # if you are mapping a list over a tool that dynamically generates lists - we won't actually
+        # know the structure of the inner list until after its job is complete.
+        if self.failed_jobs > 0:
+            for i, implicit_collection in enumerate(self.implicit_collections.values()):
+                if i == 0:
+                    implicit_collection_jobs = implicit_collection.implicit_collection_jobs
+                    implicit_collection_jobs.populated_state = "failed"
+                    trans.sa_session.add(implicit_collection_jobs)
+                implicit_collection.collection.handle_population_failed("One or more jobs failed during dataset initialization.")
+                trans.sa_session.add(implicit_collection.collection)
+        else:
+            for i, implicit_collection in enumerate(self.implicit_collections.values()):
+                if i == 0:
+                    implicit_collection_jobs = implicit_collection.implicit_collection_jobs
+                    implicit_collection_jobs.populated_state = "ok"
+                    trans.sa_session.add(implicit_collection_jobs)
+                implicit_collection.collection.finalize()
+                trans.sa_session.add(implicit_collection.collection)
+        trans.sa_session.flush()
+
+    @property
+    def implicit_inputs(self):
+        implicit_inputs = list(self.collection_info.collections.items())
+        return implicit_inputs
+
+    def is_implicit_input(self, input_name):
+        return input_name in self.collection_info.collections
+
+    def walk_implicit_collections(self):
+        return self.collection_info.structure.walk_collections(self.implicit_collections)
+
+    def new_execution_slices(self):
+        if self.collection_info is None:
+            for job_index, param_combination in enumerate(self.param_combinations):
+                yield ExecutionSlice(job_index, param_combination)
+        else:
+            for execution_slice in self.new_collection_execution_slices():
+                yield execution_slice
+
+    def record_success(self, execution_slice, job, outputs):
+        # TODO: successful_jobs need to be inserted in the correct place...
         self.successful_jobs.append(job)
         self.output_datasets.extend(outputs)
+        for job_output in job.output_dataset_collection_instances:
+            self.output_collections.append((job_output.name, job_output.dataset_collection_instance))
+        if self.implicit_collections:
+            implicit_collection_jobs = None
+            for output_name, collection_instance in self.implicit_collections.items():
+                job.add_output_dataset_collection(output_name, collection_instance)
+                if implicit_collection_jobs is None:
+                    implicit_collection_jobs = collection_instance.implicit_collection_jobs
+
+            job_assoc = model.ImplicitCollectionJobsJobAssociation()
+            job_assoc.order_index = execution_slice.job_index
+            job_assoc.implicit_collection_jobs = implicit_collection_jobs
+            job_assoc.job_id = job.id
+            self.trans.sa_session.add(job_assoc)
+
+
+# Seperate these because workflows need to track their jobs belong to the invocation
+# in the database immediately and they can be recovered.
+class ToolExecutionTracker(ExecutionTracker):
+
+    def __init__(self, trans, tool, mapping_params, collection_info):
+        super(ToolExecutionTracker, self).__init__(trans, tool, mapping_params, collection_info)
+
+        # New to track these things for tool output API response in the tool case,
+        # in the workflow case we just write stuff to the database and forget about
+        # it.
+        self.outputs_by_output_name = collections.defaultdict(list)
+
+    def record_success(self, execution_slice, job, outputs):
+        super(ToolExecutionTracker, self).record_success(execution_slice, job, outputs)
         for output_name, output_dataset in outputs:
             if ToolOutputCollectionPart.is_named_collection_part_name(output_name):
                 # Skip known collection outputs, these will be covered by
@@ -121,101 +384,58 @@ class ToolExecutionTracker(object):
             self.outputs_by_output_name[output_name].append(output_dataset)
         for job_output in job.output_dataset_collections:
             self.outputs_by_output_name[job_output.name].append(job_output.dataset_collection)
-        for job_output in job.output_dataset_collection_instances:
-            self.output_collections.append((job_output.name, job_output.dataset_collection_instance))
 
-    def record_error(self, error):
-        self.failed_jobs += 1
-        message = "There was a failure executing a job for tool [%s] - %s"
-        log.warning(message, self.tool.id, error)
-        self.execution_errors.append(error)
+    def new_collection_execution_slices(self):
+        for job_index, (param_combination, dataset_collection_elements) in enumerate(itertools.izip(self.param_combinations, self.walk_implicit_collections())):
+            for dataset_collection_element in dataset_collection_elements.values():
+                assert dataset_collection_element.element_object is None
 
-    def create_output_collections(self, trans, history, params):
-        # TODO: Move this function - it doesn't belong here but it does need
-        # the information in this class and potential extensions.
-        if self.failed_jobs > 0:
-            return []
+            yield ExecutionSlice(job_index, param_combination, dataset_collection_elements)
 
-        structure = self.collection_info.structure
 
-        # params is just one sample tool param execution with parallelized
-        # collection replaced with a specific dataset. Need to replace this
-        # with the collection and wrap everything up so can evaluate output
-        # label.
-        params.update(self.collection_info.collections)  # Replace datasets with source collections for labelling outputs.
+class WorkflowStepExecutionTracker(ExecutionTracker):
 
-        collection_names = ["collection %d" % c.hid for c in self.collection_info.collections.values()]
-        on_text = on_text_for_names(collection_names)
+    def __init__(self, trans, tool, mapping_params, collection_info, invocation_step, job_callback):
+        super(WorkflowStepExecutionTracker, self).__init__(trans, tool, mapping_params, collection_info)
+        self.invocation_step = invocation_step
+        self.job_callback = job_callback
 
-        collections = {}
+    def record_success(self, execution_slice, job, outputs):
+        super(WorkflowStepExecutionTracker, self).record_success(execution_slice, job, outputs)
+        if self.collection_info:
+            self.invocation_step.implicit_collection_jobs = self.implicit_collection_jobs
+        else:
+            self.invocation_step.job = job
+        self.job_callback(job)
 
-        implicit_inputs = list(self.collection_info.collections.items())
-        for output_name, outputs in self.outputs_by_output_name.items():
-            if not len(structure) == len(outputs):
-                # Output does not have the same structure, if all jobs were
-                # successfully submitted this shouldn't have happened.
-                log.warning("Problem matching up datasets while attempting to create implicit dataset collections")
+    def new_collection_execution_slices(self):
+        for job_index, (param_combination, dataset_collection_elements) in enumerate(itertools.izip(self.param_combinations, self.walk_implicit_collections())):
+            # Two options here - check if the element has been populated or check if the
+            # a WorkflowInvocationStepJobAssociation exists. Not sure which is better but
+            # for now I have the first so lets check.
+            found_result = False
+            for dataset_collection_element in dataset_collection_elements.values():
+                if dataset_collection_element.element_object is not None:
+                    found_result = True
+                    break
+            if found_result:
                 continue
-            output = self.tool.outputs[output_name]
+            yield ExecutionSlice(job_index, param_combination, dataset_collection_elements)
 
-            element_identifiers = None
-            if hasattr(output, "default_identifier_source"):
-                # Switch the structure for outputs if the output specified a default_identifier_source
-                collection_type_descriptions = trans.app.dataset_collections_service.collection_type_descriptions
+    def ensure_implicit_collections_populated(self, history, params):
+        if not self.collection_info:
+            return
 
-                source_collection = self.collection_info.collections.get(output.default_identifier_source)
-                if source_collection:
-                    collection_type_description = collection_type_descriptions.for_collection_type(source_collection.collection.collection_type)
-                    _structure = structure.for_dataset_collection(source_collection.collection, collection_type_description=collection_type_description)
-                    if structure.can_match(_structure):
-                        element_identifiers = _structure.element_identifiers_for_outputs(trans, outputs)
-
-            if not element_identifiers:
-                element_identifiers = structure.element_identifiers_for_outputs(trans, outputs)
-
-            implicit_collection_info = dict(
-                implicit_inputs=implicit_inputs,
-                implicit_output_name=output_name,
-                outputs=outputs
-            )
-            try:
-                output_collection_name = self.tool.tool_action.get_output_name(
-                    output,
-                    dataset=None,
-                    tool=self.tool,
-                    on_text=on_text,
-                    trans=trans,
-                    history=history,
-                    params=params,
-                    incoming=None,
-                    job_params=None,
-                )
-            except Exception:
-                output_collection_name = "%s across %s" % (self.tool.name, on_text)
-
-            child_element_identifiers = element_identifiers["element_identifiers"]
-            collection_type = element_identifiers["collection_type"]
-            collection = trans.app.dataset_collections_service.create(
-                trans=trans,
-                parent=history,
-                name=output_collection_name,
-                element_identifiers=child_element_identifiers,
-                collection_type=collection_type,
-                implicit_collection_info=implicit_collection_info,
-            )
-            for job in self.successful_jobs:
-                # TODO: Think through this, may only want this for output
-                # collections - or we may be already recording data in some
-                # other way.
-                if job not in trans.sa_session:
-                    job = trans.sa_session.query(trans.app.model.Job).get(job.id)
-                job.add_output_dataset_collection(output_name, collection)
-            collections[output_name] = collection
-
-        # Needed to flush the association created just above with
-        # job.add_output_dataset_collection.
-        trans.sa_session.flush()
-        self.implicit_collections = collections
+        history = history or self.tool.get_default_history_by_trans(self.trans)
+        if self.invocation_step.is_new:
+            self.precreate_output_collections(history, params)
+        else:
+            collections = {}
+            for output_assoc in self.invocation_step.output_dataset_collections:
+                implicit_collection = output_assoc.dataset_collection
+                assert hasattr(implicit_collection, "history_content_type")  # make sure it is an HDCA and not a DC
+                collections[output_assoc.output_name] = output_assoc.dataset_collection
+            self.implicit_collections = collections
 
 
 __all__ = ('execute', )

lib/galaxy/tools/parameters/grouping.py

@@ -565,7 +565,7 @@ class UploadDataset(Group):
             dataset.precreated_name = dataset.name = self.get_composite_dataset_name(context)
             if dataset.datatype.composite_type == 'auto_primary_file':
                 # replace sniff here with just creating an empty file
-                temp_name, is_multi_byte = sniff.stream_to_file(StringIO(d_type.generate_primary_file(dataset)), prefix='upload_auto_primary_file')
+                temp_name = sniff.stream_to_file(StringIO(d_type.generate_primary_file(dataset)), prefix='upload_auto_primary_file')
                 dataset.primary_file = temp_name
                 dataset.to_posix_lines = True
                 dataset.auto_decompress = True

lib/galaxy/tools/parser/output_objects.py

@@ -195,7 +195,10 @@ class ToolOutputCollectionStructure(object):
         if self.structured_like:
             collection_prototype = inputs[self.structured_like].collection
         else:
-            collection_prototype = type_registry.prototype(self.collection_type)
+            collection_type = self.collection_type
+            assert collection_type
+            collection_prototype = type_registry.prototype(collection_type)
+            collection_prototype.collection_type = collection_type
         return collection_prototype
 
 

lib/galaxy/util/image_util.py

@@ -39,39 +39,3 @@ def check_image_type(filename, types):
     if fmt in types:
         return True
     return False
-
-
-def get_image_ext(file_path):
-    # determine ext
-    fmt = image_type(file_path)
-    if fmt in ['JPG', 'JPEG']:
-        return 'jpg'
-    if fmt == 'PNG':
-        return 'png'
-    if fmt == 'TIFF':
-        return 'tiff'
-    if fmt == 'BMP':
-        return 'bmp'
-    if fmt == 'GIF':
-        return 'gif'
-    if fmt == 'IM':
-        return 'im'
-    if fmt == 'PCD':
-        return 'pcd'
-    if fmt == 'PCX':
-        return 'pcx'
-    if fmt == 'PPM':
-        return 'ppm'
-    if fmt == 'PSD':
-        return 'psd'
-    if fmt == 'XBM':
-        return 'xbm'
-    if fmt == 'XPM':
-        return 'xpm'
-    if fmt == 'RGB':
-        return 'rgb'
-    if fmt == 'PBM':
-        return 'pbm'
-    if fmt == 'PGM':
-        return 'pgm'
-    return None

lib/galaxy/util/multi_byte.py

@@ -1,25 +0,0 @@
-try:
-    import wchartype
-except ImportError:
-    wchartype = None
-
-from six import text_type
-
-
-def is_multi_byte(chars):
-    if wchartype is None:
-        message = "Attempted to use galaxy.util.multi_byte but dependency wchartype is unavailable."
-        raise Exception(message)
-    for char in chars:
-        try:
-            char = text_type(char)
-        except UnicodeDecodeError:
-            # Probably binary
-            return False
-        if (wchartype.is_asian(char) or wchartype.is_full_width(char) or
-                wchartype.is_kanji(char) or wchartype.is_hiragana(char) or
-                wchartype.is_katakana(char) or wchartype.is_half_katakana(char) or
-                wchartype.is_hangul(char) or wchartype.is_full_digit(char) or
-                wchartype.is_full_letter(char)):
-            return True
-    return False

lib/galaxy/webapps/galaxy/api/history_contents.py

@@ -21,6 +21,7 @@ from galaxy.managers.collections_util import (
     dictify_dataset_collection_instance,
     get_hda_and_element_identifiers
 )
+from galaxy.managers.jobs import fetch_job_states, summarize_jobs_to_dict
 from galaxy.util.json import safe_dumps
 from galaxy.util.streamball import StreamBall
 from galaxy.web import (
@@ -126,33 +127,133 @@ class HistoryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibrary
 
         return rval
 
-    def __collection_dict(self, trans, dataset_collection_instance, view="collection"):
+    def __collection_dict(self, trans, dataset_collection_instance, **kwds):
         return dictify_dataset_collection_instance(dataset_collection_instance,
-            security=trans.security, parent=dataset_collection_instance.history, view=view)
+            security=trans.security, parent=dataset_collection_instance.history, **kwds)
 
     @expose_api_anonymous
     def show(self, trans, id, history_id, **kwd):
         """
-        show( self, trans, id, history_id, **kwd )
         * GET /api/histories/{history_id}/contents/{id}
-            return detailed information about an HDA within a history
+        * GET /api/histories/{history_id}/contents/{type}/{id}
+            return detailed information about an HDA or HDCA within a history
         .. note:: Anonymous users are allowed to get their current history contents
 
         :type   id:         str
-        :param  id:        the encoded id of the HDA to return
+        :param  id:         the encoded id of the HDA or HDCA to return
+        :type   type:       str
+        :param  id:         'dataset' or 'dataset_collection'
         :type   history_id: str
-        :param  history_id: encoded id string of the HDA's History
+        :param  history_id: encoded id string of the HDA's or HDCA's History
+        :type   view:       str
+        :param  view:       if fetching a dataset collection - the view style of
+                            the dataset collection to produce.
+                            'collection' returns no element information, 'element'
+                            returns detailed element information for all datasets,
+                            'element-reference' returns a minimal set of information
+                            about datasets (for instance id, type, and state but not
+                            metadata, peek, info, or name). The default is 'element'.
+        :type  fuzzy_count: int
+        :param fuzzy_count: this value can be used to broadly restrict the magnitude
+                            of the number of elements returned via the API for large
+                            collections. The number of actual elements returned may
+                            be "a bit" more than this number or "a lot" less - varying
+                            on the depth of nesting, balance of nesting at each level,
+                            and size of target collection. The consumer of this API should
+                            not expect a stable number or pre-calculable number of
+                            elements to be produced given this parameter - the only
+                            promise is that this API will not respond with an order
+                            of magnitude more elements estimated with this value.
+                            The UI uses this parameter to fetch a "balanced" concept of
+                            the "start" of large collections at every depth of the
+                            collection.
 
         :rtype:     dict
-        :returns:   dictionary containing detailed HDA information
+        :returns:   dictionary containing detailed HDA or HDCA information
         """
-        contents_type = kwd.get('type', 'dataset')
+        contents_type = self.__get_contents_type(trans, kwd)
         if contents_type == 'dataset':
             return self.__show_dataset(trans, id, **kwd)
         elif contents_type == 'dataset_collection':
             return self.__show_dataset_collection(trans, id, history_id, **kwd)
+
+    @expose_api_anonymous
+    def index_jobs_summary(self, trans, history_id, **kwd):
+        """
+        * GET /api/histories/{history_id}/jobs_summary
+            return detailed information about an HDA or HDCAs jobs
+
+        Warning: We allow anyone to fetch job state information about any object they
+        can guess an encoded ID for - it isn't considered protected data. This keeps
+        polling IDs as part of state calculation for large histories and collections as
+        efficient as possible.
+
+        :type   history_id: str
+        :param  history_id: encoded id string of the HDA's or the HDCA's History
+        :type   ids:        str[]
+        :param  ids:        the encoded ids of job summary objects to return - if ids
+                            is specified types must also be specified and have same length.
+        :type   types:      str[]
+        :param  types:      type of object represented by elements in the ids array - either
+                            Job or ImplicitCollectionJob.
+
+        :rtype:     dict[]
+        :returns:   an array of job summary object dictionaries.
+        """
+        ids = kwd.get("ids", None)
+        types = kwd.get("types", None)
+        if ids is None:
+            assert types is None
+            # TODO: ...
+            pass
         else:
-            return self.__handle_unknown_contents_type(trans, contents_type)
+            ids = util.listify(ids)
+            types = util.listify(types)
+        return map(lambda s: self.encode_all_ids(trans, s), fetch_job_states(self.app, trans.sa_session, ids, types))
+
+    @expose_api_anonymous
+    def show_jobs_summary(self, trans, id, history_id, **kwd):
+        """
+        * GET /api/histories/{history_id}/contents/{type}/{id}/jobs_summary
+            return detailed information about an HDA or HDCAs jobs
+
+        Warning: We allow anyone to fetch job state information about any object they
+        can guess an encoded ID for - it isn't considered protected data. This keeps
+        polling IDs as part of state calculation for large histories and collections as
+        efficient as possible.
+
+        :type   id:         str
+        :param  id:         the encoded id of the HDA to return
+        :type   history_id: str
+        :param  history_id: encoded id string of the HDA's or the HDCA's History
+
+        :rtype:     dict
+        :returns:   dictionary containing jobs summary object
+        """
+        contents_type = self.__get_contents_type(trans, kwd)
+        # At most one of job or implicit_collection_jobs should be found.
+        job = None
+        implicit_collection_jobs = None
+        if contents_type == 'dataset':
+            hda = self.hda_manager.get_accessible(self.decode_id(id), trans.user)
+            job = hda.creating_job
+        elif contents_type == 'dataset_collection':
+            dataset_collection_instance = self.__get_accessible_collection(trans, id, history_id)
+            job_source_type = dataset_collection_instance.job_source_type
+            if job_source_type == "Job":
+                job = dataset_collection_instance.job
+            elif job_source_type == "ImplicitCollectionJobs":
+                implicit_collection_jobs = dataset_collection_instance.implicit_collection_jobs
+
+        assert job is None or implicit_collection_jobs is None
+        return self.encode_all_ids(trans, summarize_jobs_to_dict(trans.sa_session, job or implicit_collection_jobs))
+
+    def __get_contents_type(self, trans, kwd):
+        contents_type = kwd.get('type', 'dataset')
+        if contents_type not in ['dataset', 'dataset_collection']:
+            self.__handle_unknown_contents_type(trans, contents_type)
+
+        return contents_type
 
     def __show_dataset(self, trans, id, **kwd):
         hda = self.hda_manager.get_accessible(self.decode_id(id), trans.user)
@@ -162,18 +263,19 @@ class HistoryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibrary
                                                      **self._parse_serialization_params(kwd, 'detailed'))
 
     def __show_dataset_collection(self, trans, id, history_id, **kwd):
-        try:
-            service = trans.app.dataset_collections_service
-            dataset_collection_instance = service.get_dataset_collection_instance(
-                trans=trans,
-                instance_type='history',
-                id=id,
-            )
-            return self.__collection_dict(trans, dataset_collection_instance, view="element")
-        except Exception as e:
-            log.exception("Error in history API at listing dataset collection")
-            trans.response.status = 500
-            return {'error': str(e)}
+        dataset_collection_instance = self.__get_accessible_collection(trans, id, history_id)
+        view = kwd.get("view", "element")
+        fuzzy_count = kwd.get("fuzzy_count", None)
+        if fuzzy_count:
+            fuzzy_count = int(fuzzy_count)
+        return self.__collection_dict(trans, dataset_collection_instance, view=view, fuzzy_count=fuzzy_count)
+
+    def __get_accessible_collection(self, trans, id, history_id):
+        return trans.app.dataset_collections_service.get_dataset_collection_instance(
+            trans=trans,
+            instance_type="history",
+            id=id
+        )
 
     @expose_api_raw_anonymous
     def download_dataset_collection(self, trans, id, history_id=None, **kwd):
@@ -188,14 +290,8 @@ class HistoryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibrary
         :param history_id: encoded id string of the HDCA's History
         """
         try:
-            service = trans.app.dataset_collections_service
-            dataset_collection_instance = service.get_dataset_collection_instance(
-                trans=trans,
-                instance_type='history',
-                id=id,
-            )
+            dataset_collection_instance = self.__get_accessible_collection(trans, id, history_id)
             return self.__stream_dataset_collection(trans, dataset_collection_instance)
-
         except Exception as e:
             log.exception("Error in API while creating dataset collection archive")
             trans.response.status = 500

lib/galaxy/webapps/galaxy/buildapp.py

@@ -363,6 +363,15 @@ def populate_api_routes(webapp, app):
                           action='download_dataset_collection',
                           conditions=dict(method=["GET"]))
 
+    webapp.mapper.connect("/api/histories/{history_id}/jobs_summary",
+                          action="index_jobs_summary",
+                          controller='history_contents',
+                          conditions=dict(method=["GET"]))
+
+    webapp.mapper.connect("/api/histories/{history_id}/contents/{type:%s}s/{id}/jobs_summary" % "|".join(valid_history_contents_types),
+                          action="show_jobs_summary",
+                          controller='history_contents',
+                          conditions=dict(method=["GET"]))
     # ---- visualizations registry ---- generic template renderer
     # @deprecated: this route should be considered deprecated
     webapp.add_route('/visualization/show/{visualization_name}', controller='visualization', action='render', visualization_name=None)

lib/galaxy/workflow/modules.py

@@ -21,7 +21,7 @@ from galaxy.tools import (
     DefaultToolState,
     ToolInputsNotReadyException
 )
-from galaxy.tools.execute import execute
+from galaxy.tools.execute import execute, MappingParameters, PartialJobExecution
 from galaxy.tools.parameters import (
     check_param,
     params_to_incoming,
@@ -214,10 +214,15 @@ class WorkflowModule(object):
         state.decode(runtime_state, Bunch(inputs=self.get_runtime_inputs()), self.trans.app)
         return state
 
-    def execute(self, trans, progress, invocation, step):
-        """ Execute the given workflow step in the given workflow invocation.
+    def execute(self, trans, progress, invocation_step):
+        """ Execute the given workflow invocation step.
+
         Use the supplied workflow progress object to track outputs, find
-        inputs, etc...
+        inputs, etc....
+
+        Return a False if there is additional processing required to
+        on subsequent workflow scheduling runs, None or True means the workflow
+        step executed properly.
         """
         raise TypeError("Abstract method")
 
@@ -230,11 +235,19 @@ class WorkflowModule(object):
         """
         raise exceptions.RequestParameterInvalidException("Attempting to perform invocation step action on module that does not support actions.")
 
-    def recover_mapping(self, step, step_invocations, progress):
+    def recover_mapping(self, invocation_step, progress):
         """ Re-populate progress object with information about connections
-        from previously executed steps recorded via step_invocations.
+        from previously executed steps recorded via invocation_steps.
         """
-        raise TypeError("Abstract method")
+        outputs = {}
+
+        for output_dataset_assoc in invocation_step.output_datasets:
+            outputs[output_dataset_assoc.output_name] = output_dataset_assoc.dataset
+
+        for output_dataset_collection_assoc in invocation_step.output_dataset_collections:
+            outputs[output_dataset_collection_assoc.output_name] = output_dataset_collection_assoc.dataset_collection
+
+        progress.set_step_outputs(invocation_step, outputs, already_persisted=True)
 
 
 class SubWorkflowModule(WorkflowModule):
@@ -320,11 +333,12 @@ class SubWorkflowModule(WorkflowModule):
     def get_content_id(self):
         return self.trans.security.encode_id(self.subworkflow.id)
 
-    def execute(self, trans, progress, invocation, step):
+    def execute(self, trans, progress, invocation_step):
         """ Execute the given workflow step in the given workflow invocation.
         Use the supplied workflow progress object to track outputs, find
         inputs, etc...
         """
+        step = invocation_step.workflow_step
         subworkflow_invoker = progress.subworkflow_invoker(trans, step)
         subworkflow_invoker.invoke()
         subworkflow = subworkflow_invoker.workflow
@@ -334,7 +348,7 @@ class SubWorkflowModule(WorkflowModule):
             workflow_output_label = workflow_output.label or "%s:%s" % (step.order_index, workflow_output.output_name)
             replacement = subworkflow_progress.get_replacement_workflow_output(workflow_output)
             outputs[workflow_output_label] = replacement
-        progress.set_step_outputs(step, outputs)
+        progress.set_step_outputs(invocation_step, outputs)
         return None
 
     def get_runtime_state(self):
@@ -353,8 +367,10 @@ class InputModule(WorkflowModule):
     def get_data_inputs(self):
         return []
 
-    def execute(self, trans, progress, invocation, step):
-        job, step_outputs = None, dict(output=step.state.inputs['input'])
+    def execute(self, trans, progress, invocation_step):
+        invocation = invocation_step.workflow_invocation
+        step = invocation_step.workflow_step
+        step_outputs = dict(output=step.state.inputs['input'])
 
         # Web controller may set copy_inputs_to_history, API controller always sets
         # inputs.
@@ -378,11 +394,10 @@ class InputModule(WorkflowModule):
             content = next(iter(step_outputs.values()))
             if content:
                 invocation.add_input(content, step.id)
-        progress.set_outputs_for_input(step, step_outputs)
-        return job
+        progress.set_outputs_for_input(invocation_step, step_outputs)
 
-    def recover_mapping(self, step, step_invocations, progress):
-        progress.set_outputs_for_input(step)
+    def recover_mapping(self, invocation_step, progress):
+        progress.set_outputs_for_input(invocation_step)
 
 
 class InputDataModule(InputModule):
@@ -489,10 +504,10 @@ class InputParameterModule(WorkflowModule):
     def get_data_inputs(self):
         return []
 
-    def execute(self, trans, progress, invocation, step):
-        job, step_outputs = None, dict(output=step.state.inputs['input'])
-        progress.set_outputs_for_input(step, step_outputs)
-        return job
+    def execute(self, trans, progress, invocation_step):
+        step = invocation_step.workflow_step
+        step_outputs = dict(output=step.state.inputs['input'])
+        progress.set_outputs_for_input(invocation_step, step_outputs)
 
 
 class PauseModule(WorkflowModule):
@@ -520,18 +535,18 @@ class PauseModule(WorkflowModule):
         state.inputs = dict()
         return state
 
-    def execute(self, trans, progress, invocation, step):
+    def execute(self, trans, progress, invocation_step):
+        step = invocation_step.workflow_step
         progress.mark_step_outputs_delayed(step, why="executing pause step")
-        return None
 
-    def recover_mapping(self, step, step_invocations, progress):
-        if step_invocations:
-            step_invocation = step_invocations[0]
-            action = step_invocation.action
+    def recover_mapping(self, invocation_step, progress):
+        if invocation_step:
+            step = invocation_step.workflow_step
+            action = invocation_step.action
             if action:
                 connection = step.input_connections_by_name["input"][0]
                 replacement = progress.replacement_for_connection(connection)
-                progress.set_step_outputs(step, {'output': replacement})
+                progress.set_step_outputs(invocation_step, {'output': replacement})
                 return
             elif action is False:
                 raise CancelWorkflowEvaluation()
@@ -790,7 +805,9 @@ class ToolModule(WorkflowModule):
         else:
             raise ToolMissingException("Tool %s missing. Cannot recover runtime state." % self.tool_id)
 
-    def execute(self, trans, progress, invocation, step):
+    def execute(self, trans, progress, invocation_step):
+        invocation = invocation_step.workflow_invocation
+        step = invocation_step.workflow_step
         tool = trans.app.toolbox.get_tool(step.tool_id, tool_version=step.tool_version)
         tool_state = step.state
         # Not strictly needed - but keep Tool state clean by stripping runtime
@@ -855,57 +872,53 @@ class ToolModule(WorkflowModule):
 
             param_combinations.append(execution_state.inputs)
 
+        complete = False
         try:
+            mapping_params = MappingParameters(tool_state.inputs, param_combinations)
+            max_num_jobs = progress.maximum_jobs_to_schedule_or_none
             execution_tracker = execute(
                 trans=self.trans,
                 tool=tool,
-                param_combinations=param_combinations,
+                mapping_params=mapping_params,
                 history=invocation.history,
                 collection_info=collection_info,
-                workflow_invocation_uuid=invocation.uuid.hex
+                workflow_invocation_uuid=invocation.uuid.hex,
+                invocation_step=invocation_step,
+                max_num_jobs=max_num_jobs,
+                job_callback=lambda job: self._handle_post_job_actions(step, job, invocation.replacement_dict),
             )
+            complete = True
+        except PartialJobExecution as pje:
+            execution_tracker = pje.execution_tracker
+
         except ToolInputsNotReadyException:
             delayed_why = "tool [%s] inputs are not ready, this special tool requires inputs to be ready" % tool.id
             raise DelayedWorkflowEvaluation(why=delayed_why)
 
+        progress.record_executed_job_count(len(execution_tracker.successful_jobs))
         if collection_info:
             step_outputs = dict(execution_tracker.implicit_collections)
         else:
             step_outputs = dict(execution_tracker.output_datasets)
             step_outputs.update(execution_tracker.output_collections)
-        progress.set_step_outputs(step, step_outputs)
-        jobs = execution_tracker.successful_jobs
-        for job in jobs:
-            self._handle_post_job_actions(step, job, invocation.replacement_dict)
+        progress.set_step_outputs(invocation_step, step_outputs, already_persisted=not invocation_step.is_new)
+
         if execution_tracker.execution_errors:
-            failed_count = len(execution_tracker.execution_errors)
-            success_count = len(execution_tracker.successful_jobs)
-            all_count = failed_count + success_count
-            message = "Failed to create %d out of %s job(s) for workflow step." % (failed_count, all_count)
+            message = "Failed to create one or more job(s) for workflow step."
             raise Exception(message)
-        return jobs
 
-    def recover_mapping(self, step, step_invocations, progress):
-        # Grab a job representing this invocation - for normal workflows
-        # there will be just one job but if this step was mapped over there
-        # may be many.
-        job_0 = step_invocations[0].job
+        return complete
 
+    def recover_mapping(self, invocation_step, progress):
         outputs = {}
-        for job_output in job_0.output_datasets:
-            replacement_name = job_output.name
-            replacement_value = job_output.dataset
-            # If was a mapping step, grab the output mapped collection for
-            # replacement instead.
-            if replacement_value.hidden_beneath_collection_instance:
-                replacement_value = replacement_value.hidden_beneath_collection_instance
-            outputs[replacement_name] = replacement_value
-        for job_output_collection in job_0.output_dataset_collection_instances:
-            replacement_name = job_output_collection.name
-            replacement_value = job_output_collection.dataset_collection_instance
-            outputs[replacement_name] = replacement_value
 
-        progress.set_step_outputs(step, outputs)
+        for output_dataset_assoc in invocation_step.output_datasets:
+            outputs[output_dataset_assoc.output_name] = output_dataset_assoc.dataset
+
+        for output_dataset_collection_assoc in invocation_step.output_dataset_collections:
+            outputs[output_dataset_collection_assoc.output_name] = output_dataset_collection_assoc.dataset_collection
+
+        progress.set_step_outputs(invocation_step, outputs)
 
     def _find_collections_to_match(self, tool, progress, step):
         collections_to_match = matching.CollectionsToMatch()

lib/galaxy/workflow/run.py

@@ -1,7 +1,7 @@
 import logging
 import uuid
 
-from galaxy import model, util
+from galaxy import model
 from galaxy.util import ExecutionTimer
 from galaxy.util.odict import odict
 from galaxy.workflow import modules
@@ -140,12 +140,18 @@ class WorkflowInvoker(object):
 
         module_injector = modules.WorkflowModuleInjector(trans)
         if progress is None:
-            progress = WorkflowProgress(self.workflow_invocation, workflow_run_config.inputs, module_injector)
+            progress = WorkflowProgress(
+                self.workflow_invocation,
+                workflow_run_config.inputs,
+                module_injector,
+                jobs_per_scheduling_iteration=getattr(trans.app.config, "maximum_workflow_jobs_per_scheduling_iteration", -1),
+            )
         self.progress = progress
 
     def invoke(self):
         workflow_invocation = self.workflow_invocation
-        maximum_duration = getattr(self.trans.app.config, "maximum_workflow_invocation_duration", -1)
+        config = self.trans.app.config
+        maximum_duration = getattr(config, "maximum_workflow_invocation_duration", -1)
         if maximum_duration > 0 and workflow_invocation.seconds_since_created > maximum_duration:
             log.debug("Workflow invocation [%s] exceeded maximum number of seconds allowed for scheduling [%s], failing." % (workflow_invocation.id, maximum_duration))
             workflow_invocation.state = model.WorkflowInvocation.states.FAILED
@@ -162,24 +168,27 @@ class WorkflowInvoker(object):
 
         remaining_steps = self.progress.remaining_steps()
         delayed_steps = False
-        for step in remaining_steps:
+        for (step, workflow_invocation_step) in remaining_steps:
             step_delayed = False
             step_timer = ExecutionTimer()
-            jobs = None
             try:
                 self.__check_implicitly_dependent_steps(step)
 
-                # TODO: step may fail to invoke, do something about that.
-                jobs = self._invoke_step(step)
-                for job in (util.listify(jobs) or [None]):
-                    # Record invocation
+                if not workflow_invocation_step:
                     workflow_invocation_step = model.WorkflowInvocationStep()
                     workflow_invocation_step.workflow_invocation = workflow_invocation
                     workflow_invocation_step.workflow_step = step
-                    # Job may not be generated in this thread if bursting is enabled
-                    # https://github.com/galaxyproject/galaxy/issues/2259
-                    if job:
-                        workflow_invocation_step.job_id = job.id
+                    workflow_invocation_step.state = 'new'
+
+                    workflow_invocation.steps.append(workflow_invocation_step)
+
+                incomplete_or_none = self._invoke_step(workflow_invocation_step)
+                if incomplete_or_none is False:
+                    step_delayed = delayed_steps = True
+                    workflow_invocation_step.state = 'ready'
+                    self.progress.mark_step_outputs_delayed(step, why="Not all jobs scheduled for state.")
+                else:
+                    workflow_invocation_step.state = 'scheduled'
             except modules.DelayedWorkflowEvaluation as de:
                 step_delayed = delayed_steps = True
                 self.progress.mark_step_outputs_delayed(step, why=de.why)
@@ -218,15 +227,19 @@ class WorkflowInvoker(object):
                 self.__check_implicitly_dependent_step(output_id)
 
     def __check_implicitly_dependent_step(self, output_id):
-        step_invocations = self.workflow_invocation.step_invocations_for_step_id(output_id)
+        step_invocation = self.workflow_invocation.step_invocation_for_step_id(output_id)
 
         # No steps created yet - have to delay evaluation.
-        if not step_invocations:
+        if not step_invocation:
             delayed_why = "depends on step [%s] but that step has not been invoked yet" % output_id
             raise modules.DelayedWorkflowEvaluation(why=delayed_why)
 
-        for step_invocation in step_invocations:
-            job = step_invocation.job
+        if step_invocation.state != 'scheduled':
+            delayed_why = "depends on step [%s] job has not finished scheduling yet" % output_id
+            raise modules.DelayedWorkflowEvaluation(delayed_why)
+
+        for job_assoc in step_invocation.jobs:
+            job = job_assoc.job
             if job:
                 # At least one job in incomplete.
                 if not job.finished:
@@ -241,9 +254,9 @@ class WorkflowInvoker(object):
                 # pause steps.
                 pass
 
-    def _invoke_step(self, step):
-        jobs = step.module.execute(self.trans, self.progress, self.workflow_invocation, step)
-        return jobs
+    def _invoke_step(self, invocation_step):
+        incomplete_or_none = invocation_step.workflow_step.module.execute(self.trans, self.progress, invocation_step)
+        return incomplete_or_none
 
 
 STEP_OUTPUT_DELAYED = object()
@@ -251,16 +264,31 @@ STEP_OUTPUT_DELAYED = object()
 
 class WorkflowProgress(object):
 
-    def __init__(self, workflow_invocation, inputs_by_step_id, module_injector):
+    def __init__(self, workflow_invocation, inputs_by_step_id, module_injector, jobs_per_scheduling_iteration=-1):
         self.outputs = odict()
         self.module_injector = module_injector
         self.workflow_invocation = workflow_invocation
         self.inputs_by_step_id = inputs_by_step_id
+        self.jobs_per_scheduling_iteration = jobs_per_scheduling_iteration
+        self.jobs_scheduled_this_iteration = 0
+
+    @property
+    def maximum_jobs_to_schedule_or_none(self):
+        if self.jobs_per_scheduling_iteration > 0:
+            return self.jobs_per_scheduling_iteration - self.jobs_scheduled_this_iteration
+        else:
+            return None
+
+    def record_executed_job_count(self, job_count):
+        self.jobs_scheduled_this_iteration += job_count
 
     def remaining_steps(self):
         # Previously computed and persisted step states.
         step_states = self.workflow_invocation.step_states_by_step_id()
         steps = self.workflow_invocation.workflow.steps
+
+        # TODO: Wouldn't a generator be much better here so we don't have to reason about
+        # steps we are no where near ready to schedule?
         remaining_steps = []
         step_invocations_by_id = self.workflow_invocation.step_invocations_by_step_id()
         for step in steps:
@@ -274,11 +302,11 @@ class WorkflowProgress(object):
                 runtime_state = step_states[step_id].value
                 step.state = step.module.decode_runtime_state(runtime_state)
 
-            invocation_steps = step_invocations_by_id.get(step_id, None)
-            if invocation_steps:
-                self._recover_mapping(step, invocation_steps)
+            invocation_step = step_invocations_by_id.get(step_id, None)
+            if invocation_step and invocation_step.state == 'scheduled':
+                self._recover_mapping(invocation_step)
             else:
-                remaining_steps.append(step)
+                remaining_steps.append((step, invocation_step))
         return remaining_steps
 
     def replacement_for_tool_input(self, step, input, prefixed_name):
@@ -340,7 +368,9 @@ class WorkflowProgress(object):
         output_name = workflow_output.output_name
         return self.outputs[step.id][output_name]
 
-    def set_outputs_for_input(self, step, outputs=None):
+    def set_outputs_for_input(self, invocation_step, outputs=None):
+        step = invocation_step.workflow_step
+
         if outputs is None:
             outputs = {}
 
@@ -352,10 +382,34 @@ class WorkflowProgress(object):
                 raise ValueError(message)
             outputs['output'] = self.inputs_by_step_id[step_id]
 
-        self.set_step_outputs(step, outputs)
+        self.set_step_outputs(invocation_step, outputs)
 
-    def set_step_outputs(self, step, outputs):
+    def set_step_outputs(self, invocation_step, outputs, already_persisted=False):
+        step = invocation_step.workflow_step
         self.outputs[step.id] = outputs
+        if not already_persisted:
+            for output_name, output_object in outputs.items():
+                if hasattr(output_object, "history_content_type"):
+                    invocation_step.add_output(output_name, output_object)
+                else:
+                    # This is a problem, this non-data, non-collection output
+                    # won't be recovered on a subsequent workflow scheduling
+                    # iteration. This seems to have been a pre-existing problem
+                    # prior to #4584 though.
+                    pass
+            for workflow_output in step.workflow_outputs:
+                output_name = workflow_output.output_name
+                if output_name not in outputs:
+                    raise KeyError("Failed to find [%s] in step outputs [%s]" % (output_name, outputs))
+                output = outputs[output_name]
+                self._record_workflow_output(
+                    step,
+                    workflow_output,
+                    output=output,
+                )
+
+    def _record_workflow_output(self, step, workflow_output, output):
+        self.workflow_invocation.add_output(workflow_output, step, output)
 
     def mark_step_outputs_delayed(self, step, why=None):
         if why:
@@ -414,11 +468,11 @@ class WorkflowProgress(object):
             self.module_injector,
         )
 
-    def _recover_mapping(self, step, step_invocations):
+    def _recover_mapping(self, step_invocation):
         try:
-            step.module.recover_mapping(step, step_invocations, self)
+            step_invocation.workflow_step.module.recover_mapping(step_invocation, self)
         except modules.DelayedWorkflowEvaluation as de:
-            self.mark_step_outputs_delayed(step, de.why)
+            self.mark_step_outputs_delayed(step_invocation.workflow_step, de.why)
 
 
 __all__ = ('invoke', 'WorkflowRunConfig')