Add job state tracking to the GUI.

- ``populated`` in the API response is gone - it was very expensive on the backend to calculate. - Eliminate GUI "thread" fetching ``element_count`` for collections after the panel is loaded, this value is now available at load time thanks to recent state change optimizations on the backend. - Track job state and implicit collection job states over time in a new "thread". - New different colors of job states, add progress bar for multiple jobs.
2026-09-24 16:30:27 +08:00 · 2017-11-30 10:02:03 -05:00
parent ca09e803ae
commit 5a9f0dd11e
11 changed files with 439 additions and 55 deletions
@@ -16,10 +16,14 @@ var HDCAListItemView = _super.extend(
        /** event listeners */
        _setUpListeners: function() {
            _super.prototype._setUpListeners.call(this);
+            var renderListen = (model, options) => {
+                this.render();
+            };
+            if (this.model.jobStatesSummary) {
+                this.listenTo(this.model.jobStatesSummary, "change", renderListen);
+            }
            this.listenTo(this.model, {
-                "change:tags change:populated change:visible": function(model, options) {
-                    this.render();
-                }
+                "change:tags change:visible change:state": renderListen
            });
        },

@@ -43,13 +47,95 @@ var HDCAListItemView = _super.extend(
        _swapNewRender: function($newRender) {
            _super.prototype._swapNewRender.call(this, $newRender);
            //TODO: model currently has no state
-            var state = !this.model.get("populated") ? STATES.RUNNING : STATES.OK;
-            //if( this.model.has( 'state' ) ){
+            var state;
+            var jobStatesSummary = this.model.jobStatesSummary;
+            if (jobStatesSummary) {
+                if (jobStatesSummary.new()) {
+                    state = "new";
+                } else if (jobStatesSummary.errored()) {
+                    state = "error";
+                } else if (jobStatesSummary.terminal()) {
+                    state = "ok";
+                } else if (jobStatesSummary.running()) {
+                    state = "running";
+                } else {
+                    state = "queued";
+                }
+            } else if (this.model.get("job_source_id")) {
+                // Initial rendering - polling will fill in more details in a bit.
+                state = STATES.NEW;
+            } else {
+                state = this.model.get("populated_state") ? STATES.OK : STATES.RUNNING;
+            }
            this.$el.addClass(`state-${state}`);
-            //}
+            var stateDescription = this.stateDescription();
+            this.$(".state-description").html(stateDescription);
            return this.$el;
        },

+        stateDescription: function() {
+            var collection = this.model;
+            var elementCount = collection.get("element_count");
+            var jobStateSource = collection.get("job_source_type");
+            var collectionType = this.model.get("collection_type");
+            var collectionTypeDescription;
+            if (collectionType == "list") {
+                collectionTypeDescription = "list";
+            } else if (collectionType == "paired") {
+                collectionTypeDescription = "dataset pair";
+            } else if (collectionType == "list:paired") {
+                collectionTypeDescription = "list of pairs";
+            } else {
+                collectionTypeDescription = "nested list";
+            }
+            var itemsDescription = "";
+            if (elementCount == 1) {
+                itemsDescription = ` with 1 item`;
+            } else if (elementCount) {
+                itemsDescription = ` with ${elementCount} items`;
+            }
+            var jobStatesSummary = collection.jobStatesSummary;
+            var simpleDescription = `${collectionTypeDescription}${itemsDescription}`;
+            if (!jobStateSource || jobStateSource == "Job") {
+                return `a ${simpleDescription}`;
+            } else if (!jobStatesSummary || !jobStatesSummary.hasDetails()) {
+                return `
+                    <div class="progress state-progress">
+                        <span class="note">Loading job data for ${collectionTypeDescription}...</span>
+                        <div class="progress-bar info" style="width:100%">
+                    </div>`;
+            } else {
+                var isNew = jobStatesSummary.new();
+                var jobCount = isNew ? null : jobStatesSummary.jobCount();
+                if (isNew) {
+                    return `
+                        <div class="progress state-progress">
+                            <span class="note">Creating jobs...</span>
+                            <div class="progress-bar info" style="width:100%">
+                        </div>`;
+                } else if (jobStatesSummary.errored()) {
+                    var errorCount = jobStatesSummary.numInError();
+                    return `a ${collectionTypeDescription} with ${errorCount} / ${jobCount} jobs in error`;
+                } else if (jobStatesSummary.terminal()) {
+                    return `a ${simpleDescription}`;
+                } else {
+                    var running = jobStatesSummary.states()["running"] || 0;
+                    var ok = jobStatesSummary.states()["ok"] || 0;
+                    var okPercent = ok / (jobCount * 1.0);
+                    var runningPercent = running / (jobCount * 1.0);
+                    var otherPercent = 1.0 - okPercent - runningPercent;
+                    var jobsStr = jobCount && jobCount > 1 ? `${jobCount} jobs` : `a job`;
+                    return `
+                        <div class="progress state-progress">
+                            <span class="note">${jobsStr} generating a ${collectionTypeDescription}</span>
+                            <div class="progress-bar ok" style="width:${okPercent * 100.0}%"></div>
+                            <div class="progress-bar running" style="width:${runningPercent * 100.0}%"></div>
+                            <div class="progress-bar new" style="width:${otherPercent * 100.0}%">
+                        </div>`;
+                }
+            }
+        },
+
        // ......................................................................... misc
        /** String representation */
        toString: function() {
@@ -69,7 +155,6 @@ HDCAListItemView.prototype.templates = (() => {
        }
    });

-    // could steal this from hda-base (or use mixed content)
    var titleBarTemplate = collection => `
        <div class="title-bar clear" tabindex="0">
            <span class="state-icon"></span>
@@ -77,7 +162,8 @@ HDCAListItemView.prototype.templates = (() => {
                <span class="hid">${collection.hid}</span>
                <span class="name">${_.escape(collection.name)}</span>
            </div>
-            <div class="subtitle"></div>
+            <div class="state-description">
+            </div>
            ${HISTORY_ITEM_LI.nametagTemplate(collection)}
        </div>
    `;
@@ -2,6 +2,7 @@ import CONTROLLED_FETCH_COLLECTION from "mvc/base/controlled-fetch-collection";
 import HDA_MODEL from "mvc/history/hda-model";
 import HDCA_MODEL from "mvc/history/hdca-model";
 import HISTORY_PREFS from "mvc/history/history-preferences";
+import JOB_STATES_MODEL from "mvc/history/job-states-model";
 import BASE_MVC from "mvc/base-mvc";
 import AJAX_QUEUE from "utils/ajax-queue";

@@ -37,6 +38,10 @@ var HistoryContents = _super.extend(BASE_MVC.LoggableMixin).extend({

    /** Set up */
    initialize: function(models, options) {
+        this.on({
+            "sync add": this.trackJobStates
+        });
+
        options = options || {};
        _super.prototype.initialize.call(this, models, options);

@@ -53,6 +58,29 @@ var HistoryContents = _super.extend(BASE_MVC.LoggableMixin).extend({
        this.model.prototype.idAttribute = "type_id";
    },

+    trackJobStates: function() {
+        this.each(historyContent => {
+            if (historyContent.has("job_states_summary")) {
+                return;
+            }
+
+            if (historyContent.attributes.history_content_type === "dataset_collection") {
+                var jobSourceType = historyContent.attributes.job_source_type;
+                var jobSourceId = historyContent.attributes.job_source_id;
+                if (jobSourceType) {
+                    this.jobStateSummariesCollection.add({
+                        id: jobSourceId,
+                        model: jobSourceType,
+                        history_id: this.history_id,
+                        collection_id: historyContent.attributes.id
+                    });
+                    var jobStatesSummary = this.jobStateSummariesCollection.get(jobSourceId);
+                    historyContent.jobStatesSummary = jobStatesSummary;
+                }
+            }
+        });
+    },
+
    // ........................................................................ composite collection
    /** since history content is a mix, override model fn into a factory, creating based on history_content_type */
    model: function(attrs, options) {
@@ -82,17 +110,30 @@ var HistoryContents = _super.extend(BASE_MVC.LoggableMixin).extend({
        };
    },

+    stopPolling: function() {
+        if (this.jobStateSummariesCollection) {
+            this.jobStateSummariesCollection.active = false;
+            this.jobStateSummariesCollection.clearUpdateTimeout();
+        }
+    },
+
    setHistoryId: function(newId) {
+        this.stopPolling();
        this.historyId = newId;
-        this._setUpWebStorage();
+        if (newId) {
+            // If actually reflecting a history - setup storage and monitor jobs.
+
+            this._setUpWebStorage();
+
+            this.jobStateSummariesCollection = new JOB_STATES_MODEL.JobStatesSummaryCollection();
+            this.jobStateSummariesCollection.historyId = newId;
+            this.jobStateSummariesCollection.monitor();
+        }
    },

    /** Set up client side storage. Currently PersistanStorage keyed under 'history:<id>' */
    _setUpWebStorage: function(initialSettings) {
        // TODO: use initialSettings
-        if (!this.historyId) {
-            return;
-        }
        this.storage = new HISTORY_PREFS.HistoryPrefs({
            id: HISTORY_PREFS.HistoryPrefs.historyStorageKey(this.historyId)
        });
@@ -303,17 +344,6 @@ var HistoryContents = _super.extend(BASE_MVC.LoggableMixin).extend({
        return this.fetch(options);
    },

-    /** specialty fetch method for retrieving the element_counts of all hdcas in the history */
-    fetchCollectionCounts: function(options) {
-        options = options || {};
-        options.keys = ["type_id", "element_count"].join(",");
-        options.filters = _.extend(options.filters || {}, {
-            history_content_type: "dataset_collection"
-        });
-        options.remove = false;
-        return this.fetch(options);
-    },
-
    // ............. quasi-batch ops
    // TODO: to batch
    /** helper that fetches using filterParams then calls save on each fetched using updateWhat as the save params */
@@ -237,6 +237,13 @@ var History = Backbone.Model.extend(BASE_MVC.LoggableMixin).extend(
                }
            },

+            stopPolling: function() {
+                this.clearUpdateTimeout();
+                if (this.contents) {
+                    this.contents.stopPolling();
+                }
+            },
+
            // ........................................................................ ajax
            /** override to use actual Dates objects for create/update times */
            parse: function(response, options) {
@@ -47,9 +47,6 @@ var HistoryView = _super.extend(
        /** string used for search placeholder */
        searchPlaceholder: _l("search datasets"),

-        /** @type {Number} ms to wait after history load to fetch/decorate hdcas with element_count */
-        FETCH_COLLECTION_COUNTS_DELAY: 2000,
-
        // ......................................................................... SET UP
        /** Set up the view, bind listeners.
         *  @param {Object} attributes optional settings for the panel
@@ -60,9 +57,6 @@ var HistoryView = _super.extend(
            // control contents/behavior based on where (and in what context) the panel is being used
            /** where should pages from links be displayed? (default to new tab/window) */
            this.linkTarget = attributes.linkTarget || "_blank";
-
-            /** timeout id for detailed fetch of collection counts, etc... */
-            this.detailedFetchTimeoutId = null;
        },

        /** create and return a collection for when none is initially passed */
@@ -77,20 +71,11 @@ var HistoryView = _super.extend(
        freeModel: function() {
            _super.prototype.freeModel.call(this);
            if (this.model) {
-                this.model.clearUpdateTimeout();
+                this.model.stopPolling();
            }
-            this._clearDetailedFetchTimeout();
            return this;
        },

-        /** clear the timeout and the cached timeout id */
-        _clearDetailedFetchTimeout: function() {
-            if (this.detailedFetchTimeoutId) {
-                clearTimeout(this.detailedFetchTimeoutId);
-                this.detailedFetchTimeoutId = null;
-            }
-        },
-
        /** create any event listeners for the panel
         *  @fires: rendered:initial    on the first render
         *  @fires: empty-history       when switching to a history with no contents or creating a new history
@@ -101,13 +86,6 @@ var HistoryView = _super.extend(
                error: function(model, xhr, options, msg, details) {
                    this.errorHandler(model, xhr, options, msg, details);
                },
-                "loading-done": () => {
-                    // after the initial load, decorate with more time consuming fields (like HDCA element_counts)
-                    this.detailedFetchTimeoutId = _.delay(() => {
-                        this.detailedFetchTimeoutId = null;
-                        this.model.contents.fetchCollectionCounts();
-                    }, this.FETCH_COLLECTION_COUNTS_DELAY);
-                },
                "views:ready view:attached view:removed": function(view) {
                    this._renderSelectButton();
                },
@@ -378,17 +356,17 @@ var HistoryView = _super.extend(
        }),

        _clickPrevPage: function(ev) {
-            this.model.clearUpdateTimeout();
+            this.model.stopPolling();
            this.model.contents.fetchPrevPage();
        },

        _clickNextPage: function(ev) {
-            this.model.clearUpdateTimeout();
+            this.model.stopPolling();
            this.model.contents.fetchNextPage();
        },

        _changePageSelect: function(ev) {
-            this.model.clearUpdateTimeout();
+            this.model.stopPolling();
            var page = $(ev.currentTarget).val();
            this.model.contents.fetchPage(page);
        },
@@ -0,0 +1,168 @@
+import * as Backbone from "libs/backbone";
+import AJAX_QUEUE from "utils/ajax-queue";
+
+/** ms between fetches when checking running jobs/datasets for updates */
+var UPDATE_DELAY = 2000;
+var NON_TERMINAL_STATES = ["new", "queued", "running"];
+var ERROR_STATES = ["error", "deleted"];
+/** Fetch state on add or just wait for polling to start. */
+var FETCH_STATE_ON_ADD = false;
+var BATCH_FETCH_STATE = true;
+
+var JobStatesSummary = Backbone.Model.extend({
+    url: function() {
+        return `${Galaxy.root}api/histories/${this.attributes.history_id}/contents/dataset_collections/${
+            this.attributes.collection_id
+        }/jobs_summary`;
+    },
+
+    hasDetails: function() {
+        return this.has("populated_state");
+    },
+
+    new: function() {
+        return !this.hasDetails() || this.get("populated_state") == "new";
+    },
+
+    errored: function() {
+        return this.get("populated_state") === "error" || this.anyWithStates(ERROR_STATES);
+    },
+
+    states: function() {
+        return this.get("states") || {};
+    },
+
+    anyWithState: function(queryState) {
+        return (this.states()[queryState] || 0) > 0;
+    },
+
+    anyWithStates: function(queryStates) {
+        var states = this.states();
+        for (var index in queryStates) {
+            if ((states[queryStates[index]] || 0) > 0) {
+                return true;
+            }
+        }
+        return false;
+    },
+
+    numWithStates: function(queryStates) {
+        var states = this.states();
+        var count = 0;
+        for (var index in queryStates) {
+            count += states[queryStates[index]] || 0;
+        }
+        return count;
+    },
+
+    numInError: function() {
+        return this.numWithStates(ERROR_STATES);
+    },
+
+    running: function() {
+        return this.anyWithState("running");
+    },
+
+    terminal: function() {
+        if (this.new()) {
+            return false;
+        } else {
+            var anyNonTerminal = this.anyWithStates(NON_TERMINAL_STATES);
+            return !anyNonTerminal;
+        }
+    },
+
+    jobCount: function() {
+        var states = this.states();
+        var count = 0;
+        for (var index in states) {
+            count += states[index];
+        }
+        return count;
+    },
+
+    toString: function() {
+        return `JobStatesSummary(id=${this.get("id")})`;
+    }
+});
+
+var JobStatesSummaryCollection = Backbone.Collection.extend({
+    model: JobStatesSummary,
+
+    initialize: function() {
+        if (FETCH_STATE_ON_ADD) {
+            this.on({
+                add: model => model.fetch()
+            });
+        }
+
+        /** cached timeout id for the dataset updater */
+        this.updateTimeoutId = null;
+        // this.checkForUpdates();
+        this.active = true;
+    },
+
+    url: function() {
+        var nonTerminalModels = this.models.filter(model => {
+            return !model.terminal();
+        });
+        var ids = nonTerminalModels
+            .map(summary => {
+                return summary.get("id");
+            })
+            .join(",");
+        var types = nonTerminalModels
+            .map(summary => {
+                return summary.get("model");
+            })
+            .join(",");
+        return `${Galaxy.root}api/histories/${this.historyId}/jobs_summary?ids=${ids}&types=${types}`;
+    },
+
+    monitor: function() {
+        this.clearUpdateTimeout();
+        if (!this.active) {
+            return;
+        }
+
+        var _delayThenMonitorAgain = () => {
+            this.updateTimeoutId = setTimeout(() => {
+                this.monitor();
+            }, UPDATE_DELAY);
+        };
+
+        var nonTerminalModels = this.models.filter(model => {
+            return !model.terminal();
+        });
+
+        if (nonTerminalModels.length > 0 && !BATCH_FETCH_STATE) {
+            // Allow models to fetch their own details.
+            var updateFunctions = nonTerminalModels.map(summary => {
+                return () => {
+                    return summary.fetch();
+                };
+            });
+
+            return new AJAX_QUEUE.AjaxQueue(updateFunctions).done(_delayThenMonitorAgain);
+        } else if (nonTerminalModels.length > 0) {
+            // Batch fetch updated state...
+            this.fetch({ remove: false }).done(_delayThenMonitorAgain);
+        } else {
+            _delayThenMonitorAgain();
+        }
+    },
+
+    /** clear the timeout and the cached timeout id */
+    clearUpdateTimeout: function() {
+        if (this.updateTimeoutId) {
+            clearTimeout(this.updateTimeoutId);
+            this.updateTimeoutId = null;
+        }
+    },
+
+    toString: function() {
+        return `JobStatesSummaryCollection()`;
+    }
+});
+
+export default { JobStatesSummary, JobStatesSummaryCollection, FETCH_STATE_ON_ADD };
@@ -1,6 +1,7 @@
 import _l from "utils/localization";
 import HISTORY_MODEL from "mvc/history/history-model";
 import HISTORY_VIEW_EDIT from "mvc/history/history-view-edit";
+import JOB_STATES_MODEL from "mvc/history/job-states-model";
 import historyCopyDialog from "mvc/history/copy-dialog";
 import ERROR_MODAL from "mvc/ui/error-modal";
 import baseMVC from "mvc/base-mvc";
@@ -733,9 +734,16 @@ var MultiPanelColumns = Backbone.View.extend(baseMVC.LoggableMixin).extend({
            this.hdaQueue.add({
                name: column.model.id,
                fn: function() {
-                    return contents.fetchCurrentPage(fetchOptions).done(() => {
-                        column.panel.renderItems();
-                    });
+                    return contents
+                        .fetchCurrentPage(fetchOptions)
+                        .done(() => {
+                            column.panel.renderItems();
+                        })
+                        .done(() => {
+                            if (!JOB_STATES_MODEL.FETCH_STATE_ON_ADD) {
+                                contents.jobStateSummariesCollection.fetch();
+                            }
+                        });
                }
            });
            // the queue is re-used, so if it's not processing requests - start it again
@@ -1108,6 +1108,37 @@ ul.manage-table-actions li {
    margin-left: 0.5em;
 }

+.state-progress {
+    border: 1px solid gray;
+    position: relative;
+
+    margin-top: 2px;
+    margin-bottom: 1px;
+
+    .info {
+        color: @black;
+        background: @white;
+    }
+
+    .new {
+        background: @state-default-bg;
+    }
+
+    .running {
+        background: @state-running-bg;
+    }
+
+    .ok {
+        background: @state-success-bg;
+    }
+
+    .note {
+        margin-left: 1em;
+        position: absolute;
+    }
+
+}
+
 // State colors

 .state-color-new {
@@ -86,6 +86,13 @@
                color: inherit;
            }
        }
+        .state-description {
+            color: #777;
+            font-size: 90%;
+            a {
+                color: inherit;
+            }
+        }
    }

    .primary-actions {
@@ -8,6 +8,7 @@ from sqlalchemy.orm import aliased
 from sqlalchemy.sql import select

 from galaxy import model
+from galaxy.exceptions import RequestParameterInvalidException
 from galaxy.managers.collections import DatasetCollectionManager
 from galaxy.managers.hdas import HDAManager
 from galaxy.managers.lddas import LDDAManager
@@ -246,6 +247,31 @@ class JobSearch(object):
        return None


+def fetch_job_states(app, sa_session, job_source_ids, job_source_types):
+    decode = app.security.decode_id
+    assert len(job_source_ids) == len(job_source_types)
+    job_ids = set()
+    implicit_collection_job_ids = set()
+
+    for job_source_id, job_source_type in zip(job_source_ids, job_source_types):
+        if job_source_type == "Job":
+            job_ids.add(job_source_id)
+        elif job_source_type == "ImplicitCollectionJobs":
+            implicit_collection_job_ids.add(job_source_id)
+        else:
+            raise RequestParameterInvalidException("Invalid job source type %s found." % job_source_type)
+
+    # TODO: use above sets and optimize queries on second pass.
+    rval = []
+    for job_source_id, job_source_type in zip(job_source_ids, job_source_types):
+        if job_source_type == "Job":
+            rval.append(summarize_jobs_to_dict(sa_session, sa_session.query(model.Job).get(decode(job_source_id))))
+        else:
+            rval.append(summarize_jobs_to_dict(sa_session, sa_session.query(model.ImplicitCollectionJobs).get(decode(job_source_id))))
+
+    return rval
+
+
 def summarize_jobs_to_dict(sa_session, jobs_source):
    """Proudce a summary of jobs for job summary endpoints.

@@ -21,7 +21,7 @@ from galaxy.managers.collections_util import (
    dictify_dataset_collection_instance,
    get_hda_and_element_identifiers
 )
-from galaxy.managers.jobs import summarize_jobs_to_dict
+from galaxy.managers.jobs import fetch_job_states, summarize_jobs_to_dict
 from galaxy.util.json import safe_dumps
 from galaxy.util.streamball import StreamBall
 from galaxy.web import (
@@ -155,12 +155,50 @@ class HistoryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibrary
        elif contents_type == 'dataset_collection':
            return self.__show_dataset_collection(trans, id, history_id, **kwd)

+    @expose_api_anonymous
+    def index_jobs_summary(self, trans, history_id, **kwd):
+        """
+        * GET /api/histories/{history_id}/jobs_summary
+            return detailed information about an HDA or HDCAs jobs
+
+        Warning: We allow anyone to fetch job state information about any object they
+        can guess an encoded ID for - it isn't considered protected data. This keeps
+        polling IDs as part of state calculation for large histories and collections as
+        efficient as possible.
+
+        :type   history_id: str
+        :param  history_id: encoded id string of the HDA's or the HDCA's History
+        :type   ids:        str[]
+        :param  ids:        the encoded ids of job summary objects to return - if ids
+                            is specified types must also be specified and have same length.
+        :type   types:      str[]
+        :param  types:      type of object represented by elements in the ids array - either
+                            Job or ImplicitCollectionJob.
+
+        :rtype:     dict[]
+        :returns:   an array of job summary object dictionaries.
+        """
+        ids = kwd.get("ids", None)
+        types = kwd.get("types", None)
+        if ids is None:
+            assert types is None
+            # TODO: ...
+            pass
+        else:
+            ids = util.listify(ids)
+            types = util.listify(types)
+        return map(lambda s: self.encode_all_ids(trans, s), fetch_job_states(self.app, trans.sa_session, ids, types))
+
    @expose_api_anonymous
    def show_jobs_summary(self, trans, id, history_id, **kwd):
        """
        * GET /api/histories/{history_id}/contents/{type}/{id}/jobs_summary
            return detailed information about an HDA or HDCAs jobs
-        .. note:: Anonymous users are allowed to get their current history contents
+
+        Warning: We allow anyone to fetch job state information about any object they
+        can guess an encoded ID for - it isn't considered protected data. This keeps
+        polling IDs as part of state calculation for large histories and collections as
+        efficient as possible.

        :type   id:         str
        :param  id:         the encoded id of the HDA to return
@@ -168,7 +206,7 @@ class HistoryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibrary
        :param  history_id: encoded id string of the HDA's or the HDCA's History

        :rtype:     dict
-        :returns:   dictionary containing
+        :returns:   dictionary containing jobs summary object
        """
        contents_type = self.__get_contents_type(trans, kwd)
        # At most one of job or implicit_collection_jobs should be found.
@@ -363,6 +363,11 @@ def populate_api_routes(webapp, app):
                          action='download_dataset_collection',
                          conditions=dict(method=["GET"]))

+    webapp.mapper.connect("/api/histories/{history_id}/jobs_summary",
+                          action="index_jobs_summary",
+                          controller='history_contents',
+                          conditions=dict(method=["GET"]))
+
    webapp.mapper.connect("/api/histories/{history_id}/contents/{type:%s}s/{id}/jobs_summary" % "|".join(valid_history_contents_types),
                          action="show_jobs_summary",
                          controller='history_contents',