diff --git a/client/galaxy/scripts/mvc/history/hdca-li.js b/client/galaxy/scripts/mvc/history/hdca-li.js index 7834ef9506b..ec3f6e443fb 100644 --- a/client/galaxy/scripts/mvc/history/hdca-li.js +++ b/client/galaxy/scripts/mvc/history/hdca-li.js @@ -16,10 +16,14 @@ var HDCAListItemView = _super.extend( /** event listeners */ _setUpListeners: function() { _super.prototype._setUpListeners.call(this); + var renderListen = (model, options) => { + this.render(); + }; + if (this.model.jobStatesSummary) { + this.listenTo(this.model.jobStatesSummary, "change", renderListen); + } this.listenTo(this.model, { - "change:tags change:populated change:visible": function(model, options) { - this.render(); - } + "change:tags change:visible change:state": renderListen }); }, @@ -43,13 +47,95 @@ var HDCAListItemView = _super.extend( _swapNewRender: function($newRender) { _super.prototype._swapNewRender.call(this, $newRender); //TODO: model currently has no state - var state = !this.model.get("populated") ? STATES.RUNNING : STATES.OK; - //if( this.model.has( 'state' ) ){ + var state; + var jobStatesSummary = this.model.jobStatesSummary; + if (jobStatesSummary) { + if (jobStatesSummary.new()) { + state = "new"; + } else if (jobStatesSummary.errored()) { + state = "error"; + } else if (jobStatesSummary.terminal()) { + state = "ok"; + } else if (jobStatesSummary.running()) { + state = "running"; + } else { + state = "queued"; + } + } else if (this.model.get("job_source_id")) { + // Initial rendering - polling will fill in more details in a bit. + state = STATES.NEW; + } else { + state = this.model.get("populated_state") ? STATES.OK : STATES.RUNNING; + } this.$el.addClass(`state-${state}`); - //} + var stateDescription = this.stateDescription(); + this.$(".state-description").html(stateDescription); return this.$el; }, + stateDescription: function() { + var collection = this.model; + var elementCount = collection.get("element_count"); + var jobStateSource = collection.get("job_source_type"); + var collectionType = this.model.get("collection_type"); + var collectionTypeDescription; + if (collectionType == "list") { + collectionTypeDescription = "list"; + } else if (collectionType == "paired") { + collectionTypeDescription = "dataset pair"; + } else if (collectionType == "list:paired") { + collectionTypeDescription = "list of pairs"; + } else { + collectionTypeDescription = "nested list"; + } + var itemsDescription = ""; + if (elementCount == 1) { + itemsDescription = ` with 1 item`; + } else if (elementCount) { + itemsDescription = ` with ${elementCount} items`; + } + var jobStatesSummary = collection.jobStatesSummary; + var simpleDescription = `${collectionTypeDescription}${itemsDescription}`; + if (!jobStateSource || jobStateSource == "Job") { + return `a ${simpleDescription}`; + } else if (!jobStatesSummary || !jobStatesSummary.hasDetails()) { + return ` +
+ Loading job data for ${collectionTypeDescription}... +
+
`; + } else { + var isNew = jobStatesSummary.new(); + var jobCount = isNew ? null : jobStatesSummary.jobCount(); + if (isNew) { + return ` +
+ Creating jobs... +
+
`; + } else if (jobStatesSummary.errored()) { + var errorCount = jobStatesSummary.numInError(); + return `a ${collectionTypeDescription} with ${errorCount} / ${jobCount} jobs in error`; + } else if (jobStatesSummary.terminal()) { + return `a ${simpleDescription}`; + } else { + var running = jobStatesSummary.states()["running"] || 0; + var ok = jobStatesSummary.states()["ok"] || 0; + var okPercent = ok / (jobCount * 1.0); + var runningPercent = running / (jobCount * 1.0); + var otherPercent = 1.0 - okPercent - runningPercent; + var jobsStr = jobCount && jobCount > 1 ? `${jobCount} jobs` : `a job`; + return ` +
+ ${jobsStr} generating a ${collectionTypeDescription} +
+
+
+
`; + } + } + }, + // ......................................................................... misc /** String representation */ toString: function() { @@ -69,7 +155,6 @@ HDCAListItemView.prototype.templates = (() => { } }); - // could steal this from hda-base (or use mixed content) var titleBarTemplate = collection => `
@@ -77,7 +162,8 @@ HDCAListItemView.prototype.templates = (() => { ${collection.hid} ${_.escape(collection.name)}
-
+
+
${HISTORY_ITEM_LI.nametagTemplate(collection)}
`; diff --git a/client/galaxy/scripts/mvc/history/history-contents.js b/client/galaxy/scripts/mvc/history/history-contents.js index 045745bf131..c175456fd9e 100644 --- a/client/galaxy/scripts/mvc/history/history-contents.js +++ b/client/galaxy/scripts/mvc/history/history-contents.js @@ -2,6 +2,7 @@ import CONTROLLED_FETCH_COLLECTION from "mvc/base/controlled-fetch-collection"; import HDA_MODEL from "mvc/history/hda-model"; import HDCA_MODEL from "mvc/history/hdca-model"; import HISTORY_PREFS from "mvc/history/history-preferences"; +import JOB_STATES_MODEL from "mvc/history/job-states-model"; import BASE_MVC from "mvc/base-mvc"; import AJAX_QUEUE from "utils/ajax-queue"; @@ -37,6 +38,10 @@ var HistoryContents = _super.extend(BASE_MVC.LoggableMixin).extend({ /** Set up */ initialize: function(models, options) { + this.on({ + "sync add": this.trackJobStates + }); + options = options || {}; _super.prototype.initialize.call(this, models, options); @@ -53,6 +58,29 @@ var HistoryContents = _super.extend(BASE_MVC.LoggableMixin).extend({ this.model.prototype.idAttribute = "type_id"; }, + trackJobStates: function() { + this.each(historyContent => { + if (historyContent.has("job_states_summary")) { + return; + } + + if (historyContent.attributes.history_content_type === "dataset_collection") { + var jobSourceType = historyContent.attributes.job_source_type; + var jobSourceId = historyContent.attributes.job_source_id; + if (jobSourceType) { + this.jobStateSummariesCollection.add({ + id: jobSourceId, + model: jobSourceType, + history_id: this.history_id, + collection_id: historyContent.attributes.id + }); + var jobStatesSummary = this.jobStateSummariesCollection.get(jobSourceId); + historyContent.jobStatesSummary = jobStatesSummary; + } + } + }); + }, + // ........................................................................ composite collection /** since history content is a mix, override model fn into a factory, creating based on history_content_type */ model: function(attrs, options) { @@ -82,17 +110,30 @@ var HistoryContents = _super.extend(BASE_MVC.LoggableMixin).extend({ }; }, + stopPolling: function() { + if (this.jobStateSummariesCollection) { + this.jobStateSummariesCollection.active = false; + this.jobStateSummariesCollection.clearUpdateTimeout(); + } + }, + setHistoryId: function(newId) { + this.stopPolling(); this.historyId = newId; - this._setUpWebStorage(); + if (newId) { + // If actually reflecting a history - setup storage and monitor jobs. + + this._setUpWebStorage(); + + this.jobStateSummariesCollection = new JOB_STATES_MODEL.JobStatesSummaryCollection(); + this.jobStateSummariesCollection.historyId = newId; + this.jobStateSummariesCollection.monitor(); + } }, /** Set up client side storage. Currently PersistanStorage keyed under 'history:' */ _setUpWebStorage: function(initialSettings) { // TODO: use initialSettings - if (!this.historyId) { - return; - } this.storage = new HISTORY_PREFS.HistoryPrefs({ id: HISTORY_PREFS.HistoryPrefs.historyStorageKey(this.historyId) }); @@ -303,17 +344,6 @@ var HistoryContents = _super.extend(BASE_MVC.LoggableMixin).extend({ return this.fetch(options); }, - /** specialty fetch method for retrieving the element_counts of all hdcas in the history */ - fetchCollectionCounts: function(options) { - options = options || {}; - options.keys = ["type_id", "element_count"].join(","); - options.filters = _.extend(options.filters || {}, { - history_content_type: "dataset_collection" - }); - options.remove = false; - return this.fetch(options); - }, - // ............. quasi-batch ops // TODO: to batch /** helper that fetches using filterParams then calls save on each fetched using updateWhat as the save params */ diff --git a/client/galaxy/scripts/mvc/history/history-model.js b/client/galaxy/scripts/mvc/history/history-model.js index 923b04672c5..48d3c5e6ca3 100644 --- a/client/galaxy/scripts/mvc/history/history-model.js +++ b/client/galaxy/scripts/mvc/history/history-model.js @@ -237,6 +237,13 @@ var History = Backbone.Model.extend(BASE_MVC.LoggableMixin).extend( } }, + stopPolling: function() { + this.clearUpdateTimeout(); + if (this.contents) { + this.contents.stopPolling(); + } + }, + // ........................................................................ ajax /** override to use actual Dates objects for create/update times */ parse: function(response, options) { diff --git a/client/galaxy/scripts/mvc/history/history-view.js b/client/galaxy/scripts/mvc/history/history-view.js index 140c2edb990..2cfa71fb99a 100644 --- a/client/galaxy/scripts/mvc/history/history-view.js +++ b/client/galaxy/scripts/mvc/history/history-view.js @@ -47,9 +47,6 @@ var HistoryView = _super.extend( /** string used for search placeholder */ searchPlaceholder: _l("search datasets"), - /** @type {Number} ms to wait after history load to fetch/decorate hdcas with element_count */ - FETCH_COLLECTION_COUNTS_DELAY: 2000, - // ......................................................................... SET UP /** Set up the view, bind listeners. * @param {Object} attributes optional settings for the panel @@ -60,9 +57,6 @@ var HistoryView = _super.extend( // control contents/behavior based on where (and in what context) the panel is being used /** where should pages from links be displayed? (default to new tab/window) */ this.linkTarget = attributes.linkTarget || "_blank"; - - /** timeout id for detailed fetch of collection counts, etc... */ - this.detailedFetchTimeoutId = null; }, /** create and return a collection for when none is initially passed */ @@ -77,20 +71,11 @@ var HistoryView = _super.extend( freeModel: function() { _super.prototype.freeModel.call(this); if (this.model) { - this.model.clearUpdateTimeout(); + this.model.stopPolling(); } - this._clearDetailedFetchTimeout(); return this; }, - /** clear the timeout and the cached timeout id */ - _clearDetailedFetchTimeout: function() { - if (this.detailedFetchTimeoutId) { - clearTimeout(this.detailedFetchTimeoutId); - this.detailedFetchTimeoutId = null; - } - }, - /** create any event listeners for the panel * @fires: rendered:initial on the first render * @fires: empty-history when switching to a history with no contents or creating a new history @@ -101,13 +86,6 @@ var HistoryView = _super.extend( error: function(model, xhr, options, msg, details) { this.errorHandler(model, xhr, options, msg, details); }, - "loading-done": () => { - // after the initial load, decorate with more time consuming fields (like HDCA element_counts) - this.detailedFetchTimeoutId = _.delay(() => { - this.detailedFetchTimeoutId = null; - this.model.contents.fetchCollectionCounts(); - }, this.FETCH_COLLECTION_COUNTS_DELAY); - }, "views:ready view:attached view:removed": function(view) { this._renderSelectButton(); }, @@ -378,17 +356,17 @@ var HistoryView = _super.extend( }), _clickPrevPage: function(ev) { - this.model.clearUpdateTimeout(); + this.model.stopPolling(); this.model.contents.fetchPrevPage(); }, _clickNextPage: function(ev) { - this.model.clearUpdateTimeout(); + this.model.stopPolling(); this.model.contents.fetchNextPage(); }, _changePageSelect: function(ev) { - this.model.clearUpdateTimeout(); + this.model.stopPolling(); var page = $(ev.currentTarget).val(); this.model.contents.fetchPage(page); }, diff --git a/client/galaxy/scripts/mvc/history/job-states-model.js b/client/galaxy/scripts/mvc/history/job-states-model.js new file mode 100644 index 00000000000..fffee020ead --- /dev/null +++ b/client/galaxy/scripts/mvc/history/job-states-model.js @@ -0,0 +1,168 @@ +import * as Backbone from "libs/backbone"; +import AJAX_QUEUE from "utils/ajax-queue"; + +/** ms between fetches when checking running jobs/datasets for updates */ +var UPDATE_DELAY = 2000; +var NON_TERMINAL_STATES = ["new", "queued", "running"]; +var ERROR_STATES = ["error", "deleted"]; +/** Fetch state on add or just wait for polling to start. */ +var FETCH_STATE_ON_ADD = false; +var BATCH_FETCH_STATE = true; + +var JobStatesSummary = Backbone.Model.extend({ + url: function() { + return `${Galaxy.root}api/histories/${this.attributes.history_id}/contents/dataset_collections/${ + this.attributes.collection_id + }/jobs_summary`; + }, + + hasDetails: function() { + return this.has("populated_state"); + }, + + new: function() { + return !this.hasDetails() || this.get("populated_state") == "new"; + }, + + errored: function() { + return this.get("populated_state") === "error" || this.anyWithStates(ERROR_STATES); + }, + + states: function() { + return this.get("states") || {}; + }, + + anyWithState: function(queryState) { + return (this.states()[queryState] || 0) > 0; + }, + + anyWithStates: function(queryStates) { + var states = this.states(); + for (var index in queryStates) { + if ((states[queryStates[index]] || 0) > 0) { + return true; + } + } + return false; + }, + + numWithStates: function(queryStates) { + var states = this.states(); + var count = 0; + for (var index in queryStates) { + count += states[queryStates[index]] || 0; + } + return count; + }, + + numInError: function() { + return this.numWithStates(ERROR_STATES); + }, + + running: function() { + return this.anyWithState("running"); + }, + + terminal: function() { + if (this.new()) { + return false; + } else { + var anyNonTerminal = this.anyWithStates(NON_TERMINAL_STATES); + return !anyNonTerminal; + } + }, + + jobCount: function() { + var states = this.states(); + var count = 0; + for (var index in states) { + count += states[index]; + } + return count; + }, + + toString: function() { + return `JobStatesSummary(id=${this.get("id")})`; + } +}); + +var JobStatesSummaryCollection = Backbone.Collection.extend({ + model: JobStatesSummary, + + initialize: function() { + if (FETCH_STATE_ON_ADD) { + this.on({ + add: model => model.fetch() + }); + } + + /** cached timeout id for the dataset updater */ + this.updateTimeoutId = null; + // this.checkForUpdates(); + this.active = true; + }, + + url: function() { + var nonTerminalModels = this.models.filter(model => { + return !model.terminal(); + }); + var ids = nonTerminalModels + .map(summary => { + return summary.get("id"); + }) + .join(","); + var types = nonTerminalModels + .map(summary => { + return summary.get("model"); + }) + .join(","); + return `${Galaxy.root}api/histories/${this.historyId}/jobs_summary?ids=${ids}&types=${types}`; + }, + + monitor: function() { + this.clearUpdateTimeout(); + if (!this.active) { + return; + } + + var _delayThenMonitorAgain = () => { + this.updateTimeoutId = setTimeout(() => { + this.monitor(); + }, UPDATE_DELAY); + }; + + var nonTerminalModels = this.models.filter(model => { + return !model.terminal(); + }); + + if (nonTerminalModels.length > 0 && !BATCH_FETCH_STATE) { + // Allow models to fetch their own details. + var updateFunctions = nonTerminalModels.map(summary => { + return () => { + return summary.fetch(); + }; + }); + + return new AJAX_QUEUE.AjaxQueue(updateFunctions).done(_delayThenMonitorAgain); + } else if (nonTerminalModels.length > 0) { + // Batch fetch updated state... + this.fetch({ remove: false }).done(_delayThenMonitorAgain); + } else { + _delayThenMonitorAgain(); + } + }, + + /** clear the timeout and the cached timeout id */ + clearUpdateTimeout: function() { + if (this.updateTimeoutId) { + clearTimeout(this.updateTimeoutId); + this.updateTimeoutId = null; + } + }, + + toString: function() { + return `JobStatesSummaryCollection()`; + } +}); + +export default { JobStatesSummary, JobStatesSummaryCollection, FETCH_STATE_ON_ADD }; diff --git a/client/galaxy/scripts/mvc/history/multi-panel.js b/client/galaxy/scripts/mvc/history/multi-panel.js index 91e81ed22cd..2652d7c243c 100644 --- a/client/galaxy/scripts/mvc/history/multi-panel.js +++ b/client/galaxy/scripts/mvc/history/multi-panel.js @@ -1,6 +1,7 @@ import _l from "utils/localization"; import HISTORY_MODEL from "mvc/history/history-model"; import HISTORY_VIEW_EDIT from "mvc/history/history-view-edit"; +import JOB_STATES_MODEL from "mvc/history/job-states-model"; import historyCopyDialog from "mvc/history/copy-dialog"; import ERROR_MODAL from "mvc/ui/error-modal"; import baseMVC from "mvc/base-mvc"; @@ -733,9 +734,16 @@ var MultiPanelColumns = Backbone.View.extend(baseMVC.LoggableMixin).extend({ this.hdaQueue.add({ name: column.model.id, fn: function() { - return contents.fetchCurrentPage(fetchOptions).done(() => { - column.panel.renderItems(); - }); + return contents + .fetchCurrentPage(fetchOptions) + .done(() => { + column.panel.renderItems(); + }) + .done(() => { + if (!JOB_STATES_MODEL.FETCH_STATE_ON_ADD) { + contents.jobStateSummariesCollection.fetch(); + } + }); } }); // the queue is re-used, so if it's not processing requests - start it again diff --git a/client/galaxy/style/less/base.less b/client/galaxy/style/less/base.less index c04ac237568..f5ba8069140 100644 --- a/client/galaxy/style/less/base.less +++ b/client/galaxy/style/less/base.less @@ -1108,6 +1108,37 @@ ul.manage-table-actions li { margin-left: 0.5em; } +.state-progress { + border: 1px solid gray; + position: relative; + + margin-top: 2px; + margin-bottom: 1px; + + .info { + color: @black; + background: @white; + } + + .new { + background: @state-default-bg; + } + + .running { + background: @state-running-bg; + } + + .ok { + background: @state-success-bg; + } + + .note { + margin-left: 1em; + position: absolute; + } + +} + // State colors .state-color-new { diff --git a/client/galaxy/style/less/list-item.less b/client/galaxy/style/less/list-item.less index ca5f61b487f..36abdaea718 100644 --- a/client/galaxy/style/less/list-item.less +++ b/client/galaxy/style/less/list-item.less @@ -86,6 +86,13 @@ color: inherit; } } + .state-description { + color: #777; + font-size: 90%; + a { + color: inherit; + } + } } .primary-actions { diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py index b967f897c9e..b17fa922f1b 100644 --- a/lib/galaxy/managers/jobs.py +++ b/lib/galaxy/managers/jobs.py @@ -8,6 +8,7 @@ from sqlalchemy.orm import aliased from sqlalchemy.sql import select from galaxy import model +from galaxy.exceptions import RequestParameterInvalidException from galaxy.managers.collections import DatasetCollectionManager from galaxy.managers.hdas import HDAManager from galaxy.managers.lddas import LDDAManager @@ -246,6 +247,31 @@ class JobSearch(object): return None +def fetch_job_states(app, sa_session, job_source_ids, job_source_types): + decode = app.security.decode_id + assert len(job_source_ids) == len(job_source_types) + job_ids = set() + implicit_collection_job_ids = set() + + for job_source_id, job_source_type in zip(job_source_ids, job_source_types): + if job_source_type == "Job": + job_ids.add(job_source_id) + elif job_source_type == "ImplicitCollectionJobs": + implicit_collection_job_ids.add(job_source_id) + else: + raise RequestParameterInvalidException("Invalid job source type %s found." % job_source_type) + + # TODO: use above sets and optimize queries on second pass. + rval = [] + for job_source_id, job_source_type in zip(job_source_ids, job_source_types): + if job_source_type == "Job": + rval.append(summarize_jobs_to_dict(sa_session, sa_session.query(model.Job).get(decode(job_source_id)))) + else: + rval.append(summarize_jobs_to_dict(sa_session, sa_session.query(model.ImplicitCollectionJobs).get(decode(job_source_id)))) + + return rval + + def summarize_jobs_to_dict(sa_session, jobs_source): """Proudce a summary of jobs for job summary endpoints. diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index f6fc932b5a0..357be202da3 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -21,7 +21,7 @@ from galaxy.managers.collections_util import ( dictify_dataset_collection_instance, get_hda_and_element_identifiers ) -from galaxy.managers.jobs import summarize_jobs_to_dict +from galaxy.managers.jobs import fetch_job_states, summarize_jobs_to_dict from galaxy.util.json import safe_dumps from galaxy.util.streamball import StreamBall from galaxy.web import ( @@ -155,12 +155,50 @@ class HistoryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibrary elif contents_type == 'dataset_collection': return self.__show_dataset_collection(trans, id, history_id, **kwd) + @expose_api_anonymous + def index_jobs_summary(self, trans, history_id, **kwd): + """ + * GET /api/histories/{history_id}/jobs_summary + return detailed information about an HDA or HDCAs jobs + + Warning: We allow anyone to fetch job state information about any object they + can guess an encoded ID for - it isn't considered protected data. This keeps + polling IDs as part of state calculation for large histories and collections as + efficient as possible. + + :type history_id: str + :param history_id: encoded id string of the HDA's or the HDCA's History + :type ids: str[] + :param ids: the encoded ids of job summary objects to return - if ids + is specified types must also be specified and have same length. + :type types: str[] + :param types: type of object represented by elements in the ids array - either + Job or ImplicitCollectionJob. + + :rtype: dict[] + :returns: an array of job summary object dictionaries. + """ + ids = kwd.get("ids", None) + types = kwd.get("types", None) + if ids is None: + assert types is None + # TODO: ... + pass + else: + ids = util.listify(ids) + types = util.listify(types) + return map(lambda s: self.encode_all_ids(trans, s), fetch_job_states(self.app, trans.sa_session, ids, types)) + @expose_api_anonymous def show_jobs_summary(self, trans, id, history_id, **kwd): """ * GET /api/histories/{history_id}/contents/{type}/{id}/jobs_summary return detailed information about an HDA or HDCAs jobs - .. note:: Anonymous users are allowed to get their current history contents + + Warning: We allow anyone to fetch job state information about any object they + can guess an encoded ID for - it isn't considered protected data. This keeps + polling IDs as part of state calculation for large histories and collections as + efficient as possible. :type id: str :param id: the encoded id of the HDA to return @@ -168,7 +206,7 @@ class HistoryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibrary :param history_id: encoded id string of the HDA's or the HDCA's History :rtype: dict - :returns: dictionary containing + :returns: dictionary containing jobs summary object """ contents_type = self.__get_contents_type(trans, kwd) # At most one of job or implicit_collection_jobs should be found. diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index cab77e6cfb4..04f56c0fd5f 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -363,6 +363,11 @@ def populate_api_routes(webapp, app): action='download_dataset_collection', conditions=dict(method=["GET"])) + webapp.mapper.connect("/api/histories/{history_id}/jobs_summary", + action="index_jobs_summary", + controller='history_contents', + conditions=dict(method=["GET"])) + webapp.mapper.connect("/api/histories/{history_id}/contents/{type:%s}s/{id}/jobs_summary" % "|".join(valid_history_contents_types), action="show_jobs_summary", controller='history_contents',