diff --git a/Makefile b/Makefile index d421c11a3b6..157a3e5c08a 100644 --- a/Makefile +++ b/Makefile @@ -166,10 +166,6 @@ client-test: client ## Run qunit tests via Karma client-test-watch: client ## Watch and run qunit tests on changes via Karma cd client && yarn run test-watch -charts: node-deps ## Rebuild charts - cd client && yarn run build-charts - - # Release Targets release-create-rc: release-ensure-upstream ## Create a release-candidate branch git checkout dev diff --git a/client/galaxy/scripts/apps/analysis.js b/client/galaxy/scripts/apps/analysis.js index 96eb7b6b452..1a4f6af1dac 100644 --- a/client/galaxy/scripts/apps/analysis.js +++ b/client/galaxy/scripts/apps/analysis.js @@ -101,15 +101,20 @@ window.app = function app(options, bootstrapped) { var model = new UserPreferences.Model({ user_id: Galaxy.params.id }); - this.page.display(new FormWrapper.View(model.get(form_id))); + this.page.display(new FormWrapper.View(_.extend( + model.get(form_id), + {active_tab: "user"} + ))); }, show_visualizations: function(action_id) { + var activeTab = action_id=="list_published"?"shared":"visualization"; this.page.display( new GridShared.View({ action_id: action_id, plural: "Visualizations", - item: "visualization" + item: "visualization", + active_tab: activeTab }) ); }, @@ -118,7 +123,8 @@ window.app = function app(options, bootstrapped) { this.page.display( new FormWrapper.View({ url: `visualization/edit?id=${QueryStringParsing.get("id")}`, - redirect: "visualizations/list" + redirect: "visualizations/list", + active_tab: "visualization" }) ); }, @@ -201,11 +207,13 @@ window.app = function app(options, bootstrapped) { }, show_pages: function(action_id) { + var activeTab = action_id=="list_published"?"shared":"user"; this.page.display( new GridShared.View({ action_id: action_id, plural: "Pages", - item: "page" + item: "page", + active_tab: activeTab }) ); }, @@ -214,7 +222,8 @@ window.app = function app(options, bootstrapped) { this.page.display( new FormWrapper.View({ url: "page/create", - redirect: "pages/list" + redirect: "pages/list", + active_tab: "user" }) ); }, @@ -223,7 +232,8 @@ window.app = function app(options, bootstrapped) { this.page.display( new FormWrapper.View({ url: `page/edit?id=${QueryStringParsing.get("id")}`, - redirect: "pages/list" + redirect: "pages/list", + active_tab: "user" }) ); }, @@ -243,7 +253,8 @@ window.app = function app(options, bootstrapped) { this.page.display( new FormWrapper.View({ url: `workflow/create`, - redirect: "workflow/editor" + redirect: "workflow/editor", + active_tab: "workflow" }) ); }, @@ -319,14 +330,18 @@ window.app = function app(options, bootstrapped) { Utils.get({ url: `${Galaxy.root}api/workflows/${Utils.getQueryString("id")}/download?style=run`, success: response => { - this.page.display(new ToolFormComposite.View(response)); + this.page.display(new ToolFormComposite.View(_.extend( + response, + {active_tab: "workflow"} + ))); }, error: response => { var error_msg = response.err_msg || "Error occurred while loading the resource."; var options = { message: error_msg, status: "danger", - persistent: true + persistent: true, + active_tab: "workflow" }; this.page.display(new Ui.Message(options)); } diff --git a/client/galaxy/scripts/layout/menu.js b/client/galaxy/scripts/layout/menu.js index 0171d756e5a..e79aad3b05b 100644 --- a/client/galaxy/scripts/layout/menu.js +++ b/client/galaxy/scripts/layout/menu.js @@ -343,11 +343,13 @@ var Tab = Backbone.View.extend({ .attr("title", this.model.get("tooltip")) .tooltip("destroy"); this.model.get("tooltip") && this.$toggle.tooltip({ placement: "bottom" }); - this.$dropdown - .removeClass() - .addClass("dropdown") - .addClass(this.model.get("disabled") && "disabled") - .addClass(this.model.get("active") && "active"); + if(!this.model.get("menu")){ + this.$dropdown + .removeClass() + .addClass("dropdown") + .addClass(this.model.get("disabled") && "disabled") + .addClass(this.model.get("active") && "active"); + }; if (this.model.get("menu") && this.model.get("show_menu")) { this.$menu.show(); $("#dd-helper") diff --git a/client/galaxy/scripts/mvc/collection/collection-model.js b/client/galaxy/scripts/mvc/collection/collection-model.js index fdb00c1760d..c3b0e692dc1 100644 --- a/client/galaxy/scripts/mvc/collection/collection-model.js +++ b/client/galaxy/scripts/mvc/collection/collection-model.js @@ -60,10 +60,17 @@ var DatasetCollectionElementMixin = { /** merge the attributes of the sub-object 'object' into this model */ _mergeObject: function(attributes) { - // if we don't preserve and correct ids here, the element id becomes the object id - // and collision in backbone's _byId will occur and only + // Don't let the dataset ID replace the DCE's ID so record it as the + // element_id and when fetching dataset details below use the element_id + // instead of this.id. + const object = attributes.object; + let elementId = this.elementId; + if (object) { + elementId = attributes.object.id; + delete attributes.object.id; + } _.extend(attributes, attributes.object, { - element_id: attributes.id + element_id: elementId }); delete attributes.object; return attributes; @@ -123,7 +130,16 @@ var DatasetDCE = DATASET_MODEL.DatasetAssociation.extend( // (a little silly since this api endpoint *also* points at hdas) return `${Galaxy.root}api/datasets`; } - return `${Galaxy.root}api/histories/${this.get("history_id")}/contents/${this.get("id")}`; + const datasetId = this._getDatasetId(); + const url = `${Galaxy.root}api/histories/${this.get("history_id")}/contents/${datasetId}`; + return url; + }, + + _getDatasetId: function() { + // I'm a DCE acting as dataset, this URL needs to be the dataset URL so + // use element_id instead of id. See note above in _mergeObject and + // discussion on #3782 for more context. + return this.get("element_id"); }, defaults: _.extend( diff --git a/client/galaxy/scripts/mvc/collection/collection-view.js b/client/galaxy/scripts/mvc/collection/collection-view.js index 2b7de39e3c3..76d8e079cae 100644 --- a/client/galaxy/scripts/mvc/collection/collection-view.js +++ b/client/galaxy/scripts/mvc/collection/collection-view.js @@ -32,7 +32,7 @@ var CollectionView = _super.extend( initialize: function(attributes) { _super.prototype.initialize.call(this, attributes); this.linkTarget = attributes.linkTarget || "_blank"; - + this.dragItems = true; this.hasUser = attributes.hasUser; /** A stack of panels that currently cover or hide this panel */ this.panelStack = []; diff --git a/client/galaxy/scripts/mvc/dataset/dataset-edit-attributes.js b/client/galaxy/scripts/mvc/dataset/dataset-edit-attributes.js index 4448b5809f4..adbb78fe1be 100644 --- a/client/galaxy/scripts/mvc/dataset/dataset-edit-attributes.js +++ b/client/galaxy/scripts/mvc/dataset/dataset-edit-attributes.js @@ -12,6 +12,7 @@ var View = Backbone.View.extend({ }); this.message = new Ui.Message({ persistent: true }); this.tabs = this._createTabs(); + this.active_tab = "user"; this.$el .append($("

").append("Edit dataset attributes")) .append(this.message.$el) diff --git a/client/galaxy/scripts/mvc/dataset/dataset-error.js b/client/galaxy/scripts/mvc/dataset/dataset-error.js index b52019cd3e3..74f735eab7e 100644 --- a/client/galaxy/scripts/mvc/dataset/dataset-error.js +++ b/client/galaxy/scripts/mvc/dataset/dataset-error.js @@ -9,6 +9,7 @@ var View = Backbone.View.extend({ this.model = new Backbone.Model({ dataset_id: Galaxy.params.dataset_id }); + this.active_tab = "user"; this.render(); }, diff --git a/client/galaxy/scripts/mvc/dataset/dataset-model.js b/client/galaxy/scripts/mvc/dataset/dataset-model.js index cda92de5348..2e14af512d8 100644 --- a/client/galaxy/scripts/mvc/dataset/dataset-model.js +++ b/client/galaxy/scripts/mvc/dataset/dataset-model.js @@ -54,9 +54,13 @@ var DatasetAssociation = Backbone.Model.extend(BASE_MVC.LoggableMixin).extend( this._setUpListeners(); }, + _getDatasetId: function() { + return this.get("id"); + }, + /** returns misc. web urls for rendering things like re-run, display, etc. */ _generateUrls: function() { - var id = this.get("id"); + const id = this._getDatasetId(); if (!id) { return {}; } diff --git a/client/galaxy/scripts/mvc/form/form-wrapper.js b/client/galaxy/scripts/mvc/form/form-wrapper.js index 07ea81de971..49247a8f8c0 100644 --- a/client/galaxy/scripts/mvc/form/form-wrapper.js +++ b/client/galaxy/scripts/mvc/form/form-wrapper.js @@ -6,6 +6,9 @@ var View = Backbone.View.extend({ this.model = new Backbone.Model(options); this.url = this.model.get("url"); this.redirect = this.model.get("redirect"); + if (options && options.active_tab){ + this.active_tab = options.active_tab; + } this.setElement("
"); this.render(); }, diff --git a/client/galaxy/scripts/mvc/grid/grid-shared.js b/client/galaxy/scripts/mvc/grid/grid-shared.js index c9dbb27a0cc..170ad392ba6 100644 --- a/client/galaxy/scripts/mvc/grid/grid-shared.js +++ b/client/galaxy/scripts/mvc/grid/grid-shared.js @@ -9,6 +9,9 @@ var View = Backbone.View.extend({ this.model = new Backbone.Model(options); this.item = this.model.get("item"); this.title = this.model.get("plural"); + if (options && options.active_tab){ + this.active_tab = options.active_tab; + } $.ajax({ url: `${Galaxy.root + this.item}/${this.model.get("action_id")}?${$.param(Galaxy.params)}`, success: function(response) { diff --git a/client/galaxy/scripts/mvc/history/history-list.js b/client/galaxy/scripts/mvc/history/history-list.js index d50cf00033c..4a470694915 100644 --- a/client/galaxy/scripts/mvc/history/history-list.js +++ b/client/galaxy/scripts/mvc/history/history-list.js @@ -107,6 +107,12 @@ var View = Backbone.View.extend({ var self = this; LoadingIndicator.markViewAsLoading(this); + if(options.action_id == "list_published"){ + this.active_tab = "shared"; + } + else if (options.action_id = "list"){ + this.active_tab = "user"; + } this.model = new Backbone.Model(); Utils.get({ url: `${Galaxy.root}history/${options.action_id}?${$.param(Galaxy.params)}`, diff --git a/client/galaxy/scripts/mvc/tool/tool-form-composite.js b/client/galaxy/scripts/mvc/tool/tool-form-composite.js index 01d1a50dee6..a2123487dcc 100644 --- a/client/galaxy/scripts/mvc/tool/tool-form-composite.js +++ b/client/galaxy/scripts/mvc/tool/tool-form-composite.js @@ -15,6 +15,9 @@ var View = Backbone.View.extend({ this.modal = parent.Galaxy.modal || new Modal.View(); this.model = (options && options.model) || new Backbone.Model(options); this.deferred = new Deferred(); + if (options && options.active_tab){ + this.active_tab = options.active_tab; + } this.setElement( $("
") .addClass("ui-form-composite") @@ -189,6 +192,7 @@ var View = Backbone.View.extend({ this._renderParameters(); this._renderHistory(); this._renderUseCachedJob(); + this._renderResourceParameters(); _.each(this.steps, step => { self._renderStep(step); }); @@ -299,6 +303,19 @@ var View = Backbone.View.extend({ }); this._append(this.$steps, this.history_form.$el); }, + + /** Render Workflow Options */ + _renderResourceParameters: function() { + this.workflow_resource_parameters_form = null; + if(!_.isEmpty(this.model.get('workflow_resource_parameters'))){ + this.workflow_resource_parameters_form = new Form({ + cls : 'ui-portlet-narrow', + title : 'Workflow Resource Options', + inputs : this.model.get('workflow_resource_parameters') + }); + this._append( this.$steps, this.workflow_resource_parameters_form.$el ); + } + }, /** Render job caching option */ _renderUseCachedJob: function() { @@ -521,6 +538,7 @@ var View = Backbone.View.extend({ var job_def = { new_history_name: history_form_data["new_history|name"] ? history_form_data["new_history|name"] : null, history_id: !history_form_data["new_history|name"] ? this.model.get("history_id") : null, + resource_params: this.workflow_resource_parameters_form ? this.workflow_resource_parameters_form.data.create() : {}, replacement_params: this.wp_form ? this.wp_form.data.create() : {}, parameters: {}, // Tool form will submit flat maps for each parameter diff --git a/client/galaxy/scripts/mvc/ui/ui-misc.js b/client/galaxy/scripts/mvc/ui/ui-misc.js index 7ad33d347c3..c5ee45392b8 100644 --- a/client/galaxy/scripts/mvc/ui/ui-misc.js +++ b/client/galaxy/scripts/mvc/ui/ui-misc.js @@ -47,6 +47,9 @@ export var Message = Backbone.View.extend({ fade: true }).set(options); this.listenTo(this.model, "change", this.render, this); + if (options && options.active_tab){ + this.active_tab = options.active_tab; + } this.render(); }, update: function(options) { diff --git a/client/galaxy/scripts/mvc/ui/ui-select-content.js b/client/galaxy/scripts/mvc/ui/ui-select-content.js index 16f919aa750..c963ca3e1ef 100644 --- a/client/galaxy/scripts/mvc/ui/ui-select-content.js +++ b/client/galaxy/scripts/mvc/ui/ui-select-content.js @@ -414,9 +414,20 @@ var View = Backbone.View.extend({ var field = this.fields[current]; var drop_data = JSON.parse(ev.originalEvent.dataTransfer.getData("text"))[0]; var new_id = drop_data.id; - var new_src = drop_data.history_content_type == "dataset" ? "hda" : "hdca"; + var new_src = drop_data.history_content_type == "dataset_collection" ? "hdca" : "hda"; var new_value = { id: new_id, src: new_src }; - if (data && _.findWhere(data[new_src], new_value)) { + if (data && drop_data.history_id) { + if (!_.findWhere(data[new_src], new_value)) { + data[new_src].push({ + id: new_id, + src: new_src, + hid: drop_data.hid || "Dropped", + name: drop_data.hid ? drop_data.name : new_id, + keep: true, + tags: [] + }); + this._changeData(); + } if (config.src == new_src) { var current_value = field.value(); if (current_value && config.multiple) { diff --git a/client/galaxy/scripts/mvc/upload/default/default-row.js b/client/galaxy/scripts/mvc/upload/default/default-row.js index 054c2cba775..b6d1e79751c 100644 --- a/client/galaxy/scripts/mvc/upload/default/default-row.js +++ b/client/galaxy/scripts/mvc/upload/default/default-row.js @@ -19,6 +19,7 @@ export default Backbone.View.extend({ initialize: function(app, options) { var self = this; this.app = app; + this.list_extensions = app.list_extensions; this.model = options.model; this.setElement(this._template(options.model)); this.$mode = this.$(".upload-mode"); @@ -58,7 +59,7 @@ export default Backbone.View.extend({ // create select extension this.select_extension = new Select.View({ css: "upload-extension", - data: self.app.list_extensions, + data: _.filter(this.list_extensions, ext => !ext.composite_files), container: this.$(".upload-extension"), value: default_extension, onchange: function(extension) { diff --git a/client/galaxy/scripts/mvc/upload/upload-view.js b/client/galaxy/scripts/mvc/upload/upload-view.js index 5f4684f53e7..ec9aaaeacd2 100644 --- a/client/galaxy/scripts/mvc/upload/upload-view.js +++ b/client/galaxy/scripts/mvc/upload/upload-view.js @@ -169,7 +169,9 @@ export default Backbone.View.extend({ var inputs = { file_count: items.length, dbkey: items[0].get("genome", "?"), - file_type: items[0].get("extension", "auto") + // sometimes extension set to "" in automated testing after first upload of + // a session. https://github.com/galaxyproject/galaxy/issues/5169 + file_type: items[0].get("extension") || "auto" }; for (var index in items) { var it = items[index]; diff --git a/client/galaxy/scripts/mvc/user/user-custom-builds.js b/client/galaxy/scripts/mvc/user/user-custom-builds.js index b7a5d731981..d3eb594be52 100644 --- a/client/galaxy/scripts/mvc/user/user-custom-builds.js +++ b/client/galaxy/scripts/mvc/user/user-custom-builds.js @@ -15,6 +15,7 @@ var Collection = Backbone.Collection.extend({ var View = Backbone.View.extend({ initialize: function(options) { var self = this; + this.active_tab = "user"; var history_id = Galaxy.currHistoryPanel && Galaxy.currHistoryPanel.model.id; this.model = new Backbone.Model(); this.model.url = `${Galaxy.root}api/histories/${history_id}/custom_builds_metadata`; diff --git a/client/galaxy/scripts/mvc/user/user-preferences.js b/client/galaxy/scripts/mvc/user/user-preferences.js index ea92b5dfb50..3923d9f0b3c 100644 --- a/client/galaxy/scripts/mvc/user/user-preferences.js +++ b/client/galaxy/scripts/mvc/user/user-preferences.js @@ -101,6 +101,7 @@ var Model = Backbone.Model.extend({ /** View of the main user preference panel with links to individual user forms */ var View = Backbone.View.extend({ title: _l("User Preferences"), + active_tab: "user", initialize: function() { this.model = new Model(); this.setElement("
"); diff --git a/client/galaxy/scripts/mvc/visualization/visualization-model.js b/client/galaxy/scripts/mvc/visualization/visualization-model.js index 40b56109554..4c246b71db1 100644 --- a/client/galaxy/scripts/mvc/visualization/visualization-model.js +++ b/client/galaxy/scripts/mvc/visualization/visualization-model.js @@ -1,4 +1,8 @@ -import * as Backbone from "libs/backbone"; +import * as Backbone from "backbone"; +import * as _ from "underscore"; + +/* global Galaxy */ + //============================================================================== /** @class Model for a saved Galaxy visualization. * diff --git a/client/galaxy/scripts/mvc/workflow/workflow.js b/client/galaxy/scripts/mvc/workflow/workflow.js index c81fb9c36b0..7e1d699a2b9 100644 --- a/client/galaxy/scripts/mvc/workflow/workflow.js +++ b/client/galaxy/scripts/mvc/workflow/workflow.js @@ -346,6 +346,7 @@ const WorkflowListView = Backbone.View.extend({ const ImportWorkflowView = Backbone.View.extend({ initialize: function() { this.setElement("
"); + this.active_tab = "workflow"; this.render(); }, diff --git a/client/galaxy/scripts/ui/pagination.js b/client/galaxy/scripts/ui/pagination.js index 87176338a2a..f3fa2cc94ed 100644 --- a/client/galaxy/scripts/ui/pagination.js +++ b/client/galaxy/scripts/ui/pagination.js @@ -1,13 +1,16 @@ -import jQuery from "jquery"; -("use_strict"); +//import $ from "jquery"; +// TODO: This manipulates whatever jquery is available -- needs restructuring, +// or removal (jquery plugins are a bad design choice for us at this point) +// It is *only* used in the scatterplot viz, so this is safe. +/* global $ */ -var $ = jQuery; /** Builds (twitter bootstrap styled) pagination controls. * If the totalDataSize is not null, a horizontal list of page buttons is displayed. * If totalDataSize is null, two links ('Prev' and 'Next) are displayed. * When pages are changed, a 'pagination.page-change' event is fired * sending the event and the (0-based) page requested. */ + function Pagination(element, options) { /** the total number of pages */ this.numPages = null; @@ -37,7 +40,7 @@ Pagination.prototype.defaults = { Pagination.prototype.init = function _init($element, options) { options = options || {}; this.$element = $element; - this.options = jQuery.extend(true, {}, this.defaults, options); + this.options = $.extend(true, {}, this.defaults, options); this.currPage = this.options.startingPage; if (this.options.totalDataSize !== null) { @@ -203,12 +206,12 @@ Pagination.create = function _create($element, options) { }; // as jq plugin -jQuery.fn.extend({ +$.fn.extend({ pagination: function $pagination(options) { - var nonOptionsArgs = jQuery.makeArray(arguments).slice(1); + var nonOptionsArgs = $.makeArray(arguments).slice(1); // if passed an object - use that as an options map to create pagination for each selected - if (jQuery.type(options) === "object") { + if ($.type(options) === "object") { return this.map(function() { Pagination.create($(this), options); return this; @@ -222,9 +225,9 @@ jQuery.fn.extend({ // if a pagination control was found for this element, either... if (previousControl) { // invoke a function on the pagination object if passed a string (the function name) - if (jQuery.type(options) === "string") { + if ($.type(options) === "string") { var fn = previousControl[options]; - if (jQuery.type(fn) === "function") { + if ($.type(fn) === "function") { return fn.apply(previousControl, nonOptionsArgs); } diff --git a/client/galaxy/scripts/ui/peek-column-selector.js b/client/galaxy/scripts/ui/peek-column-selector.js index f804c1e0dca..dd091d6380c 100644 --- a/client/galaxy/scripts/ui/peek-column-selector.js +++ b/client/galaxy/scripts/ui/peek-column-selector.js @@ -1,7 +1,8 @@ -// from: https://raw.githubusercontent.com/umdjs/umd/master/jqueryPlugin.js -// Uses AMD or browser globals to create a jQuery plugin. -import jQuery from "jquery"; -var $ = jQuery; +//import $ from "jquery"; +// TODO: This manipulates whatever jquery is available -- needs restructuring, +// or removal (jquery plugins are a bad design choice for us at this point) +// It is *only* used in the scatterplot viz, so this is safe. +/* global $ */ //============================================================================== /** Column selection using the peek display as the control. @@ -63,38 +64,38 @@ var defaults = { topLeftContent: "Columns:" }; -var /** class added to the pre.peek element (to allow css on just the control) */ -PEEKCONTROL_CLASS = "peek-column-selector"; +/** class added to the pre.peek element (to allow css on just the control) */ +const PEEKCONTROL_CLASS = "peek-column-selector"; -var /** the string of the event fired when a control row changes */ -CHANGE_EVENT = "peek-column-selector.change"; +/** the string of the event fired when a control row changes */ +const CHANGE_EVENT = "peek-column-selector.change"; -var /** the string of the event fired when a column is renamed */ -RENAME_EVENT = "peek-column-selector.rename"; +/** the string of the event fired when a column is renamed */ +const RENAME_EVENT = "peek-column-selector.rename"; -var /** class added to the control rows */ -ROW_CLASS = "control"; +/** class added to the control rows */ +const ROW_CLASS = "control"; -var /** class added to the left-hand cells that serve as row prompts */ -PROMPT_CLASS = "control-prompt"; +/** class added to the left-hand cells that serve as row prompts */ +const PROMPT_CLASS = "control-prompt"; -var /** class added to selected _cells_/tds */ -SELECTED_CLASS = "selected"; +/** class added to selected _cells_/tds */ +const SELECTED_CLASS = "selected"; -var /** class added to disabled/un-clickable cells/tds */ -DISABLED_CLASS = "disabled"; +/** class added to disabled/un-clickable cells/tds */ +const DISABLED_CLASS = "disabled"; -var /** class added to the clickable surface within a cell to select it */ -BUTTON_CLASS = "button"; +/** class added to the clickable surface within a cell to select it */ +const BUTTON_CLASS = "button"; -var /** class added to peek table header (th) cells to indicate they can be clicked and are renamable */ -RENAMABLE_HEADER_CLASS = "renamable-header"; +/** class added to peek table header (th) cells to indicate they can be clicked and are renamable */ +const RENAMABLE_HEADER_CLASS = "renamable-header"; -var /** the data key used for each cell to store the column index ('data-...') */ -COLUMN_INDEX_DATA_KEY = "column-index"; +/** the data key used for each cell to store the column index ('data-...') */ +const COLUMN_INDEX_DATA_KEY = "column-index"; -var /** renamable header data key used to store the column name (w/o the number and dot: '1.Bler') */ -COLUMN_NAME_DATA_KEY = "column-name"; +/** renamable header data key used to store the column name (w/o the number and dot: '1.Bler') */ +const COLUMN_NAME_DATA_KEY = "column-name"; //TODO: not happy with pure functional here - rows should polymorph (multi, single, etc.) //TODO: needs clean up, move handlers to outer scope @@ -102,10 +103,10 @@ COLUMN_NAME_DATA_KEY = "column-name"; // ........................................................................ /** validate the control data sent in for each row */ function validateControl(control) { - if (control.disabled && jQuery.type(control.disabled) !== "array") { + if (control.disabled && $.type(control.disabled) !== "array") { throw new Error(`"disabled" must be defined as an array of indeces: ${JSON.stringify(control)}`); } - if (control.multiselect && control.selected && jQuery.type(control.selected) !== "array") { + if (control.multiselect && control.selected && $.type(control.selected) !== "array") { throw new Error(`Mulitselect rows need an array for "selected": ${JSON.stringify(control)}`); } if (!control.label || !control.id) { @@ -212,7 +213,7 @@ function buildMultiSelectCell(control, columnIndex) { var eventData = {}; var key = $cell.parent().attr("id"); - var val = jQuery.makeArray(selectedColumnIndeces); + var val = $.makeArray(selectedColumnIndeces); eventData[key] = val; $cell.parents(".peek").trigger(CHANGE_EVENT, eventData); }); @@ -252,7 +253,7 @@ function buildControlRow(cellCount, control, includePrompts) { // ........................................................................ /** add to the peek, using options for configuration, return the peek */ function peekColumnSelector(options) { - options = jQuery.extend(true, {}, defaults, options); + options = $.extend(true, {}, defaults, options); var $peek = $(this).addClass(PEEKCONTROL_CLASS); var $peektable = $peek.find("table"); @@ -312,7 +313,7 @@ function peekColumnSelector(options) { var index = $this.index() + (options.includePrompts ? 0 : 1); var prevName = $this.data(COLUMN_NAME_DATA_KEY); - var newColumnName = prompt("New column name:", prevName); + var newColumnName = window.prompt("New column name:", prevName); if (newColumnName !== null && newColumnName !== prevName) { // set the new text and data $this @@ -320,7 +321,7 @@ function peekColumnSelector(options) { .data(COLUMN_NAME_DATA_KEY, newColumnName) .attr("data-", COLUMN_NAME_DATA_KEY, newColumnName); // fire event for new column names - var columnNames = jQuery.makeArray( + var columnNames = $.makeArray( $this .parent() .children("th:not(.top-left)") @@ -344,7 +345,7 @@ function peekColumnSelector(options) { // ........................................................................ // as jq plugin -jQuery.fn.extend({ +$.fn.extend({ peekColumnSelector: function $peekColumnSelector(options) { return this.map(function() { return peekColumnSelector.call(this, options); diff --git a/client/package.json b/client/package.json index c96fa5edb14..7c51eb18ff7 100644 --- a/client/package.json +++ b/client/package.json @@ -49,8 +49,6 @@ "gulp-production-maps": "GXY_BUILD_SOURCEMAPS=1 NODE_ENV=production gulp", "build-toolshed": "grunt --app=toolshed", "jshint": "jshint --exclude='galaxy/scripts/libs/**' galaxy/scripts/**/*.js", - "build-charts": "webpack -p --config ../config/plugins/visualizations/charts/webpack.config.js", - "build-scatterplot": "NODE_PATH=./node_modules webpack -p --config ../config/plugins/visualizations/scatterplot/webpack.config.js", "prettier": "prettier --write --tab-width 4 --print-width 120 \"galaxy/scripts/{,!(libs)/**/}/{*.js,*.vue}\"" }, "devDependencies": { diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index 4da482becf8..3048ef2a539 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -74,6 +74,7 @@ + @@ -343,7 +344,7 @@ - + @@ -497,7 +498,7 @@ - + diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample index 6951fc2762e..a18fe1afa5b 100644 --- a/config/galaxy.yml.sample +++ b/config/galaxy.yml.sample @@ -23,6 +23,12 @@ uwsgi: # ':8080' to listen on all available network interfaces. http: 127.0.0.1:8080 + # By default uWSGI allocates a very small buffer (4096 bytes) for the + # headers of each request. If you start receiving "invalid request + # block size" in your logs, it could mean you need a bigger buffer. + # Increase it up to 65535. + buffer-size: 4096 + # Number of web server (worker) processes to fork after the # application has loaded. processes: 1 @@ -1635,6 +1641,23 @@ galaxy: # processors, memory and walltime. #job_resource_params_file: config/job_resource_params_conf.xml + # Similar to the above parameter, workflows can describe parameters + # used to influence scheduling of jobs within the workflow. This + # requires both a description of the fields available (which defaults + # to the definitions in job_resource_params_file if not set). + #workflow_resource_params_file: config/workflow_resource_params_conf.xml + + # This parameter describes how to map users and workflows to a set of + # workflow resource parameter to present (typically input IDs from + # workflow_resource_params_file). If this this is a function reference + # it will be passed various inputs (workflow model object and user) + # and it should produce a list of input IDs. If it is a path it is + # expected to an XML or YAML file describing how to map group names to + # parameter descriptions (additional types of mappings via these files + # could be implemented but haven't yet - for instance using workflow + # tags to do the mapping). + #workflow_resource_params_mapper: config/workflow_resource_mapper_conf.yml + # If using job concurrency limits (configured in job_config_file), # several extra database queries must be performed to determine the # number of jobs a user has dispatched to a given destination. By diff --git a/config/job_conf.xml.sample_advanced b/config/job_conf.xml.sample_advanced index 9a26eaa1427..ffbeebd0a76 100644 --- a/config/job_conf.xml.sample_advanced +++ b/config/job_conf.xml.sample_advanced @@ -121,7 +121,7 @@ - - - - - + - - + @@ -62,11 +57,20 @@ +
+ + + + + + + + +
-
-
- - - - - - - - - - - - - - -
-
- -
-
- -
-
+
@@ -114,13 +96,29 @@
+
+ + + +
- -
+
+ + + + + + + + + + + +
diff --git a/config/tool_shed.yml.sample b/config/tool_shed.yml.sample index 824f1134279..b7d125f9a9d 100644 --- a/config/tool_shed.yml.sample +++ b/config/tool_shed.yml.sample @@ -5,6 +5,12 @@ uwsgi: # ':9009' to listen on all available network interfaces. http: 127.0.0.1:9009 + # By default uWSGI allocates a very small buffer (4096 bytes) for the + # headers of each request. If you start receiving "invalid request + # block size" in your logs, it could mean you need a bigger buffer. + # Increase it up to 65535. + buffer-size: 4096 + # Number of web server (worker) processes to fork after the # application has loaded. processes: 1 diff --git a/config/workflow_resource_mapper_conf.yml.sample b/config/workflow_resource_mapper_conf.yml.sample new file mode 100644 index 00000000000..648dd09ddee --- /dev/null +++ b/config/workflow_resource_mapper_conf.yml.sample @@ -0,0 +1,14 @@ +by_group: + default: default + groups: + default: [project, priority] + prio_basic: [{name: priority, options: ["low", "med"]}] + prio_advanced: [{name: priority, options: ["low", "med", "high"]}] + prio_super: + - time + - memory + - processors + - name: priority + options: + - ultra + - plus_ultra diff --git a/config/workflow_resource_params_conf.xml.sample b/config/workflow_resource_params_conf.xml.sample new file mode 100644 index 00000000000..c04d2909aa5 --- /dev/null +++ b/config/workflow_resource_params_conf.xml.sample @@ -0,0 +1,13 @@ + + + + + + + diff --git a/doc/source/admin/cluster.md b/doc/source/admin/cluster.md index 5f40c91440b..3ff9eaff053 100644 --- a/doc/source/admin/cluster.md +++ b/doc/source/admin/cluster.md @@ -195,7 +195,7 @@ Most options available to `qsub(1b)` and `pbs_submit(3b)` are supported. Except ``` -The value of *ppn=* is used by PBS to define the environment variable $PBS_NCPUS which in turn is used by galaxy for [GALAXY_SLOTS](https://galaxyproject.org/admin/config/galaxy_slots/). +The value of *ppn=* is used by PBS to define the environment variable `$PBS_NCPUS` which in turn is used by galaxy for [GALAXY_SLOTS](https://galaxyproject.org/admin/config/galaxy_slots/). ### Condor @@ -355,6 +355,30 @@ It is also a good idea to make sure that only trusted users, e.g. root, have wri Some maintenance and support of this code will be provided via the usual [Support](https://galaxyproject.org/support/) channels, but improvements and fixes would be greatly welcomed, as this is a complex feature which is not used by the Galaxy Development Team. +## Special environment variables for job resources + +Galaxy *tries* to define special environment variables for each job that contain +the information on the number of available slots and the amount of available +memory: + +* `GALAXY_SLOTS`: number of available slots +* `GALAXY_MEMORY_MB`: total amount of available memory in MB +* `GALAXY_MEMORY_MB_PER_SLOT`: amount of memory that is available for each slot in MB + +More precisely Galaxy inserts bash code in the job submit script that +tries to determine these values. This bash code is defined here: + +* lib/galaxy/jobs/runners/util/job_script/CLUSTER_SLOTS_STATEMENT.sh +* lib/galaxy/jobs/runners/util/job_script/MEMORY_STATEMENT.sh + +If this code is unable to determine the variables, then they will not be set. +Therefore in the tool XML files the variables should be used with a default, +e.g. `\${GALAXY_SLOTS:-1}` (see also https://planemo.readthedocs.io/en/latest/writing_advanced.html#cluster-usage). + +In particular `GALAXY_MEMORY_MB` and `GALAXY_MEMORY_MB_PER_SLOT` are currently +defined only for a few cluster types. Contributions are very welcome, e.g. let +the Galaxy developers know how to modify that file to support your cluster. + ## Contributors * **Oleksandr Moskalenko**, debugged a number of problems related to running jobs as the real user and using DRMAA with TORQUE. diff --git a/doc/source/admin/nginx.md b/doc/source/admin/nginx.md index c6df513f4ac..e3f0752837a 100644 --- a/doc/source/admin/nginx.md +++ b/doc/source/admin/nginx.md @@ -69,10 +69,9 @@ http { gzip on; gzip_http_version 1.1; gzip_vary on; - gzip_comp_level 4; + gzip_comp_level 6; gzip_proxied any; gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; - gzip_comp_level 6; gzip_buffers 16 8k; # allow up to 3 minutes for Galaxy to respond to slow requests before timing out @@ -296,7 +295,7 @@ nginx did not support shared modules, and the upload module is not yet shared-co and complicated process, the Galaxy Committers team maintains (for some platforms) versions of nginx modified from their upstream package sources (APT, EPEL, etc.) to include the upload module: -- [Ubuntu (PPA)](https://launchpad.net/~galaxyproject/+archive/ubuntu/nginx) +- [Ubuntu (PPA)](https://launchpad.net/~galaxyproject/+archive/ubuntu/nginx) (Be sure to install `nginx-extras`, not `nginx`) - [Enterprise Linux](https://depot.galaxyproject.org/yum/) To contribute support for additional platforms, please see the [Galaxy diff --git a/doc/source/admin/scaling.md b/doc/source/admin/scaling.md index d1bee86c866..a500ed0ec91 100644 --- a/doc/source/admin/scaling.md +++ b/doc/source/admin/scaling.md @@ -193,7 +193,7 @@ uwsgi: # fix up signal handling die-on-term: true hook-master-start: unix_signal:2 gracefully_kill_them_all - hook-master-start: unix_signal:5 gracefully_kill_them_all + hook-master-start: unix_signal:15 gracefully_kill_them_all # listening options @@ -398,7 +398,7 @@ $ ./scripts/galaxy-main -c config/galaxy.yml --server-name handler2 --daemonize However, a better option to managing processes by hand is to use a process manager as documented in the [Starting and Stopping](#starting-and-stopping) section. -#### uWSGI Minutiea +#### uWSGI Minutiae **Threads** diff --git a/doc/source/releases/18.01_announce.rst b/doc/source/releases/18.01_announce.rst index c6b00ecff6d..e2032f067bb 100644 --- a/doc/source/releases/18.01_announce.rst +++ b/doc/source/releases/18.01_announce.rst @@ -14,10 +14,8 @@ Highlights **Web Server and Configuration** The default web server used by Galaxy has changed from Paste to `uWSGI `__ and the default configuration file for Galaxy is now ``config/galaxy.yml`` instead of ``config/galaxy.ini``. - uWSGI is more production ready and allows Galaxy to scale better in its default - configuration. In the future uWSGI will allow Galaxy to setup GIE proxies without additional - configuration and use modern web technologies such as web sockets. - Read more about the server, configuration, and documentation changes in the `uWSGI details`_ section of this document. + To minimize the impact of this change on existing Galaxy instances, if a Galaxy has a ``galaxy.ini`` file configured, it will continue to use Paste by default unless additional steps are taken by the administrator. + uWSGI is more production ready and allows Galaxy to scale better in its default configuration. Read more about the server, configuration, and documentation changes in the `uWSGI details`_ section of this document. **Dataset Collection Usability** This release has significantly improved the usability of Galaxy dataset collections. Dozens of improvements @@ -78,7 +76,7 @@ To get a new Galaxy repository run: To update an existing Galaxy repository run: .. code-block:: shell - $ git checkout release_18.01 && git pull --ff-only origin release_18.01 + $ git fetch origin && git checkout release_18.01 && git pull --ff-only origin release_18.01 See the `community hub `__ for additional details regarding the source code locations. @@ -128,17 +126,16 @@ In addition to that, the PlantTribes datatypes have been commented out in the so uWSGI details ============= - -To minimize the impact of this change on existing Galaxy instances, if a Galaxy has a ``galaxy.ini`` -file configured, it will continue to use Paste by default unless additional steps are -taken by the administrator (Galaxy can be forced to start under uWSGI even with an older configuration file -by setting ``APP_WEBSERVER=uwsgi`` in the environment). As part of the transition to YAML-based +Galaxy can be forced to start under uWSGI even with an older configuration file +by setting ``APP_WEBSERVER=uwsgi`` in the environment. As part of the transition to YAML-based configuration files, we have implemented a schema to validate Galaxy configuration files. Run ``make config-validate`` from Galaxy's root directory to validate a schema and ``make config-lint`` to check for best practices. While there is no need to convert your configuration file (``galaxy.ini`` hasn't been deprecated), you can run ``make config-convert-dry-run`` and ``make config-convert`` to respectively test and perform the conversion of an ``ini`` configuration file to a YAML one. +In the future uWSGI will allow Galaxy to setup GIE proxies without additional configuration and use modern web technologies such as web sockets. + These are big changes that affect many parts of Galaxy's administration documentation and makes this documentation very dependent on which Galaxy version they are targeting. To address this, we have moved a significant amount of administration documentation into Galaxy's code diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 26ea0bc56d9..1f054d552df 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -47,6 +47,7 @@ PATH_DEFAULTS = dict( error_report_file=['config/error_report.yml', 'config/error_report.yml.sample'], dependency_resolvers_config_file=['config/dependency_resolvers_conf.xml', 'dependency_resolvers_conf.xml'], job_resource_params_file=['config/job_resource_params_conf.xml', 'job_resource_params_conf.xml'], + workflow_resource_params_file=['config/workflow_resource_params_conf.xml', 'workflow_resource_params_conf.xml'], migrated_tools_config=['migrated_tools_conf.xml', 'config/migrated_tools_conf.xml'], object_store_config_file=['config/object_store_conf.xml', 'object_store_conf.xml'], openid_config_file=['config/openid_conf.xml', 'openid_conf.xml', 'config/openid_conf.xml.sample'], @@ -391,6 +392,15 @@ class Configuration(object): self.maximum_workflow_invocation_duration = int(kwargs.get("maximum_workflow_invocation_duration", 2678400)) self.maximum_workflow_jobs_per_scheduling_iteration = int(kwargs.get("maximum_workflow_jobs_per_scheduling_iteration", -1)) + workflow_resource_params_mapper = kwargs.get("workflow_resource_params_mapper", None) + if not workflow_resource_params_mapper: + workflow_resource_params_mapper = None + elif ":" not in workflow_resource_params_mapper: + # Assume it is not a Python function, so a file + workflow_resource_params_mapper = self.resolve_path(workflow_resource_params_mapper) + # else: a Python a function! + self.workflow_resource_params_mapper = workflow_resource_params_mapper + self.cache_user_job_count = string_as_bool(kwargs.get('cache_user_job_count', False)) self.pbs_application_server = kwargs.get('pbs_application_server', "") self.pbs_dataset_server = kwargs.get('pbs_dataset_server', "") diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 9d75f1232fb..f7bd98dac53 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -188,7 +188,7 @@ class GenericAsn1Binary(Binary): edam_data = "data_0849" -class BamNative(Binary): +class BamNative(CompressedArchive): """Class describing a BAM binary file that is not necessarily sorted""" edam_format = "format_2572" edam_data = "data_0863" @@ -592,7 +592,7 @@ class CRAM(Binary): return False -class BaseBcf(Binary): +class BaseBcf(CompressedArchive): edam_format = "format_3020" edam_data = "data_3498" diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index e2e21904d05..66842ac9ef7 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -116,6 +116,15 @@ class Data(object): self.composite_files = self.composite_files.copy() self.display_applications = odict() + @property + def validate_mode(self): + """Indicate that a sniffer should run, even if disabled. + + Some sniffers (e.g. fastq.gz) work but are not enabled for certain reasons, but when running a sniffer to + "validate" a selected filetype, those sniffers should be enabled. + """ + return os.environ.get('GALAXY_SNIFFER_VALIDATE_MODE', '0') == '1' + def get_raw_data(self, dataset): """Returns the full data. To stream it open the file_name and read/write as needed""" try: @@ -752,6 +761,8 @@ class Text(Data): file_ext = 'txt' line_class = 'line' + is_binary = False + # Add metadata elements MetadataElement(name="data_lines", default=0, desc="Number of data lines", readonly=True, optional=True, visible=False, no_value=0) diff --git a/lib/galaxy/datatypes/genetics.py b/lib/galaxy/datatypes/genetics.py index 6349a4fdc7f..3c8b5ede0ca 100644 --- a/lib/galaxy/datatypes/genetics.py +++ b/lib/galaxy/datatypes/genetics.py @@ -606,7 +606,6 @@ class RexpBase(Html): MetadataElement(name="pheno_path", desc="Path to phenotype data for this experiment", default="rexpression.pheno", visible=True) file_ext = 'rexpbase' html_table = None - is_binary = True composite_type = 'auto_primary_file' allow_datatype_change = False diff --git a/lib/galaxy/datatypes/ngsindex.py b/lib/galaxy/datatypes/ngsindex.py index 18ee180807e..f5623a81060 100644 --- a/lib/galaxy/datatypes/ngsindex.py +++ b/lib/galaxy/datatypes/ngsindex.py @@ -18,7 +18,6 @@ class BowtieIndex(Html): MetadataElement(name="base_name", desc="base name for this index set", default='galaxy_generated_bowtie_index', set_in_upload=True, readonly=True) MetadataElement(name="sequence_space", desc="sequence_space for this index set", default='unknown', set_in_upload=True, readonly=True) - is_binary = True composite_type = 'auto_primary_file' allow_datatype_change = False diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index da6f6445e61..20dbec03a74 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -16,7 +16,10 @@ import bx.align.maf from galaxy import util from galaxy.datatypes import metadata -from galaxy.datatypes.binary import Binary +from galaxy.datatypes.binary import ( + Binary, + CompressedArchive +) from galaxy.datatypes.metadata import MetadataElement from galaxy.datatypes.sniff import ( get_headers, @@ -311,7 +314,7 @@ class Alignment(data.Text): raise NotImplementedError("Can't split generic alignment files") -class FastaGz(Sequence, Binary): +class FastaGz(Sequence, CompressedArchive): """Class representing a generic compressed FASTA sequence""" edam_format = "format_1929" file_ext = "fasta.gz" @@ -319,7 +322,7 @@ class FastaGz(Sequence, Binary): def sniff(self, filename): """Determines whether the file is in gzip-compressed FASTA format""" - if not SNIFF_COMPRESSED_FASTAS: + if not SNIFF_COMPRESSED_FASTAS and not self.validate_mode: return False if not is_gzip(filename): return False @@ -738,7 +741,7 @@ class FastqCSSanger(Fastq): file_ext = "fastqcssanger" -class FastqGz(BaseFastq, Binary): +class FastqGz(BaseFastq, CompressedArchive): """Class representing a generic compressed FASTQ sequence""" edam_format = "format_1930" file_ext = "fastq.gz" @@ -746,7 +749,7 @@ class FastqGz(BaseFastq, Binary): def sniff(self, filename): """Determines whether the file is in gzip-compressed FASTQ format""" - if not SNIFF_COMPRESSED_FASTQS: + if not SNIFF_COMPRESSED_FASTQS and not self.validate_mode: return False if not is_gzip(filename): return False @@ -776,7 +779,7 @@ class FastqCSSangerGz(FastqGz): file_ext = "fastqcssanger.gz" -class FastqBz2(BaseFastq, Binary): +class FastqBz2(BaseFastq, CompressedArchive): """Class representing a generic compressed FASTQ sequence""" edam_format = "format_1930" file_ext = "fastq.bz2" @@ -784,7 +787,7 @@ class FastqBz2(BaseFastq, Binary): def sniff(self, filename): """Determine whether the file is in bzip2-compressed FASTQ format""" - if not SNIFF_COMPRESSED_FASTQS: + if not SNIFF_COMPRESSED_FASTQS and not self.validate_mode: return False if not is_bz2(filename): return False diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 221887ec82c..709d5b02eb7 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -14,15 +14,18 @@ import tempfile import zipfile from six import text_type +from six.moves import filter from six.moves.urllib.request import urlopen from galaxy import util from galaxy.util import compression_utils from galaxy.util.checkers import ( check_binary, + check_bz2, + check_gzip, check_html, - is_bz2, - is_gzip + check_zip, + is_tar, ) if sys.version_info < (3, 3): @@ -94,21 +97,6 @@ def stream_to_file(stream, suffix='', prefix='', dir=None, text=False, **kwd): return stream_to_open_named_file(stream, fd, temp_name, **kwd) -def check_newlines(fname, bytes_to_read=52428800): - """ - Determines if there are any non-POSIX newlines in the first - number_of_bytes (by default, 50MB) of the file. - """ - CHUNK_SIZE = 2 ** 20 - with open(fname, 'r') as f: - for chunk in f.read(CHUNK_SIZE): - if f.tell() > bytes_to_read: - break - if chunk.count('\r'): - return True - return False - - def convert_newlines(fname, in_place=True, tmp_dir=None, tmp_prefix="gxupload"): """ Converts in place a file from universal line endings @@ -278,7 +266,7 @@ def is_column_based(fname, sep='\t', skip=0): return True -def guess_ext(fname, sniff_order): +def guess_ext(fname, sniff_order, is_binary=False): """ Returns an extension that can be used in the datatype factory to generate a data for the 'fname' file @@ -402,7 +390,8 @@ def guess_ext(fname, sniff_order): successfully discovered. """ try: - if datatype.sniff(fname): + if ((is_binary and datatype.is_binary) or + (not is_binary)) and datatype.sniff(fname): file_ext = datatype.file_ext break except Exception: @@ -416,46 +405,78 @@ def guess_ext(fname, sniff_order): if file_ext is not None: return file_ext + # skip header check if data is already known to be binary + if is_binary: + return file_ext or 'binary' try: get_headers(fname, None) except UnicodeDecodeError: - return 'data' # default binary data type file extension + return 'data' # default data type file extension if is_column_based(fname, '\t', 1): return 'tabular' # default tabular data type file extension return 'txt' # default text data type file extension -def handle_compressed_file(filename, datatypes_registry, ext='auto'): +def zip_single_fileobj(path): + z = zipfile.ZipFile(path) + for name in z.namelist(): + if not name.endswith('/'): + return z.open(name) + + +def handle_compressed_file( + filename, + datatypes_registry, + ext='auto', + tmp_prefix='sniff_uncompress_', + tmp_dir=None, + in_place=False, + check_content=True, + auto_decompress=True, +): + """ + Check uploaded files for compression, check compressed file contents, and uncompress if necessary. + + Supports GZip, BZip2, and the first file in a Zip file. + + For performance reasons, the temporary file used for uncompression is located in the same directory as the + input/output file. This behavior can be changed with the `tmp_dir` param. + + ``ext`` as returned will only be changed from the ``ext`` input param if the param was an autodetect type (``auto``) + and the file was sniffed as a keep-compressed datatype. + + ``is_valid`` as returned will only be set if the file is compressed and contains invalid contents (or the first file + in the case of a zip file), this is so lengthy decompression can be bypassed if there is invalid content in the + first 32KB. Otherwise the caller should be checking content. + """ CHUNK_SIZE = 2 ** 20 # 1Mb is_compressed = False compressed_type = None keep_compressed = False is_valid = False + uncompressed = filename + tmp_dir = tmp_dir or os.path.dirname(filename) for compressed_type, check_compressed_function in COMPRESSION_CHECK_FUNCTIONS: - is_compressed = check_compressed_function(filename) + is_compressed, is_valid = check_compressed_function(filename, check_content=check_content) if is_compressed: break # found compression type - if is_compressed: + if is_compressed and is_valid: if ext in AUTO_DETECT_EXTENSIONS: - check_exts = COMPRESSION_DATATYPES[compressed_type] - elif ext in COMPRESSED_EXTENSIONS: - check_exts = [ext] + # attempt to sniff for a keep-compressed datatype (observing the sniff order) + sniff_datatypes = filter(lambda d: getattr(d, 'compressed', False), datatypes_registry.sniff_order) + for datatype in sniff_datatypes: + if datatype.sniff(filename): + ext = datatype.file_ext + keep_compressed = True + break else: - check_exts = [] - for compressed_ext in check_exts: - compressed_datatype = datatypes_registry.get_datatype_by_extension(compressed_ext) - if compressed_datatype.sniff(filename): - ext = compressed_ext - keep_compressed = True - is_valid = True - break - - if not is_compressed: - is_valid = True - elif not keep_compressed: - is_valid = True - fd, uncompressed = tempfile.mkstemp() + datatype = datatypes_registry.get_datatype_by_extension(ext) + keep_compressed = getattr(datatype, 'compressed', False) + # don't waste time decompressing if we sniff invalid contents + if is_compressed and is_valid and auto_decompress and not keep_compressed: + fd, uncompressed = tempfile.mkstemp(prefix=tmp_prefix, dir=tmp_dir) compressed_file = DECOMPRESSION_FUNCTIONS[compressed_type](filename) + # TODO: it'd be ideal to convert to posix newlines and space-to-tab here as well while True: try: chunk = compressed_file.read(CHUNK_SIZE) @@ -469,35 +490,75 @@ def handle_compressed_file(filename, datatypes_registry, ext='auto'): os.write(fd, chunk) os.close(fd) compressed_file.close() - # Replace the compressed file with the uncompressed file - shutil.move(uncompressed, filename) - return is_valid, ext + if in_place: + # Replace the compressed file with the uncompressed file + shutil.move(uncompressed, filename) + uncompressed = filename + elif not is_compressed: + is_valid = True + return is_valid, ext, uncompressed, compressed_type -def handle_uploaded_dataset_file(filename, datatypes_registry, ext='auto'): - is_valid, ext = handle_compressed_file(filename, datatypes_registry, ext=ext) +def handle_uploaded_dataset_file( + filename, + datatypes_registry, + ext='auto', + tmp_prefix='sniff_upload_', + tmp_dir=None, + in_place=False, + check_content=True, + is_binary=None, + auto_decompress=True, + uploaded_file_ext=None, + convert_to_posix_lines=None, + convert_spaces_to_tabs=None, +): + is_valid, ext, converted_path, compressed_type = handle_compressed_file( + filename, + datatypes_registry, + ext=ext, + tmp_prefix=tmp_prefix, + tmp_dir=tmp_dir, + in_place=in_place, + check_content=check_content, + auto_decompress=auto_decompress, + ) + try: + if not is_valid: + if is_tar(converted_path): + raise InappropriateDatasetContentError('TAR file uploads are not supported') + raise InappropriateDatasetContentError('The uploaded compressed file contains invalid content') - if not is_valid: - raise InappropriateDatasetContentError('The compressed uploaded file contains inappropriate content.') + # This needs to be checked again after decompression + is_binary = check_binary(converted_path) - if ext in AUTO_DETECT_EXTENSIONS: - ext = guess_ext(filename, sniff_order=datatypes_registry.sniff_order) + if not is_binary and convert_to_posix_lines: + # Convert universal line endings to Posix line endings, spaces to tabs (if desired) + if convert_spaces_to_tabs: + convert_fxn = convert_newlines_sep2tabs + else: + convert_fxn = convert_newlines + line_count, _converted_path = convert_fxn(converted_path, in_place=in_place, tmp_dir=tmp_dir, tmp_prefix=tmp_prefix) + if not in_place: + if converted_path and filename != converted_path: + os.unlink(converted_path) + converted_path = _converted_path - if check_binary(filename): - if not datatypes_registry.is_extension_unsniffable_binary(ext) and not datatypes_registry.get_datatype_by_extension(ext).sniff(filename): - raise InappropriateDatasetContentError('The binary uploaded file contains inappropriate content.') - elif check_html(filename): - raise InappropriateDatasetContentError('The uploaded file contains inappropriate HTML content.') - return ext + if ext in AUTO_DETECT_EXTENSIONS: + ext = guess_ext(converted_path, sniff_order=datatypes_registry.sniff_order, is_binary=is_binary) + + if not is_binary and check_content and check_html(converted_path): + raise InappropriateDatasetContentError('The uploaded file contains invalid HTML content') + except Exception: + if filename != converted_path: + os.unlink(converted_path) + raise + return ext, converted_path, compressed_type AUTO_DETECT_EXTENSIONS = ['auto'] # should 'data' also cause auto detect? -DECOMPRESSION_FUNCTIONS = dict(gzip=gzip.GzipFile, bz2=bz2.BZ2File) -COMPRESSION_CHECK_FUNCTIONS = [('gzip', is_gzip), ('bz2', is_bz2)] -COMPRESSION_DATATYPES = dict(gzip=['bam', 'fasta.gz', 'fastq.gz', 'fastqsanger.gz', 'fastqillumina.gz', 'fastqsolexa.gz', 'fastqcssanger.gz'], bz2=['fastq.bz2', 'fastqsanger.bz2', 'fastqillumina.bz2', 'fastqsolexa.bz2', 'fastqcssanger.bz2']) -COMPRESSED_EXTENSIONS = [] -for exts in COMPRESSION_DATATYPES.values(): - COMPRESSED_EXTENSIONS.extend(exts) +DECOMPRESSION_FUNCTIONS = dict(gz=gzip.GzipFile, bz2=bz2.BZ2File, zip=zip_single_fileobj) +COMPRESSION_CHECK_FUNCTIONS = [('gz', check_gzip), ('bz2', check_bz2), ('zip', check_zip)] class InappropriateDatasetContentError(Exception): diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 1addf7d7400..364bf048431 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -681,8 +681,14 @@ class BaseVcf(Tabular): MetadataElement(name="sample_names", default=[], desc="Sample names", readonly=True, visible=False, optional=True, no_value=[]) def sniff(self, filename): - headers = get_headers(filename, '\n', count=1) - return headers[0][0].startswith("##fileformat=VCF") + # Because this sniffer is run on compressed files that might be BGZF (due to the VcfGz subclass), we should + # handle unicode decode errors. This should ultimately be done in get_headers(), but guess_ext() currently + # relies on get_headers() raising this exception. + try: + headers = get_headers(filename, '\n', count=1) + return headers[0][0].startswith("##fileformat=VCF") + except UnicodeDecodeError: + return False def display_peek(self, dataset): """Returns formated html of peek""" diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 73d00a8ec1e..5aed3f7924a 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -3,6 +3,7 @@ Support for running a tool in Galaxy via an internal job management system """ import copy import datetime +import errno import logging import os import pwd @@ -397,17 +398,7 @@ class JobConfiguration(ConfiguresHandlers): return conditional_element def __parse_resource_parameters(self): - if os.path.exists(self.app.config.job_resource_params_file): - resource_param_file = self.app.config.job_resource_params_file - try: - resource_definitions = util.parse_xml(resource_param_file) - except Exception as e: - raise config_exception(e, resource_param_file) - resource_definitions_root = resource_definitions.getroot() - # TODO: Also handling conditionals would be awesome! - for parameter_elem in resource_definitions_root.findall("param"): - name = parameter_elem.get("name") - self.resource_parameters[name] = parameter_elem + self.resource_parameters = util.parse_resource_parameters(self.app.config.job_resource_params_file) def __get_params(self, parent): """Parses any child tags in to a dictionary suitable for persistence. @@ -831,6 +822,18 @@ class JobWrapper(HasResourceParameters): def get_version_string_path(self): return os.path.abspath(os.path.join(self.app.config.new_file_path, "GALAXY_VERSION_STRING_%s" % self.job_id)) + def __prepare_upload_paramfile(self, tool_evaluator): + """Special case paramfile handling for the upload tool. Moves the paramfile to the working directory + """ + new = os.path.join(self.working_directory, 'upload_params.json') + try: + shutil.move(tool_evaluator.param_dict['paramfile'], new) + except (OSError, IOError) as exc: + # It won't exist at the old path if setup was interrupted and tried again later + if exc.errno != errno.ENOENT or not os.path.exists(new): + raise + tool_evaluator.param_dict['paramfile'] = new + def prepare(self, compute_environment=None): """ Prepare the job to run by creating the working directory and the @@ -855,6 +858,10 @@ class JobWrapper(HasResourceParameters): self.sa_session.flush() + # TODO: The upload tool actions that create the paramfile can probably be turned in to a configfile to remove this special casing + if job.tool_id == 'upload1': + self.__prepare_upload_paramfile(tool_evaluator) + self.command_line, self.extra_filenames, self.environment_variables = tool_evaluator.build() # Ensure galaxy_lib_dir is set in case there are any later chdirs self.galaxy_lib_dir @@ -947,6 +954,11 @@ class JobWrapper(HasResourceParameters): ) return tool_evaluator + def _fix_output_permissions(self): + for path in [dp.real_path for dp in self.get_mutable_output_fnames()]: + if os.path.exists(path): + util.umask_fix_perms(path, self.app.config.umask, 0o666, self.app.config.gid) + def fail(self, message, exception=False, stdout="", stderr="", exit_code=None): """ Indicate job failure by setting state and message on all output @@ -1009,6 +1021,7 @@ class JobWrapper(HasResourceParameters): # the partial files to the object store regardless of whether job.state == DELETED self.__update_output(job, dataset, clean_only=True) + self._fix_output_permissions() self._report_error() # Perform email action even on failure. for pja in [pjaa.post_job_action for pjaa in job.post_job_actions if pjaa.post_job_action.action_type == "EmailAction"]: @@ -1273,7 +1286,7 @@ class JobWrapper(HasResourceParameters): if retry_internally and not self.external_output_metadata.external_metadata_set_successfully(dataset, self.sa_session): # If Galaxy was expected to sniff type and didn't - do so. if dataset.ext == "_sniff_": - extension = sniff.handle_uploaded_dataset_file(dataset.dataset.file_name, self.app.datatypes_registry) + extension = sniff.handle_uploaded_dataset_file(dataset.dataset.file_name, self.app.datatypes_registry)[0] dataset.extension = extension # call datatype.set_meta directly for the initial set_meta call during dataset creation @@ -1418,10 +1431,7 @@ class JobWrapper(HasResourceParameters): # user). self.sa_session.flush() - # fix permissions - for path in [dp.real_path for dp in self.get_mutable_output_fnames()]: - if os.path.exists(path): - util.umask_fix_perms(path, self.app.config.umask, 0o666, self.app.config.gid) + self._fix_output_permissions() # Finally set the job state. This should only happen *after* all # dataset creation, and will allow us to eliminate force_history_refresh. diff --git a/lib/galaxy/jobs/dynamic_tool_destination.py b/lib/galaxy/jobs/dynamic_tool_destination.py index 1c6e5823287..a0480692c2e 100755 --- a/lib/galaxy/jobs/dynamic_tool_destination.py +++ b/lib/galaxy/jobs/dynamic_tool_destination.py @@ -3,15 +3,18 @@ from __future__ import print_function import argparse import collections import copy +import json import logging import os import re import sys from functools import reduce +from xml.etree import ElementTree as ET +import numpy as np from yaml import load -__version__ = '1.0.0' +__version__ = '1.1.0' # log to galaxy's logger log = logging.getLogger(__name__) @@ -19,6 +22,37 @@ log = logging.getLogger(__name__) # does a lot more logging when set to true verbose = True +""" +list of all valid priorities, inferred from the global +default_desinations section of the config +""" +priority_list = set() + +""" +Instantiated to a list of all valid destinations in the job configuration file +if run directly to validate configs. Otherwise, remains None. We often check +to see if app is None, because if it is then we'll try using the +destination_list instead. +-""" +destination_list = set() + +""" +The largest the edit distance can be for a word to be considered +A correction for another word. +""" +max_edit_dist = 2 + +""" +List of valid categories that can be expected in the configuration. +""" +valid_categories = ['verbose', 'tools', 'default_destination', + 'users', 'default_priority'] + +# --- destination validation error messages --- # +dest_err_default_dest = "Default destination '%s' does not appear in the job configuration." # destination +dest_err_tool_default_dest = "Default destination for '%s': '%s' does not appear in the job configuration." # tool, destination +dest_err_tool_rule_dest = "Destination for '%s', rule %s: '%s' does not exist in job configuration." # tool, counter, destination + class MalformedYMLException(Exception): pass @@ -41,60 +75,63 @@ def get_keys_from_dict(dl, keys_list): class RuleValidator(object): """ - This class is the primary facility for validating configs. It's always called - in map_tool_to_destination and it's called for validating config directly through - DynamicToolDestination.py + This class is the primary facility for validating configs. It's always + called in map_tool_to_destination and it's called for validating config + directly through DynamicToolDestination.py """ @classmethod - def validate_rule(cls, rule_type, return_bool=False, *args, **kwargs): + def validate_rule(cls, rule_type, app, return_bool=False, *args, **kwargs): """ - This function is responsible for passing each rule to its relevant function. + This function is responsible for passing each rule to its relevant + function. @type rule_type: str @param rule_type: the current rule's type @type return_bool: bool - @param return_bool: True when we are only interested in the result of the - validation, and not the validated rule itself. + @param return_bool: True when we are only interested in the result of + the validation, and not the validated rule itself. @rtype: bool, dict (depending on return_bool) - @return: validated rule or result of validation (depending on return_bool) + @return: validated rule or result of validation (depending on + return_bool) """ if rule_type == 'file_size': - return cls.__validate_file_size_rule(return_bool, *args, **kwargs) + return cls.__validate_file_size_rule(app, return_bool, *args, **kwargs) elif rule_type == 'num_input_datasets': - return cls.__validate_num_input_datasets_rule(return_bool, *args, **kwargs) + return cls.__validate_num_input_datasets_rule(app, return_bool, *args, **kwargs) elif rule_type == 'records': - return cls.__validate_records_rule(return_bool, *args, **kwargs) + return cls.__validate_records_rule(app, return_bool, *args, **kwargs) elif rule_type == 'arguments': - return cls.__validate_arguments_rule(return_bool, *args, **kwargs) + return cls.__validate_arguments_rule(app, return_bool, *args, **kwargs) @classmethod def __validate_file_size_rule( - cls, return_bool, original_rule, counter, tool): + cls, app, return_bool, original_rule, counter, tool): """ This function is responsible for validating 'file_size' rules. @type return_bool: bool - @param return_bool: True when we are only interested in the result of the - validation, and not the validated rule itself. + @param return_bool: True when we are only interested in the result of + the validation, and not the validated rule itself. @type original_rule: dict @param original_rule: contains the original received rule @type counter: int - @param counter: this counter is used to identify what rule # is currently being - validated. Necessary for log output. + @param counter: this counter is used to identify what rule # is + currently being validated. Necessary for log output. @type tool: str @param tool: the name of the current tool. Necessary for log output. @rtype: bool, dict (depending on return_bool) - @return: validated rule or result of validation (depending on return_bool) + @return: validated rule or result of validation (depending on + return_bool) """ rule = copy.deepcopy(original_rule) @@ -113,7 +150,7 @@ class RuleValidator(object): # Destination Verification # if rule is not None: valid_rule, rule = cls.__validate_destination( - valid_rule, return_bool, rule, tool, counter) + valid_rule, app, return_bool, rule, tool, counter) # Bounds Verification # if rule is not None: @@ -128,26 +165,27 @@ class RuleValidator(object): @classmethod def __validate_num_input_datasets_rule( - cls, return_bool, original_rule, counter, tool): + cls, app, return_bool, original_rule, counter, tool): """ This function is responsible for validating 'num_input_datasets' rules. @type return_bool: bool - @param return_bool: True when we are only interested in the result of the - validation, and not the validated rule itself. + @param return_bool: True when we are only interested in the result of + the validation, and not the validated rule itself. @type original_rule: dict @param original_rule: contains the original received rule @type counter: int - @param counter: this counter is used to identify what rule # is currently being - validated. Necessary for log output. + @param counter: this counter is used to identify what rule # is + currently being validated. Necessary for log output. @type tool: str @param tool: the name of the current tool. Necessary for log output. @rtype: bool, dict (depending on return_bool) - @return: validated rule or result of validation (depending on return_bool) + @return: validated rule or result of validation (depending on + return_bool) """ rule = copy.deepcopy(original_rule) @@ -166,7 +204,7 @@ class RuleValidator(object): # Destination Verification # if rule is not None: valid_rule, rule = cls.__validate_destination( - valid_rule, return_bool, rule, tool, counter) + valid_rule, app, return_bool, rule, tool, counter) # Bounds Verification # if rule is not None: @@ -180,26 +218,27 @@ class RuleValidator(object): return rule @classmethod - def __validate_records_rule(cls, return_bool, original_rule, counter, tool): + def __validate_records_rule(cls, app, return_bool, original_rule, counter, tool): """ This function is responsible for validating 'records' rules. @type return_bool: bool - @param return_bool: True when we are only interested in the result of the - validation, and not the validated rule itself. + @param return_bool: True when we are only interested in the result of + the validation, and not the validated rule itself. @type original_rule: dict @param original_rule: contains the original received rule @type counter: int - @param counter: this counter is used to identify what rule # is currently being - validated. Necessary for log output. + @param counter: this counter is used to identify what rule # is + currently being validated. Necessary for log output. @type tool: str @param tool: the name of the current tool. Necessary for log output. @rtype: bool, dict (depending on return_bool) - @return: validated rule or result of validation (depending on return_bool) + @return: validated rule or result of validation (depending on + return_bool) """ rule = copy.deepcopy(original_rule) @@ -218,7 +257,7 @@ class RuleValidator(object): # Destination Verification # if rule is not None: valid_rule, rule = cls.__validate_destination( - valid_rule, return_bool, rule, tool, counter) + valid_rule, app, return_bool, rule, tool, counter) # Bounds Verification # if rule is not None: @@ -233,26 +272,27 @@ class RuleValidator(object): @classmethod def __validate_arguments_rule( - cls, return_bool, original_rule, counter, tool): + cls, app, return_bool, original_rule, counter, tool): """ This is responsible for validating 'arguments' rules. @type return_bool: bool - @param return_bool: True when we are only interested in the result of the - validation, and not the validated rule itself. + @param return_bool: True when we are only interested in the result of + the validation, and not the validated rule itself. @type original_rule: dict @param original_rule: contains the original received rule @type counter: int - @param counter: this counter is used to identify what rule # is currently being - validated. Necessary for log output. + @param counter: this counter is used to identify what rule # is + currently being validated. Necessary for log output. @type tool: str @param tool: the name of the current tool. Necessary for log output. @rtype: bool, dict (depending on return_bool) - @return: validated rule or result of validation (depending on return_bool) + @return: validated rule or result of validation (depending on + return_bool) """ rule = copy.deepcopy(original_rule) @@ -271,7 +311,7 @@ class RuleValidator(object): # Destination Verification # if rule is not None: valid_rule, rule = cls.__validate_destination( - valid_rule, return_bool, rule, tool, counter) + valid_rule, app, return_bool, rule, tool, counter) # Arguments Verification (for rule_type arguments; read comment block at top # of function for clarification. @@ -291,19 +331,19 @@ class RuleValidator(object): This function is responsible for validating nice_value. @type return_bool: bool - @param return_bool: True when we are only interested in the result of the - validation, and not the validated rule itself. + @param return_bool: True when we are only interested in the result of + the validation, and not the validated rule itself. @type valid_rule: bool - @param valid_rule: returns True if everything is valid. False if it encounters any - abnormalities in the config. + @param valid_rule: returns True if everything is valid. False if it + encounters any abnormalities in the config. @type original_rule: dict @param original_rule: contains the original received rule @type counter: int - @param counter: this counter is used to identify what rule # is currently being - validated. Necessary for log output. + @param counter: this counter is used to identify what rule # is + currently being validated. Necessary for log output. @type tool: str @param tool: the name of the current tool. Necessary for log output. @@ -326,8 +366,8 @@ class RuleValidator(object): valid_rule = False else: - error = "No nice_value found for rule " + str(counter) + " in '" + str(tool) - error += "'." + error = "No nice_value found for rule " + str(counter) + " in '" + error += str(tool) + "'." if not return_bool: error += " Setting nice_value to 0." rule["nice_value"] = 0 @@ -338,24 +378,24 @@ class RuleValidator(object): return valid_rule, rule @classmethod - def __validate_destination(cls, valid_rule, return_bool, rule, tool, counter): + def __validate_destination(cls, valid_rule, app, return_bool, rule, tool, counter): """ This function is responsible for validating destination. @type return_bool: bool - @param return_bool: True when we are only interested in the result of the - validation, and not the validated rule itself. + @param return_bool: True when we are only interested in the result of + the validation, and not the validated rule itself. @type valid_rule: bool - @param valid_rule: returns True if everything is valid. False if it encounters any - abnormalities in the config. + @param valid_rule: returns True if everything is valid. False if it + encounters any abnormalities in the config. - @type original_rule: dict - @param original_rule: contains the original received rule + @type rule: dict + @param rule: contains the original received rule @type counter: int - @param counter: this counter is used to identify what rule # is currently being - validated. Necessary for log output. + @param counter: this counter is used to identify what rule # is + currently being validated. Necessary for log output. @type tool: str @param tool: the name of the current tool. Necessary for log output. @@ -378,6 +418,7 @@ class RuleValidator(object): rule["destination"] = "fail" if "destination" in rule: + suggestion = None if isinstance(rule["destination"], str): if rule["destination"] == "fail" and "fail_message" not in rule: error = "Missing a fail_message for rule " + str(counter) @@ -390,35 +431,48 @@ class RuleValidator(object): if verbose: log.debug(error) valid_rule = False - elif isinstance(rule["destination"], dict): - if ("priority" in rule["destination"] and isinstance(rule["destination"]["priority"], dict)): - if "med" not in rule["destination"]["priority"]: - error = "No 'med' priority destination for rule " + str(counter) - error += " in '" + str(tool) + "'." - if not return_bool: - error += " Ignoring..." - if verbose: - log.debug(error) + else: + is_valid = validate_destination(app, rule["destination"], + dest_err_tool_rule_dest, (tool, counter, rule["destination"]), + return_bool) + if not is_valid: valid_rule = False - else: - for priority in rule["destination"]["priority"]: - if priority not in ["low", "med", "high"]: - error = "Invalid priority destination '" + str(priority) - error += "' for rule " + str(counter) - error += " in '" + str(tool) + "'." - if not return_bool: - error += " Ignoring..." - if verbose: - log.debug(error) - valid_rule = False - elif not isinstance(rule["destination"]["priority"][priority], str): - error = "No '" + str(priority) - error += "'priority destination for rule " + str(counter) - error += " in '" + str(tool) + "'." - if not return_bool: - error += " Ignoring..." - if verbose: - log.debug(error) + elif isinstance(rule["destination"], dict): + if ("priority" in rule["destination"] + and isinstance(rule["destination"]["priority"], dict)): + + for priority in rule["destination"]["priority"]: + if priority not in priority_list: + error = "Invalid priority '" + error += str(priority) + "' for rule " + error += str(counter) + " in '" + str(tool) + "'." + suggestion = get_typo_correction(priority, + priority_list, max_edit_dist) + if suggestion: + error += " Did you mean '" + str(suggestion) + "'?" + if not return_bool: + error += " Ignoring..." + if verbose: + log.debug(error) + valid_rule = False + + elif not isinstance(rule["destination"]["priority"][priority], str): + error = "Cannot parse tool destination '" + error += str(rule["destination"]["priority"][priority]) + error += "' for rule " + str(counter) + error += " in '" + str(tool) + "'." + if not return_bool: + error += " Ignoring..." + if verbose: + log.debug(error) + valid_rule = False + else: + is_valid = validate_destination(app, + rule["destination"]["priority"][priority], + dest_err_tool_rule_dest, + (tool, counter, rule["destination"]["priority"][priority]), + return_bool) + if not is_valid: valid_rule = False else: error = "No destination specified for rule " + str(counter) @@ -453,19 +507,19 @@ class RuleValidator(object): This function is responsible for validating bounds. @type return_bool: bool - @param return_bool: True when we are only interested in the result of the - validation, and not the validated rule itself. + @param return_bool: True when we are only interested in the result of + the validation, and not the validated rule itself. @type valid_rule: bool - @param valid_rule: returns True if everything is valid. False if it encounters any - abnormalities in the config. + @param valid_rule: returns True if everything is valid. False if it + encounters any abnormalities in the config. @type original_rule: dict @param original_rule: contains the original received rule @type counter: int - @param counter: this counter is used to identify what rule # is currently being - validated. Necessary for log output. + @param counter: this counter is used to identify what rule # is + currently being validated. Necessary for log output. @type tool: str @param tool: the name of the current tool. Necessary for log output. @@ -483,8 +537,8 @@ class RuleValidator(object): lower_bound = rule["lower_bound"] if lower_bound == "Infinity": - error = "Error: lower_bound is set to Infinity, but must be lower than " - error += "upper_bound!" + error = "Error: lower_bound is set to Infinity, but must be " + error += "lower than upper_bound!" if not return_bool: error += " Setting lower_bound to 0!" lower_bound = 0 @@ -528,19 +582,19 @@ class RuleValidator(object): This function is responsible for validating arguments. @type return_bool: bool - @param return_bool: True when we are only interested in the result of the - validation, and not the validated rule itself. + @param return_bool: True when we are only interested in the result of + the validation, and not the validated rule itself. @type valid_rule: bool - @param valid_rule: returns True if everything is valid. False if it encounters any - abnormalities in the config. + @param valid_rule: returns True if everything is valid. False if it + encounters any abnormalities in the config. @type original_rule: dict @param original_rule: contains the original received rule @type counter: int - @param counter: this counter is used to identify what rule # is currently being - validated. Necessary for log output. + @param counter: this counter is used to identify what rule # is + currently being validated. Necessary for log output. @type tool: str @param tool: the name of the current tool. Necessary for log output. @@ -567,19 +621,19 @@ class RuleValidator(object): This function is responsible for validating users (if present). @type return_bool: bool - @param return_bool: True when we are only interested in the result of the - validation, and not the validated rule itself. + @param return_bool: True when we are only interested in the result of + the validation, and not the validated rule itself. @type valid_rule: bool - @param valid_rule: returns True if everything is valid. False if it encounters any - abnormalities in the config. + @param valid_rule: returns True if everything is valid. False if it + encounters any abnormalities in the config. @type original_rule: dict @param original_rule: contains the original received rule @type counter: int - @param counter: this counter is used to identify what rule # is currently being - validated. Necessary for log output. + @param counter: this counter is used to identify what rule # is + currently being validated. Necessary for log output. @type tool: str @param tool: the name of the current tool. Necessary for log output. @@ -595,8 +649,8 @@ class RuleValidator(object): for user in reversed(rule["users"]): if not isinstance(user, str): error = "Entry '" + str(user) + "' in users for rule " - error += str(counter) + " in tool '" + str(tool) + "' is in an " - error += "invalid format!" + error += str(counter) + " in tool '" + str(tool) + error += "' is in an " + "invalid format!" if not return_bool: error += " Ignoring entry." if verbose: @@ -606,9 +660,9 @@ class RuleValidator(object): else: if re.match(emailregex, user) is None: - error = "Supplied email '" + str(user) + "' for rule " - error += str(counter) + " in tool '" + str(tool) + "' is in " - error += "an invalid format!" + error = "Supplied email '" + str(user) + error += "' for rule " + str(counter) + " in tool '" + error += str(tool) + "' is in " + "an invalid format!" if not return_bool: error += " Ignoring email." if verbose: @@ -640,12 +694,17 @@ class RuleValidator(object): return valid_rule, rule -def parse_yaml(path="/config/tool_destinations.yml", test=False, return_bool=False): +def parse_yaml(path="/config/tool_destinations.yml", + job_conf_path="/config/job_conf.xml", app=None, test=False, + return_bool=False): """ Get a yaml file from path and send it to validate_config for validation. @type path: str - @param path: the path to the config file + @param path: the path to the tool destinations config file + + @type job_conf_path: str + @param job_conf_path: the path to the job config file @type test: bool @param test: indicates whether to run in test mode or production mode @@ -658,6 +717,11 @@ def parse_yaml(path="/config/tool_destinations.yml", test=False, return_bool=Fal @return: validated rule or result of validation (depending on return_bool) """ + + if app is None: + global destination_list + destination_list = get_destination_list_from_job_config(job_conf_path) + # Import file from path try: if test: @@ -680,9 +744,9 @@ def parse_yaml(path="/config/tool_destinations.yml", test=False, return_bool=Fal # Test imported file try: if return_bool: - valid_config = validate_config(config, return_bool) + valid_config = validate_config(config, app, return_bool) else: - config = validate_config(config) + config = validate_config(config, app) except MalformedYMLException: if verbose: log.error(str(sys.exc_value)) @@ -699,7 +763,63 @@ def parse_yaml(path="/config/tool_destinations.yml", test=False, return_bool=Fal return config -def validate_config(obj, return_bool=False): +def validate_destination(app, destination, err_message, err_message_contents, + return_bool=True): + """ + Validate received destination id. + + @type app: + @param app: Current app + + @type destination: str + @param destination: string containing the destination id that is being + validated + + @type err_message: str + @param err_message: Error message to be formatted with the contents of + `err_message_contents` upon the event of invalid + destination + + @type err_message_contents: tuple + @param err_message_contents: A tuple of strings to be placed in + `err_message` + + @type return_bool: bool + @param return_bool: Whether or not the calling function has been told to + return a boolean value or not. Determines whether or + not to print 'Ignoring...' after error messages. + + @rtype: bool + @return: True if the destination is valid and False otherwise. + """ + + valid_destination = False + suggestion = None + + if destination is 'fail' and err_message is dest_err_tool_rule_dest: # It's a tool rule that is set to fail. It's valid + valid_destination = True + elif app is None: + if destination in destination_list: + valid_destination = True + else: + suggestion = get_typo_correction(destination, + destination_list, max_edit_dist) + elif app.job_config.get_destination(destination): + valid_destination = True + + if not valid_destination: + error = err_message % err_message_contents + if suggestion: + error += " Did you mean '" + suggestion + "'?" + if not return_bool: + error += " Ignoring..." + if verbose: + log.debug(error) + + return valid_destination + + +def validate_config(obj, app=None, return_bool=False,): """ Validate received config. @@ -714,6 +834,9 @@ def validate_config(obj, return_bool=False): @return: validated rule or result of validation (depending on return_bool) """ + global priority_list + priority_list = set() + def infinite_defaultdict(): return collections.defaultdict(infinite_defaultdict) @@ -728,11 +851,13 @@ def validate_config(obj, return_bool=False): if return_bool: verbose = True - elif obj is not None and 'verbose' in obj and isinstance(obj['verbose'], bool): verbose = obj['verbose'] else: valid_config = False + if obj: + log.debug("Verbose value '" + str(obj['verbose']) + "' is not True or False! Falling back to verbose...") + verbose = True if not return_bool and verbose: log.debug("Running config validation...") @@ -745,43 +870,78 @@ def validate_config(obj, return_bool=False): available_rule_types = ['file_size', 'num_input_datasets', 'records', 'arguments'] if obj is not None: - # in obj, there should always be only 4 categories: tools, default_destination, - # users, and verbose + # in obj, there should always be only 5 categories: tools, default_destination, + # default_priority, users, and verbose if 'default_destination' in obj: + suggestion = None if isinstance(obj['default_destination'], str): - new_config["default_destination"] = obj['default_destination'] + is_valid = validate_destination(app, obj['default_destination'], + dest_err_default_dest, + (obj['default_destination'])) + if is_valid: + new_config["default_destination"] = obj['default_destination'] + else: + valid_config = False + elif isinstance(obj['default_destination'], dict): + if ('priority' in obj['default_destination'] and isinstance(obj['default_destination']['priority'], dict)): - if 'med' not in obj['default_destination']['priority']: - error = "No default 'med' priority destination!" + + for priority in obj['default_destination']['priority']: + if isinstance(obj['default_destination']['priority'][priority], + str): + priority_list.add(priority) + is_valid = validate_destination( + app, obj['default_destination']['priority'][priority], + dest_err_default_dest, + (obj['default_destination']['priority'][priority])) + + if is_valid: + new_config["default_destination"]['priority'][priority] = ( + obj['default_destination']['priority'][priority]) + else: + valid_config = False + if len(priority_list) < 1: + error = ("No valid priorities found!") if verbose: log.debug(error) valid_config = False else: - for priority in obj['default_destination']['priority']: - if priority in ['low', 'med', 'high']: - if isinstance( - obj['default_destination']['priority'][priority], - str): - new_config['default_destination']['priority'][ - priority] = obj[ - 'default_destination']['priority'][priority] + if 'default_priority' in obj: + if isinstance(obj['default_priority'], str): + if obj['default_priority'] in priority_list: + new_config['default_priority'] = obj['default_priority'] else: - error = ("No default '" + str(priority) + - "' priority destination in config!") + error = ("Default priority '" + str(obj['default_priority']) + + "' is not a valid priority.") + suggestion = get_typo_correction(obj['default_priority'], + priority_list, max_edit_dist) + if suggestion: + error += " Did you mean '" + str(suggestion) + "'?" if verbose: log.debug(error) - valid_config = False else: - error = ("Invalid default priority destination '" + - str(priority) + "' found in config!") + error = "default_priority in config is not valid." if verbose: log.debug(error) valid_config = False + else: + error = "No default_priority section found in config." + if 'med' in priority_list: + # set 'med' as fallback default priority, so + # old tool_destination.yml configs still work + error += " Setting 'med' as default priority." + new_config['default_priority'] = 'med' + else: + error += " Things may not run as expected!" + valid_config = False + if verbose: + log.debug(error) + else: - error = "No default priority destinations specified in config!" + error = "No global default destinations specified in config!" if verbose: log.debug(error) valid_config = False @@ -790,6 +950,7 @@ def validate_config(obj, return_bool=False): if verbose: log.debug(error) valid_config = False + else: error = "No global default destination specified in config!" if verbose: @@ -803,11 +964,17 @@ def validate_config(obj, return_bool=False): if isinstance(curr, dict): if 'priority' in curr and isinstance(curr['priority'], str): - if curr['priority'] in ['low', 'med', 'high']: + + if curr['priority'] in priority_list: new_config['users'][user]['priority'] = curr['priority'] else: - error = ("User '" + user + "', priority is not valid!" + - " Must be either low, med, or high.") + error = ("User '" + user + "', priority '" + + str(curr['priority']) + "' is not defined " + + "in the global default_destination section") + suggestion = get_typo_correction(curr['priority'], + priority_list, max_edit_dist) + if suggestion: + error += " Did you mean '" + str(suggestion) + "'?" if verbose: log.debug(error) valid_config = False @@ -841,41 +1008,57 @@ def validate_config(obj, return_bool=False): # in each tool, there should always be only 2 sub-categories: # default_destination (not mandatory) and rules (mandatory) if "default_destination" in curr: + suggestion = None if isinstance(curr['default_destination'], str): - new_config['tools'][tool]['default_destination'] = (curr['default_destination']) - tool_has_default = True + is_valid = validate_destination(app, + curr['default_destination'], + dest_err_tool_default_dest, + (tool, curr['default_destination'])) + if is_valid: + new_config['tools'][tool]['default_destination'] = ( + (curr['default_destination'])) + tool_has_default = True + else: + valid_config = False elif isinstance(curr['default_destination'], dict): - if ('priority' in curr['default_destination'] and isinstance(curr['default_destination']['priority'], dict)): - if ('med' not in curr['default_destination']['priority']): - error = "No default 'med' priority destination " - error += "for " + str(tool) + "!" - if verbose: - log.debug(error) - valid_config = False - else: - for priority in curr['default_destination']['priority']: - destination = curr['default_destination']['priority'][priority] - if priority in ['low', 'med', 'high']: - if isinstance(destination, str): + + if ('priority' in curr['default_destination'] + and isinstance(curr['default_destination']['priority'], dict)): + + for priority in curr['default_destination']['priority']: + destination = curr['default_destination']['priority'][priority] + if priority in priority_list: + if isinstance(destination, str): + + is_valid = validate_destination( + app, destination, + dest_err_tool_default_dest, + (tool, curr['default_destination']['priority'][priority])) + if is_valid: new_config['tools'][tool]['default_destination']['priority'][priority] = destination tool_has_default = True else: - error = ("No default '" + - str(priority) + - "' priority destination " + - "for " + str(tool) + - " in config!") - if verbose: - log.debug(error) valid_config = False + else: - error = ("Invalid default priority " + - "destination '" + str(priority) + - "' for " + str(tool) + - "found in config!") + error = ("No default '" + str(priority) + + "' priority destination for tool " + + str(tool) + " in config!") if verbose: log.debug(error) valid_config = False + + else: + error = ("Invalid default destination priority '" + + str(priority) + "' for '" + str(tool) + + "'.") + suggestion = get_typo_correction(priority, + priority_list, max_edit_dist) + if suggestion: + error += " Did you mean '" + str(suggestion) + "'?" + if verbose: + log.debug(error) + valid_config = False else: error = "No default priority destinations specified" error += " for " + str(tool) + " in config!" @@ -899,7 +1082,7 @@ def validate_config(obj, return_bool=False): # result if return_bool: valid_rule = RuleValidator.validate_rule( - rule['rule_type'], return_bool, + rule['rule_type'], app, return_bool, rule, counter, tool) # otherwise, retrieve the processed rule @@ -907,7 +1090,7 @@ def validate_config(obj, return_bool=False): validated_rule = ( RuleValidator.validate_rule( rule['rule_type'], - return_bool, + app, return_bool, rule, counter, tool)) # if the result we get is False, then @@ -947,8 +1130,8 @@ def validate_config(obj, return_bool=False): # if "rules" in curr and isinstance(curr['rules'], list): elif not tool_has_default: valid_config = False - error = "Tool '" + str(tool) + "' does not have rules nor a" - error += " default_destination!" + error = "Tool '" + str(tool) + "' does not have" + error += " rules nor a default_destination!" if verbose: log.debug(error) @@ -972,8 +1155,7 @@ def validate_config(obj, return_bool=False): # quickly run through categories to detect unrecognized types for category in obj.keys(): - if not (category == 'verbose' or category == 'tools' or - category == 'default_destination' or category == 'users'): + if category not in valid_categories: error = "Unrecognized category '" + category error += "' found in config file!" if verbose: @@ -1103,7 +1285,7 @@ def importer(test): global JobDestination global JobMappingException if test: - class JobDestionation(object): + class JobDestination(object): def __init__(self, *kwd): self.id = kwd.get('id') self.nativeSpec = kwd.get('params')['nativeSpecification'] @@ -1115,7 +1297,7 @@ def importer(test): def map_tool_to_destination( - job, app, tool, user_email, test=False, path=None): + job, app, tool, user_email, test=False, path=None, job_conf_path=None): """ Dynamically allocate resources @@ -1128,23 +1310,28 @@ def map_tool_to_destination( @type path: str @param path: path to tool_destinations.yml + + @type job_conf_path: str + @param job_conf_path: path to job_conf.xml """ importer(test) - # set verbose to True by default, just in case (some tests fail without this due to - # how the tests apparently work) + # set verbose to True by default, just in case (some tests fail without + # this due to how the tests apparently work) global verbose verbose = True filesize_rule_present = False num_input_datasets_rule_present = False records_rule_present = False - # Get configuration from tool_destinations.yml + # Get configuration from tool_destinations.yml and job_conf.xml if path is None: path = app.config.tool_destinations_config_file + if job_conf_path is None: + job_conf_path = app.config.job_config_file try: - config = parse_yaml(path) + config = parse_yaml(path, job_conf_path, app) except MalformedYMLException as e: raise JobMappingException(e) @@ -1176,7 +1363,7 @@ def map_tool_to_destination( records += 1 except NameError: pass - # Loop through each input file and adds the size to the total + # Loops through each input file and adds the size to the total # or looks through db for records for da in inp_data: try: @@ -1225,12 +1412,55 @@ def map_tool_to_destination( # For each different rule for the tool that's running fail_message = None - # set default priority to med - default_priority = 'med' - priority = default_priority + if fail_message is not None: + destination = "fail" + elif config is not None: - if config is not None: - # get the users priority + # Get the default priority from the config if necessary. + # If there isn't one, choose an arbitrary one as a fallback + if "default_destination" in config: + if isinstance(config['default_destination'], dict): + if 'default_priority' in config: + default_priority = config['default_priority'] + priority = default_priority + + else: + if len(priority_list) > 0: + default_priority = next(iter(priority_list)) + priority = default_priority + error = ("No default priority found, arbitrarily setting '" + + default_priority + "' as the default priority." + + " Things may not work as expected!") + if verbose: + log.debug(error) + + # fetch priority information from workflow/job parameters + job_parameter_list = job.get_parameters() + workflow_params = None + job_params = None + if job_parameter_list is not None: + for param in job_parameter_list: + if param.name == "__workflow_resource_params__": + workflow_params = param.value + if param.name == "__job_resource": + job_params = param.value + + # Priority coming from workflow invocation takes precedence over job specific priorities + if workflow_params is not None: + resource_params = json.loads(workflow_params) + if 'priority' in resource_params: + # For by_group mapping, this priority has already been validated when the + # request was created. + if resource_params['priority'] is not None: + priority = resource_params['priority'] + + elif job_params is not None: + resource_params = json.loads(job_params) + if 'priority' in resource_params: + if resource_params['priority'] is not None: + priority = resource_params['priority'] + + # get the user's priority if "users" in config: if user_email in config["users"]: priority = config["users"][user_email]["priority"] @@ -1241,7 +1471,7 @@ def map_tool_to_destination( else: if priority in config['default_destination']['priority']: destination = config['default_destination']['priority'][priority] - else: + elif default_priority in config['default_destination']['priority']: destination = (config['default_destination']['priority'][default_priority]) config = config['tools'] if str(tool.old_id) in config: @@ -1303,7 +1533,7 @@ def map_tool_to_destination( matched = True # check if the args in the config file are available for arg in rule["arguments"]: - arg_dict = {arg : rule["arguments"][arg]} + arg_dict = {arg: rule["arguments"][arg]} arg_keys_list = [] get_keys_from_dict(arg_dict, arg_keys_list) try: @@ -1320,7 +1550,8 @@ def map_tool_to_destination( # if we matched a rule if matched: - if (matched_rule is None or rule["nice_value"] < matched_rule["nice_value"]): + if (matched_rule is None or rule["nice_value"] + < matched_rule["nice_value"]): matched_rule = rule # if user_authorized else: @@ -1346,16 +1577,18 @@ def map_tool_to_destination( else: if priority in default_tool_destination['priority']: destination = default_tool_destination['priority'][priority] - else: + elif default_priority in default_tool_destination['priority']: destination = (default_tool_destination['priority'][default_priority]) + # else global default destination is used else: if isinstance(matched_rule["destination"], str): destination = matched_rule["destination"] else: if priority in matched_rule["destination"]["priority"]: destination = matched_rule["destination"]["priority"][priority] - else: + elif default_priority in matched_rule["destination"]["priority"]: destination = (matched_rule["destination"]["priority"][default_priority]) + # else global default destination is used # if "default_destination" in config else: @@ -1363,7 +1596,8 @@ def map_tool_to_destination( fail_message = "Job '" + str(tool.old_id) + "' failed; " fail_message += "no global default destination specified in config!" - # if config is not None + # if fail_message is not None + # elif config is not None else: destination = "fail" fail_message = "No config file supplied!" @@ -1377,7 +1611,7 @@ def map_tool_to_destination( if config is not None: if destination == "fail": output = "An error occurred: " + fail_message - + log.debug(output) else: output = "Running '" + str(tool.old_id) + "' with '" output += destination + "'." @@ -1386,14 +1620,184 @@ def map_tool_to_destination( return destination +def get_destination_list_from_job_config(job_config_location): + """ + returns A list of all destination IDs declared in the job configuration + + @type job_config_location: str + @param job_config_location: The location of the job config file relative + to the galaxy root directory. If NoneType, defaults to + galaxy/config/job_conf.xml, + galaxy/config/job_conf.xml.sample_advanced, or + galaxy/config/job_conf.xml.sample_basic + (first one that exists) + + @rtype: list + @return: A list of all of the destination IDs declared in the job + configuration file. + """ + global destination_list + + # os.path.realpath gets the path of DynamicToolDestination.py + # and then os.path.join is used to go back four directories + + config_location = os.path.join( + os.path.dirname(os.path.realpath(__file__)), '../../..') + + if job_config_location: + local_path = re.compile('^/config/.+$') + if local_path.match(job_config_location): + job_config_location = config_location + job_config_location + else: # Pick one of the default ones + message = "* No job config specified, " + if os.path.isfile(config_location + "/config/job_conf.xml"): + job_config_location = config_location + "/config/job_conf.xml" + message += "using 'config/job_conf.xml'. *" + + elif os.path.isfile(config_location + + "/config/job_conf.xml.sample_advanced"): + job_config_location = (config_location + + "/config/job_conf.xml.sample_advanced") + message += "using 'config/job_conf.xml.sample_advanced'. *" + + elif os.path.isfile(config_location + + "/config/job_conf.xml.sample_basic"): + job_config_location = (config_location + + "/config/job_conf.xml.sample_basic") + message += "using 'config/job_conf.xml.sample_basic'. *" + else: + message += ("and no default job configs in 'config/'. " + + "Expect lots of failures. *") + + if verbose: + log.debug(message) + + if job_config_location: + job_conf = ET.parse(job_config_location) + + # Add all destination IDs from the job configuration xml file + for destination in job_conf.getroot().iter("destination"): + if isinstance(destination.get("id"), str): + destination_list.add(destination.get("id")) + + else: + error = "Destination ID '" + str(destination) + error += "' in job configuration file cannot be" + error += " parsed. Things may not work as expected!" + log.debug(error) + + return destination_list + + +def get_edit_distance(source, target): + """ + returns the edit distance (levenshtein distance) between two strings. + code from: + en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Levenshtein_distance + + @type str1: str + @param str1: The first string + + @type str2: str + @param str2: The second string + + @rtype: int + @return: The edit distance between str1 and str2 + """ + + if len(source) < len(target): + return get_edit_distance(target, source) + + # So now we have len(source) >= len(target). + if len(target) == 0: + return len(source) + + # We call tuple() to force strings to be used as sequences + # ('c', 'a', 't', 's') - numpy uses them as values by default. + source = np.array(tuple(source)) + target = np.array(tuple(target)) + + # We use a dynamic programming algorithm, but with the + # added optimization that we only need the last two rows + # of the matrix. + previous_row = np.arange(target.size + 1) + for s in source: + # Insertion (target grows longer than source): + current_row = previous_row + 1 + + # Substitution or matching: + # Target and source items are aligned, and either + # are different (cost of 1), or are the same (cost of 0). + current_row[1:] = np.minimum( + current_row[1:], + np.add(previous_row[:-1], target != s)) + + # Deletion (target grows shorter than source): + current_row[1:] = np.minimum( + current_row[1:], + current_row[0:-1] + 1) + + previous_row = current_row + + return previous_row[-1] + + +def get_typo_correction(typo_str, word_set, max_dist): + """ + returns the string in a set that closest matches the + input string, as long as the edit distance between them + is equal to or smaller than a value, or the words are + the same when case is not considered. If there are no + appropriate matches, nothing is returned instead. + + @type typo_str: str + @param typo_str: The string to be compared + + @type word_set: set of str + @param word_set: The set of strings to compare to + + @type max_dist: int + @param max_dist: the largest allowed edit distance between + the word and the result. If nothing is + within this range, nothing is returned + + @rtype: str or NoneType + @return: The closest matching string, or None, if no strings + being compared to are within max_dist edit distance. + """ + + # Start curr_best out as the largest + # edit distance we will tolerate plus one + curr_best = max_dist + 1 + suggestion = None + + for valid_word in word_set: + # If we've already found a best match, + # don't bother checking anything else. + if curr_best > 0: + if typo_str.lower() == valid_word.lower(): + # if something matches when case insensitive, + # it is automatically set as the best + suggestion = valid_word + curr_best = 0 + else: + edit_distance = get_edit_distance(typo_str, valid_word) + if edit_distance < curr_best: + suggestion = valid_word + curr_best = edit_distance + + return suggestion + + if __name__ == '__main__': """ - This function is responsible for running the app if directly run through the - commandline. It offers the ability to specify a config through the commandline - for checking whether or not it is a valid config. It's to be run from within Galaxy, - assuming it is installed correctly within the proper directories in Galaxy, and it - looks for the config file in galaxy/config/. It can also be run with a path pointing - to a config file if not being run directly from inside Galaxy install directory. + This function is responsible for running the app if directly run through + the commandline. It offers the ability to specify a config through the + commandline for checking whether or not it is a valid config. It's to be + run from within Galaxy, assuming it is installed correctly within the + proper directories in Galaxy, and it looks for the config file in + galaxy/config/. It can also be run with a path pointing to a config file if + not being run directly from inside Galaxy install directory. """ verbose = True @@ -1402,9 +1806,14 @@ if __name__ == '__main__': parser.add_argument( '-c', '--check-config', dest='check_config', nargs='?', - help='Use this option to validate tool_destinations.yml.' + - ' Optionally, provide the path to the tool_destinations.yml' + - ' that you would like to check. Default: galaxy/config/tool_destinations.yml') + help='Use this option to validate tool_destinations.yml.' + + ' Optionally, provide the path to the tool_destinations.yml' + + ' that you would like to check, and/or the path to the related' + + ' job_conf.xml. Default: galaxy/config/tool_destinations.yml' + + 'and galaxy/config/job_conf.xml') + + parser.add_argument( + '-j', '--job-config', dest='job_config') parser.add_argument( '-V', '--version', action='version', version="%(prog)s " + __version__) @@ -1416,11 +1825,16 @@ if __name__ == '__main__': parser.print_help() sys.exit(1) - if args.check_config: - valid_config = parse_yaml(path=args.check_config, return_bool=True) + job_config_location = args.job_config + if args.check_config: + valid_config = parse_yaml(path=args.check_config, + job_conf_path=job_config_location, + return_bool=True) else: - valid_config = parse_yaml(path="/config/tool_destinations.yml", return_bool=True) + valid_config = parse_yaml(path="/config/tool_destinations.yml", + job_conf_path=job_config_location, + return_bool=True) if valid_config: print("Configuration is valid!") diff --git a/lib/galaxy/jobs/mapper.py b/lib/galaxy/jobs/mapper.py index c9f8f0da874..bb26d802779 100644 --- a/lib/galaxy/jobs/mapper.py +++ b/lib/galaxy/jobs/mapper.py @@ -137,6 +137,11 @@ class JobRunnerMapper(object): workflow_invocation_uuid = param_values.get("__workflow_invocation_uuid__", None) actual_args["workflow_invocation_uuid"] = workflow_invocation_uuid + if "workflow_resource_params" in function_arg_names: + param_values = job.raw_param_dict() + workflow_resource_params = param_values.get("__workflow_resource_params__", None) + actual_args["workflow_resource_params"] = workflow_resource_params + return expand_function(**actual_args) def __job_params(self, job): diff --git a/lib/galaxy/jobs/runners/util/job_script/MEMORY_STATEMENT.sh b/lib/galaxy/jobs/runners/util/job_script/MEMORY_STATEMENT.sh index fef76e5fff9..5d58124365c 100644 --- a/lib/galaxy/jobs/runners/util/job_script/MEMORY_STATEMENT.sh +++ b/lib/galaxy/jobs/runners/util/job_script/MEMORY_STATEMENT.sh @@ -1,4 +1,11 @@ if [ -n "$SLURM_JOB_ID" ]; then GALAXY_MEMORY_MB=`scontrol -do show job "$SLURM_JOB_ID" | sed 's/.*\( \|^\)Mem=\([0-9][0-9]*\)\( \|$\).*/\2/p;d'` 2>memory_statement.log fi + +if [ -z "$GALAXY_MEMORY_MB_PER_SLOT" -a -n "$GALAXY_MEMORY_MB" ]; then + GALAXY_MEMORY_MB_PER_SLOT=$(($GALAXY_MEMORY_MB / $GALAXY_SLOTS)) +elif [ -z "$GALAXY_MEMORY_MB" -a -n "$GALAXY_MEMORY_MB_PER_SLOT" ]; then + GALAXY_MEMORY_MB=$(($GALAXY_MEMORY_MB_PER_SLOT * $GALAXY_SLOTS)) +fi [ "${GALAXY_MEMORY_MB--1}" -gt 0 ] 2>>memory_statement.log && export GALAXY_MEMORY_MB || unset GALAXY_MEMORY_MB +[ "${GALAXY_MEMORY_MB_PER_SLOT--1}" -gt 0 ] 2>>memory_statement.log && export GALAXY_MEMORY_MB_PER_SLOT || unset GALAXY_MEMORY_MB_PER_SLOT diff --git a/lib/galaxy/managers/workflows.py b/lib/galaxy/managers/workflows.py index 18367a63663..432bfa92f39 100644 --- a/lib/galaxy/managers/workflows.py +++ b/lib/galaxy/managers/workflows.py @@ -36,6 +36,7 @@ from galaxy.workflow.modules import ( ToolModule, WorkflowModuleInjector ) +from galaxy.workflow.resources import get_resource_mapper_function from galaxy.workflow.steps import attach_ordered_steps from .base import decode_id @@ -205,6 +206,7 @@ class WorkflowContentsManager(UsesAnnotations): def __init__(self, app): self.app = app + self._resource_mapper_function = get_resource_mapper_function(app) def build_workflow_from_dict( self, @@ -432,14 +434,20 @@ class WorkflowContentsManager(UsesAnnotations): step_model['messages'] = step.upgrade_messages step_models.append(step_model) return { - 'id' : trans.app.security.encode_id(stored.id), - 'history_id' : trans.app.security.encode_id(trans.history.id) if trans.history else None, - 'name' : stored.name, - 'steps' : step_models, - 'step_version_changes' : step_version_changes, - 'has_upgrade_messages' : has_upgrade_messages + 'id': trans.app.security.encode_id(stored.id), + 'history_id': trans.app.security.encode_id(trans.history.id) if trans.history else None, + 'name': stored.name, + 'steps': step_models, + 'step_version_changes': step_version_changes, + 'has_upgrade_messages': has_upgrade_messages, + 'workflow_resource_parameters': self._workflow_resource_parameters(trans, stored, workflow), } + def _workflow_resource_parameters(self, trans, stored, workflow): + """Get workflow scheduling resource parameters for this user and workflow or None if unconfigured. + """ + return self._resource_mapper_function(trans=trans, stored_workflow=stored, workflow=workflow) + def _workflow_to_dict_editor(self, trans, stored): workflow = stored.latest_workflow # Pack workflow data into a dictionary and return diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 005de46f6e5..fa79e401d7b 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4150,6 +4150,16 @@ class WorkflowInvocation(UsesCreateAndUpdateTime, Dictifiable): request_to_content.workflow_step_id = step_id self.input_step_parameters.append(request_to_content) + @property + def resource_parameters(self): + resource_type = WorkflowRequestInputParameter.types.RESOURCE_PARAMETERS + _resource_parameters = {} + for input_parameter in self.input_parameters: + if input_parameter.type == resource_type: + _resource_parameters[input_parameter.name] = input_parameter.value + + return _resource_parameters + def has_input_for_step(self, step_id): for content in self.input_datasets: if content.workflow_step_id == step_id: @@ -4258,7 +4268,8 @@ class WorkflowRequestInputParameter(Dictifiable): dict_collection_visible_keys = ['id', 'name', 'value', 'type'] types = Bunch( REPLACEMENT_PARAMETERS='replacements', - META_PARAMETERS='meta', # + META_PARAMETERS='meta', + RESOURCE_PARAMETERS='resource', ) def __init__(self, name=None, value=None, type=None): diff --git a/lib/galaxy/model/orm/engine_factory.py b/lib/galaxy/model/orm/engine_factory.py index c9248dc8fca..3f3a5ce05b4 100644 --- a/lib/galaxy/model/orm/engine_factory.py +++ b/lib/galaxy/model/orm/engine_factory.py @@ -19,9 +19,8 @@ def build_engine(url, engine_options, database_query_profiling_proxy=False, trac else: proxy = None if slow_query_log_threshold or thread_local_log: - @event.listens_for(Engine, "before_cursor_execute") - def before_cursor_execute(conn, cursor, statement, - parameters, context, executemany): + @event.listens_for(Engine, "before_execute") + def before_execute(conn, clauseelement, multiparams, params): conn.info.setdefault('query_start_time', []).append(time.time()) @event.listens_for(Engine, "after_cursor_execute") diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 8144f8167df..1e0bee2e468 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -433,7 +433,9 @@ class DiskObjectStore(ObjectStore): if preserve_symlinks and os.path.islink(file_name): force_symlink(os.readlink(file_name), self.get_filename(obj, **kwargs)) else: - shutil.copy(file_name, self.get_filename(obj, **kwargs)) + path = self.get_filename(obj, **kwargs) + shutil.copy(file_name, path) + umask_fix_perms(path, self.config.umask, 0o666) except IOError as ex: log.critical('Error copying %s to %s: %s' % (file_name, self._get_filename(obj, **kwargs), ex)) raise ex diff --git a/lib/galaxy/security/__init__.py b/lib/galaxy/security/__init__.py index 4c832d4163c..26986c678e4 100644 --- a/lib/galaxy/security/__init__.py +++ b/lib/galaxy/security/__init__.py @@ -777,7 +777,7 @@ class GalaxyRBACAgent(RBACAgent): return role def get_role(self, name, type=None): - type = type or self.model.Role.types.ADMIN + type = type or self.model.Role.types.SYSTEM # will raise exception if not found return self.sa_session.query(self.model.Role) \ .filter(and_(self.model.Role.table.c.name == name, @@ -785,7 +785,7 @@ class GalaxyRBACAgent(RBACAgent): .one() def create_role(self, name, description, in_users, in_groups, create_group_for_role=False, type=None): - type = type or self.model.Role.types.ADMIN + type = type or self.model.Role.types.SYSTEM role = self.model.Role(name=name, description=description, type=type) self.sa_session.add(role) # Create the UserRoleAssociations diff --git a/lib/galaxy/security/passwords.py b/lib/galaxy/security/passwords.py index 0745d0d2665..644e57c6d7c 100644 --- a/lib/galaxy/security/passwords.py +++ b/lib/galaxy/security/passwords.py @@ -31,7 +31,7 @@ def check_password(guess, hashed): return True else: # Passwords were originally encoded with sha1 and hexed - if hashlib.sha1(guess).hexdigest() == hashed: + if safe_str_cmp(hashlib.sha1(guess).hexdigest(), hashed): return True # Password does not match return False diff --git a/lib/galaxy/security/validate_user_input.py b/lib/galaxy/security/validate_user_input.py index 5375a4afbb7..7150c0831e8 100644 --- a/lib/galaxy/security/validate_user_input.py +++ b/lib/galaxy/security/validate_user_input.py @@ -24,12 +24,12 @@ FILL_CHAR = '-' PASSWORD_MIN_LEN = 6 -def validate_email(trans, email, user=None, check_dup=True): +def validate_email(trans, email, user=None, check_dup=True, allow_empty=False): """ Validates the email format, also checks whether the domain is blacklisted in the disposable domains configuration. """ message = '' - if user and user.email == email: + if (user and user.email == email) or (email == "" and allow_empty): return message if not(VALID_EMAIL_RE.match(email)): message = "The format of the email address is not correct." diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index 4a686a107d4..dae851cd8cf 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -6,7 +6,7 @@ import socket import subprocess import tempfile from cgi import FieldStorage -from json import dumps +from json import dump, dumps from six import StringIO from sqlalchemy.orm import eagerload_all @@ -308,10 +308,8 @@ def create_paramfile(trans, uploaded_datasets): except Exception as e: log.warning('Changing ownership of uploaded file %s failed: %s' % (path, str(e))) - # TODO: json_file should go in the working directory - json_file = tempfile.mkstemp() - json_file_path = json_file[1] - json_file = os.fdopen(json_file[0], 'w') + tool_params = [] + json_file_path = None for uploaded_dataset in uploaded_datasets: data = uploaded_dataset.data if uploaded_dataset.type == 'composite': @@ -321,14 +319,14 @@ def create_paramfile(trans, uploaded_datasets): setattr(data.metadata, meta_name, meta_value) trans.sa_session.add(data) trans.sa_session.flush() - json = dict(file_type=uploaded_dataset.file_type, - dataset_id=data.dataset.id, - dbkey=uploaded_dataset.dbkey, - type=uploaded_dataset.type, - metadata=uploaded_dataset.metadata, - primary_file=uploaded_dataset.primary_file, - composite_file_paths=uploaded_dataset.composite_files, - composite_files=dict((k, v.__dict__) for k, v in data.datatype.get_composite_files(data).items())) + params = dict(file_type=uploaded_dataset.file_type, + dataset_id=data.dataset.id, + dbkey=uploaded_dataset.dbkey, + type=uploaded_dataset.type, + metadata=uploaded_dataset.metadata, + primary_file=uploaded_dataset.primary_file, + composite_file_paths=uploaded_dataset.composite_files, + composite_files=dict((k, v.__dict__) for k, v in data.datatype.get_composite_files(data).items())) else: try: is_binary = uploaded_dataset.datatype.is_binary @@ -352,31 +350,31 @@ def create_paramfile(trans, uploaded_datasets): user_ftp_dir = None if user_ftp_dir and uploaded_dataset.path.startswith(user_ftp_dir): uploaded_dataset.type = 'ftp_import' - json = dict(file_type=uploaded_dataset.file_type, - ext=uploaded_dataset.ext, - name=uploaded_dataset.name, - dataset_id=data.dataset.id, - dbkey=uploaded_dataset.dbkey, - type=uploaded_dataset.type, - is_binary=is_binary, - link_data_only=link_data_only, - uuid=uuid_str, - to_posix_lines=getattr(uploaded_dataset, "to_posix_lines", True), - auto_decompress=getattr(uploaded_dataset, "auto_decompress", True), - purge_source=purge_source, - space_to_tab=uploaded_dataset.space_to_tab, - run_as_real_user=trans.app.config.external_chown_script is not None, - check_content=trans.app.config.check_upload_content, - path=uploaded_dataset.path) + params = dict(file_type=uploaded_dataset.file_type, + ext=uploaded_dataset.ext, + name=uploaded_dataset.name, + dataset_id=data.dataset.id, + dbkey=uploaded_dataset.dbkey, + type=uploaded_dataset.type, + is_binary=is_binary, + link_data_only=link_data_only, + uuid=uuid_str, + to_posix_lines=getattr(uploaded_dataset, "to_posix_lines", True), + auto_decompress=getattr(uploaded_dataset, "auto_decompress", True), + purge_source=purge_source, + space_to_tab=uploaded_dataset.space_to_tab, + run_as_real_user=trans.app.config.external_chown_script is not None, + check_content=trans.app.config.check_upload_content, + path=uploaded_dataset.path) # TODO: This will have to change when we start bundling inputs. # Also, in_place above causes the file to be left behind since the # user cannot remove it unless the parent directory is writable. if link_data_only == 'copy_files' and trans.app.config.external_chown_script: _chown(uploaded_dataset.path) - json_file.write(dumps(json) + '\n') - json_file.close() - if trans.app.config.external_chown_script: - _chown(json_file_path) + tool_params.append(params) + with tempfile.NamedTemporaryFile(prefix='upload_params_', delete=False) as fh: + json_file_path = fh.name + dump(tool_params, fh) return json_file_path diff --git a/lib/galaxy/tools/data_fetch.py b/lib/galaxy/tools/data_fetch.py index 858d0c8d234..8a6f274889b 100644 --- a/lib/galaxy/tools/data_fetch.py +++ b/lib/galaxy/tools/data_fetch.py @@ -220,11 +220,11 @@ def _directory_to_items(directory): target = dir_elements[root] else: target = items - for dir in dirs: + for dir in sorted(dirs): dir_dict = {"name": dir, "elements": []} dir_elements[os.path.join(root, dir)] = dir_dict["elements"] target.append(dir_dict) - for file in files: + for file in sorted(files): target.append({"src": "path", "path": os.path.join(root, file)}) return items diff --git a/lib/galaxy/tools/execute.py b/lib/galaxy/tools/execute.py index cda1755aa62..d7dc35cb5cb 100644 --- a/lib/galaxy/tools/execute.py +++ b/lib/galaxy/tools/execute.py @@ -31,7 +31,7 @@ class PartialJobExecution(Exception): MappingParameters = collections.namedtuple("MappingParameters", ["param_template", "param_combinations"]) -def execute(trans, tool, mapping_params, history, rerun_remap_job_id=None, collection_info=None, workflow_invocation_uuid=None, invocation_step=None, max_num_jobs=None, job_callback=None, completed_jobs=None): +def execute(trans, tool, mapping_params, history, rerun_remap_job_id=None, collection_info=None, workflow_invocation_uuid=None, invocation_step=None, max_num_jobs=None, job_callback=None, completed_jobs=None, workflow_resource_parameters=None): """ Execute a tool and return object containing summary (output data, number of failures, etc...). @@ -58,7 +58,12 @@ def execute(trans, tool, mapping_params, history, rerun_remap_job_id=None, colle # Only workflow invocation code gets to set this, ignore user supplied # values or rerun parameters. del params['__workflow_invocation_uuid__'] - + if workflow_resource_parameters: + params['__workflow_resource_params__'] = workflow_resource_parameters + elif '__workflow_resource_params__' in params: + # Only workflow invocation code gets to set this, ignore user supplied + # values or rerun parameters. + del params['__workflow_resource_params__'] job, result = tool.handle_single_execution(trans, rerun_remap_job_id, execution_slice, history, execution_cache, completed_job) if job: message = EXECUTION_SUCCESS_MESSAGE % (tool.id, job.id, job_timer) diff --git a/lib/galaxy/tools/loader_directory.py b/lib/galaxy/tools/loader_directory.py index 146afd5de7c..4820f17f4b2 100644 --- a/lib/galaxy/tools/loader_directory.py +++ b/lib/galaxy/tools/loader_directory.py @@ -165,9 +165,9 @@ def looks_like_a_tool_xml(path): if(checkers.check_binary(full_path) or checkers.check_image(full_path) or - checkers.check_gzip(full_path)[0] or - checkers.check_bz2(full_path)[0] or - checkers.check_zip(full_path)): + checkers.is_gzip(full_path) or + checkers.is_bz2(full_path) or + checkers.is_zip(full_path)): return False with open(path, "r") as f: diff --git a/lib/galaxy/tools/parameters/output_collect.py b/lib/galaxy/tools/parameters/output_collect.py index 93bf92eee08..4a562e3651a 100644 --- a/lib/galaxy/tools/parameters/output_collect.py +++ b/lib/galaxy/tools/parameters/output_collect.py @@ -436,7 +436,7 @@ class JobContext(object): add_datasets_timer = ExecutionTimer() job.history.add_datasets(sa_session, [d for (ei, d) in element_datasets]) log.debug( - "(%s) Add dynamic collection datsets to history for output [%s] %s", + "(%s) Add dynamic collection datasets to history for output [%s] %s", self.job.id, name, add_datasets_timer, diff --git a/lib/galaxy/tools/verify/interactor.py b/lib/galaxy/tools/verify/interactor.py index 5623a7faa44..14b9563e78e 100644 --- a/lib/galaxy/tools/verify/interactor.py +++ b/lib/galaxy/tools/verify/interactor.py @@ -401,7 +401,7 @@ class GalaxyInteractorApi(object): def _summarize_history(self, history_id): if history_id is None: raise ValueError("_summarize_history passed empty history_id") - print("Problem in history with id %s - summary of datasets below." % history_id) + print("Problem in history with id %s - summary of history's datasets and jobs below." % history_id) try: history_contents = self.__contents(history_id) except Exception: @@ -417,6 +417,7 @@ class GalaxyInteractorApi(object): if history_content['history_content_type'] == 'dataset_collection': history_contents_json = self._get("histories/%s/contents/dataset_collections/%s" % (history_id, history_content["id"])).json() print("| Dataset Collection: %s" % history_contents_json) + print("|") continue try: @@ -440,7 +441,23 @@ class GalaxyInteractorApi(object): except Exception: print("| *TEST FRAMEWORK ERROR FETCHING JOB DETAILS*") print("|") - print(ERROR_MESSAGE_DATASET_SEP) + try: + jobs_json = self._get("jobs?history_id=%s" % history_id).json() + for job_json in jobs_json: + print(ERROR_MESSAGE_DATASET_SEP) + print("| Job %s" % job_json["id"]) + print("| State: ") + print(self.format_for_summary(job_json.get("state", ""), "Job state is unknown.")) + print("| Update Time:") + print(self.format_for_summary(job_json.get("update_time", ""), "Job update time is unknown.")) + print("| Create Time:") + print(self.format_for_summary(job_json.get("create_time", ""), "Job create time is unknown.")) + print("|") + print(ERROR_MESSAGE_DATASET_SEP) + except Exception: + print(ERROR_MESSAGE_DATASET_SEP) + print("*TEST FRAMEWORK FAILED TO FETCH HISTORY JOBS*") + print(ERROR_MESSAGE_DATASET_SEP) def format_for_summary(self, blob, empty_message, prefix="| "): contents = "\n".join(["%s%s" % (prefix, line.strip()) for line in StringIO(blob).readlines() if line.rstrip("\n\r")]) diff --git a/lib/galaxy/tools/xsd/galaxy.xsd b/lib/galaxy/tools/xsd/galaxy.xsd index 2497b7ad974..7651eb1fa20 100644 --- a/lib/galaxy/tools/xsd/galaxy.xsd +++ b/lib/galaxy/tools/xsd/galaxy.xsd @@ -141,9 +141,10 @@ hyperlink in the tool menu. - This string defaults to ``1.0.0`` if it is not -included in the tag. It allows for tool versioning and should be increased with each new version -of the tool. + This string allows for tool versioning +and should be increased with each new version of the tool. The value should +follow the [PEP 440](https://www.python.org/dev/peps/pep-0440/) specification. +It defaults to ``1.0.0`` if it is not included in the tag. @@ -2656,7 +2657,8 @@ be escaped with a backslash (``\``) when appearing in ``command`` or ``configfil Name | Description ---- | ----------- ``\${GALAXY_SLOTS:-4}`` | Number of cores/threads allocated by the job runner or resource manager to the tool for the given job (here 4 is the default number of threads to use if running via custom runner that does not configure GALAXY_SLOTS or in an older Galaxy runtime). -``\$GALAXY_MEMORY_MB`` | Amount of memory in megabytes (1024^2 bytes) allocated by the administrator (via the resource manager) to the tool for the given job. If unset, tools should not attempt to limit memory usage. +``\$GALAXY_MEMORY_MB`` | Total amount of memory in megabytes (1024^2 bytes) allocated by the administrator (via the resource manager) to the tool for the given job. If unset, tools should not attempt to limit memory usage. +``\$GALAXY_MEMORY_MB_PER_SLOT`` | Amount of memory per slot in megabytes (1024^2 bytes) allocated by the administrator (via the resource manager) to the tool for the given job. If unset, tools should not attempt to limit memory usage. See the [Planemo docs](https://planemo.readthedocs.io/en/latest/writing_advanced.html#cluster-usage) on the topic of ``GALAXY_SLOTS`` for more information and examples. diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index e9142c52980..4934de90d79 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -817,6 +817,22 @@ def xml_text(root, name=None): return '' +def parse_resource_parameters(resource_param_file): + """Code shared between jobs and workflows for reading resource parameter configuration files. + + TODO: Allow YAML in addition to XML. + """ + resource_parameters = {} + if os.path.exists(resource_param_file): + resource_definitions = parse_xml(resource_param_file) + resource_definitions_root = resource_definitions.getroot() + for parameter_elem in resource_definitions_root.findall("param"): + name = parameter_elem.get("name") + resource_parameters[name] = parameter_elem + + return resource_parameters + + # asbool implementation pulled from PasteDeploy truthy = frozenset(['true', 'yes', 'on', 'y', 't', '1']) falsy = frozenset(['false', 'no', 'off', 'n', 'f', '0']) diff --git a/lib/galaxy/util/checkers.py b/lib/galaxy/util/checkers.py index ce449517a3f..2d7eeefe493 100644 --- a/lib/galaxy/util/checkers.py +++ b/lib/galaxy/util/checkers.py @@ -1,9 +1,11 @@ import gzip import re import sys +import tarfile import zipfile from six import StringIO +from six.moves import filter from galaxy import util from galaxy.util.image_util import image_type @@ -52,19 +54,14 @@ def check_html(file_path, chunk=None): def check_binary(name, file_path=True): # Handles files if file_path is True or text if file_path is False - is_binary = False if file_path: temp = open(name, "U") else: temp = StringIO(name) try: - for char in temp.read(100): - if util.is_binary(char): - is_binary = True - break + return util.is_binary(temp.read(1024)) finally: temp.close() - return is_binary def check_gzip(file_path, check_content=True): @@ -124,10 +121,23 @@ def check_bz2(file_path, check_content=True): return (True, True) -def check_zip(file_path): - if zipfile.is_zipfile(file_path): - return True - return False +def check_zip(file_path, check_content=True, files=1): + if not zipfile.is_zipfile(file_path): + return (False, False) + + if not check_content: + return (True, True) + + CHUNK_SIZE = 2 ** 15 # 32Kb + chunk = None + for filect, member in enumerate(iter_zip(file_path)): + handle, name = member + chunk = handle.read(CHUNK_SIZE) + if chunk and check_html(file_path, chunk): + return (True, False) + if filect >= files: + break + return (True, True) def is_bz2(file_path): @@ -140,6 +150,28 @@ def is_gzip(file_path): return is_gzipped +def is_zip(file_path): + is_zipped, is_valid = check_zip(file_path, check_content=False) + return is_zipped + + +def is_single_file_zip(file_path): + for i, member in enumerate(iter_zip(file_path)): + if i > 1: + return False + return True + + +def is_tar(file_path): + return tarfile.is_tarfile(file_path) + + +def iter_zip(file_path): + with zipfile.ZipFile(file_path) as z: + for f in filter(lambda x: not x.endswith('/'), z.namelist()): + yield (z.open(f), f) + + def check_image(file_path): """ Simple wrapper around image_type to yield a True/False verdict """ if image_type(file_path): @@ -156,4 +188,5 @@ __all__ = ( 'check_zip', 'is_gzip', 'is_bz2', + 'is_zip', ) diff --git a/lib/galaxy/webapps/config_manage.py b/lib/galaxy/webapps/config_manage.py index 714d4628544..7ff45eb3f29 100644 --- a/lib/galaxy/webapps/config_manage.py +++ b/lib/galaxy/webapps/config_manage.py @@ -1,7 +1,6 @@ from __future__ import absolute_import, print_function import argparse -import copy import os import shutil import string @@ -54,9 +53,14 @@ UWSGI_OPTIONS = OrderedDict([ 'default': '127.0.0.1:$default_port', 'type': 'str', }), + ('buffer-size', { + 'desc': """By default uWSGI allocates a very small buffer (4096 bytes) for the headers of each request. If you start receiving "invalid request block size" in your logs, it could mean you need a bigger buffer. Increase it up to 65535.""", + 'default': 4096, + 'type': 'int', + }), ('processes', { 'desc': """Number of web server (worker) processes to fork after the application has loaded.""", - 'default': '1', + 'default': 1, 'type': 'int', }), ('threads', { @@ -66,7 +70,7 @@ UWSGI_OPTIONS = OrderedDict([ }), ('offload-threads', { 'desc': """Number of threads for serving static content and handling internal routing requests.""", - 'default': '2', + 'default': 2, 'type': 'int', }), ('static-map.1', { @@ -83,8 +87,8 @@ UWSGI_OPTIONS = OrderedDict([ }), ('master', { 'desc': """Enable the master process manager. Disabled by default for maximum compatibility with CTRL+C, but should be enabled for use with --daemon and/or production deployments.""", - 'default': 'false', - 'type': 'str', + 'default': False, + 'type': 'bool', }), ('virtualenv', { 'desc': """Path to the application's Python virtual environment.""", @@ -103,8 +107,8 @@ UWSGI_OPTIONS = OrderedDict([ }), ('die-on-term', { 'desc': """Cause uWSGI to respect the traditional behavior of dying on SIGTERM (its default is to brutally reload workers)""", - 'default': 'true', - 'type': 'str', + 'default': True, + 'type': 'bool', }), ('hook-master-start.1', { 'key': 'hook-master-start', @@ -120,13 +124,13 @@ UWSGI_OPTIONS = OrderedDict([ }), ('py-call-osafterfork', { 'desc': """Feature necessary for proper mule signal handling""", - 'default': 'true', - 'type': 'str', + 'default': True, + 'type': 'bool', }), ('enable-threads', { 'desc': """Ensure application threads will run if `threads` is unset.""", - 'default': 'true', - 'type': 'str', + 'default': True, + 'type': 'bool', }), # ('route-uri', { # 'default': '^/proxy/ goto:proxy' @@ -144,7 +148,7 @@ UWSGI_OPTIONS = OrderedDict([ # 'default': "['log:Proxy ${HTTP_HOST} to ${TARGET_HOST}', 'httpdumb:${TARGET_HOST}']", # }), # ('http-raw-body', { - # 'default': 'True' + # 'default': True # }), ]) @@ -659,8 +663,7 @@ def _replace_file(args, f, app_desc, from_path, to_path): def _build_sample_yaml(args, app_desc): schema = app_desc.schema f = StringIO() - options = copy.deepcopy(UWSGI_OPTIONS) - for key, value in options.items(): + for key, value in UWSGI_OPTIONS.items(): for field in ["desc", "default"]: if field not in value: continue @@ -679,7 +682,7 @@ def _build_sample_yaml(args, app_desc): description = description.lstrip() as_comment = "\n".join(["# %s" % l for l in description.split("\n")]) + "\n" f.write(as_comment) - _write_sample_section(args, f, 'uwsgi', Schema(options), as_comment=False, uwsgi_hack=True) + _write_sample_section(args, f, 'uwsgi', Schema(UWSGI_OPTIONS), as_comment=False, uwsgi_hack=True) _write_sample_section(args, f, app_desc.app_name, schema) destination = os.path.join(args.galaxy_root, app_desc.sample_destination) _write_to_file(args, f, destination) @@ -739,6 +742,8 @@ def _write_option(args, f, key, option_value, as_comment=False, uwsgi_hack=False comment += "\n" as_comment_str = "#" if as_comment else "" if uwsgi_hack: + if option.get("type", "str") == "bool": + value = str(value).lower() key_val_str = "%s: %s" % (key, value) else: key_val_str = yaml.dump({key: value}, width=float("inf")).lstrip("{").rstrip("\n}") diff --git a/lib/galaxy/webapps/galaxy/api/_fetch_util.py b/lib/galaxy/webapps/galaxy/api/_fetch_util.py index 7c5e2ea8e54..085fef13dc7 100644 --- a/lib/galaxy/webapps/galaxy/api/_fetch_util.py +++ b/lib/galaxy/webapps/galaxy/api/_fetch_util.py @@ -24,7 +24,7 @@ ELEMENTS_FROM_TRANSIENT_TYPES = ["archive", "bagit_archive"] def validate_and_normalize_targets(trans, payload): """Validate and normalize all src references in fetch targets. - - Normalize ftp_import and server_dir src entries into simple path entires + - Normalize ftp_import and server_dir src entries into simple path entries with the relevant paths resolved and permissions / configuration checked. - Check for file:// URLs in items src of "url" and convert them into path src items - after verifying path pastes are allowed and user is admin. diff --git a/lib/galaxy/webapps/galaxy/api/remote_files.py b/lib/galaxy/webapps/galaxy/api/remote_files.py index 434f8a835c4..b48dc4f7583 100644 --- a/lib/galaxy/webapps/galaxy/api/remote_files.py +++ b/lib/galaxy/webapps/galaxy/api/remote_files.py @@ -8,8 +8,14 @@ import time from operator import itemgetter from galaxy import exceptions -from galaxy.util import jstree, unicodify -from galaxy.util.path import safe_path, safe_walk +from galaxy.util import ( + jstree, + smart_str +) +from galaxy.util.path import ( + safe_path, + safe_walk +) from galaxy.web import _future_expose_api as expose_api from galaxy.web.base.controller import BaseAPIController @@ -135,13 +141,13 @@ class RemoteFilesAPIController(BaseAPIController): for (dirpath, dirnames, filenames) in safe_walk(directory, whitelist=whitelist): for dirname in dirnames: dir_path = os.path.relpath(os.path.join(dirpath, dirname), directory) - dir_path_hash = hashlib.sha1(unicodify(dir_path).encode('utf-8')).hexdigest() + dir_path_hash = hashlib.sha1(smart_str(dir_path)).hexdigest() disabled = True if disable == 'folders' else False jstree_paths.append(jstree.Path(dir_path, dir_path_hash, {'type': 'folder', 'state': {'disabled': disabled}, 'li_attr': {'full_path': dir_path}})) for filename in filenames: file_path = os.path.relpath(os.path.join(dirpath, filename), directory) - file_path_hash = hashlib.sha1(unicodify(file_path).encode('utf-8')).hexdigest() + file_path_hash = hashlib.sha1(smart_str(file_path)).hexdigest() disabled = True if disable == 'files' else False jstree_paths.append(jstree.Path(file_path, file_path_hash, {'type': 'file', 'state': {'disabled': disabled}, 'li_attr': {'full_path': file_path}})) else: diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index 08576ee812d..d1c06f732e1 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -370,7 +370,6 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin): def fetch(self, trans, payload, **kwd): """Adapt clean API to tool-constrained API. """ - log.info("Keywords are %s" % payload) request_version = '1' history_id = payload.pop("history_id") clean_payload = {} @@ -382,11 +381,9 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin): files_payload[key] = value continue clean_payload[key] = value - log.info("payload %s" % clean_payload) validate_and_normalize_targets(trans, clean_payload) clean_payload["check_content"] = trans.app.config.check_upload_content request = dumps(clean_payload) - log.info(request) create_payload = { 'tool_id': "__DATA_FETCH__", 'history_id': history_id, diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index 5679ffb01f1..31993800c9f 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -2539,6 +2539,29 @@ mapping: overwrite default job resources such as number of processors, memory and walltime. + workflow_resource_params_file: + type: str + default: config/workflow_resource_params_conf.xml + required: false + desc: | + Similar to the above parameter, workflows can describe parameters used to + influence scheduling of jobs within the workflow. This requires both a description + of the fields available (which defaults to the definitions in + job_resource_params_file if not set). + + workflow_resource_params_mapper: + type: str + default: config/workflow_resource_mapper_conf.yml + required: false + desc: | + This parameter describes how to map users and workflows to a set of workflow + resource parameter to present (typically input IDs from workflow_resource_params_file). + If this this is a function reference it will be passed various inputs (workflow model + object and user) and it should produce a list of input IDs. If it is a path + it is expected to an XML or YAML file describing how to map group names to parameter + descriptions (additional types of mappings via these files could be implemented but + haven't yet - for instance using workflow tags to do the mapping). + cache_user_job_count: type: bool default: false diff --git a/lib/galaxy/webapps/galaxy/controllers/user.py b/lib/galaxy/webapps/galaxy/controllers/user.py index 96ba8b0a786..ff138960711 100644 --- a/lib/galaxy/webapps/galaxy/controllers/user.py +++ b/lib/galaxy/webapps/galaxy/controllers/user.py @@ -532,7 +532,7 @@ class User(BaseUIController, UsesFormDefinitionsMixin, CreatesUsersMixin, Create if autoreg["auto_reg"]: kwd['email'] = autoreg["email"] kwd['username'] = autoreg["username"] - message = " ".join([validate_email(trans, kwd['email']), + message = " ".join([validate_email(trans, kwd['email'], allow_empty=True), validate_publicname(trans, kwd['username'])]).rstrip() if not message: message, status, user, success = self.__register(trans, cntrller, False, no_redirect=skip_login_handling, **kwd) @@ -572,7 +572,6 @@ class User(BaseUIController, UsesFormDefinitionsMixin, CreatesUsersMixin, Create log.debug("trans.app.config.auth_config_file: %s" % trans.app.config.auth_config_file) if not user: message, status, user, success = self.__autoregistration(trans, login, password, status, kwd) - elif user.deleted: message = "This account has been marked deleted, contact your local Galaxy administrator to restore the account." if trans.app.config.error_email_to is not None: diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index d090ef51280..33b9b8504c5 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -846,6 +846,7 @@ class ToolModule(WorkflowModule): else: iteration_elements_iter = [None] + resource_parameters = invocation.resource_parameters for iteration_elements in iteration_elements_iter: execution_state = tool_state.copy() # TODO: Move next step into copy() @@ -918,7 +919,8 @@ class ToolModule(WorkflowModule): invocation_step=invocation_step, max_num_jobs=max_num_jobs, job_callback=lambda job: self._handle_post_job_actions(step, job, invocation.replacement_dict), - completed_jobs=completed_jobs + completed_jobs=completed_jobs, + workflow_resource_parameters=resource_parameters ) complete = True except PartialJobExecution as pje: diff --git a/lib/galaxy/workflow/resources/__init__.py b/lib/galaxy/workflow/resources/__init__.py new file mode 100644 index 00000000000..a9bf1d07320 --- /dev/null +++ b/lib/galaxy/workflow/resources/__init__.py @@ -0,0 +1,176 @@ +"""This package is something a placeholder for workflow resource parameters. + +This file defines the baked in resource mapper types, and this package contains an +example of a more open, pluggable approach with greater control. +""" +import functools +import logging +import os +import sys +from copy import deepcopy + +import yaml + +import galaxy.util + +log = logging.getLogger(__name__) + + +def get_resource_mapper_function(app): + config = app.config + mapper = getattr(config, "workflow_resource_params_mapper", None) + + if mapper is None: + return _null_mapper_function + elif ":" in mapper: + raw_function = _import_resource_mapping_function(mapper) + # Bind resource parameters here just to not re-parse over and over. + workflow_resource_params = _read_defined_parameter_definitions(config) + return functools.partial(raw_function, workflow_resource_params=workflow_resource_params) + else: + workflow_resource_params = _read_defined_parameter_definitions(config) + with open(mapper, "r") as f: + mapper_definition = yaml.load(f) + + if "by_group" in mapper_definition: + by_group = mapper_definition["by_group"] + return functools.partial(_resource_parameters_by_group, by_group=by_group, workflow_resource_params=workflow_resource_params) + else: + raise Exception("Currently workflow parameter mapper definitions require a by_group definition.") + + +def _read_defined_parameter_definitions(config): + params_file = getattr(config, "workflow_resource_params_file", None) + if not params_file or not os.path.exists(params_file): + # Just re-use job resource parameters. + params_file = getattr(config, "job_resource_params_file", None) + if not params_file or not os.path.exists(params_file): + params_file = None + log.debug("Loading workflow resource parameter definitions from %s" % params_file) + if params_file: + return galaxy.util.parse_resource_parameters(params_file) + else: + return {} + + +def _resource_parameters_by_group(trans, **kwds): + user = trans.user + by_group = kwds["by_group"] + workflow_resource_params = kwds["workflow_resource_params"] + + params = [] + if validate_by_group_workflow_parameters_mapper(by_group, workflow_resource_params): + user_permissions = {} + user_groups = [] + for g in user.groups: + user_groups.append(g.group.name) + default_group = by_group.get('default', None) + for group_name, group_def in by_group.get("groups", {}).items(): + if group_name == default_group or group_name in user_groups: + for tag in group_def: + if type(tag) is dict: + if tag.get('name') not in user_permissions: + user_permissions[tag.get('name')] = {} + for option in tag.get('options'): + user_permissions[tag.get('name')][option] = {} + else: + if tag not in user_permissions: + user_permissions[tag] = {} + + # user_permissions is now set. + params = get_workflow_parameter_list(workflow_resource_params, user_permissions) + return params + + +# returns an array of parameters that a users set of permissions can access. +def get_workflow_parameter_list(params, user_permissions): + param_list = [] + for param_name, param_elem in params.items(): + attr = deepcopy(param_elem.attrib) + if attr['name'] in user_permissions: + # Allow 'select' type parameters to be used + if attr['type'] == 'select': + option_data = [] + reject_list = [] + for option_elem in param_elem.findall("option"): + if option_elem.attrib['value'] in user_permissions[attr['name']]: + option_data.append({ + 'label': option_elem.attrib['label'], + 'value': option_elem.attrib['value'] + }) + else: + reject_list.append(option_elem.attrib['label']) + attr['data'] = option_data + attr_help = "" + if 'help' in attr: + attr_help = attr['help'] + if reject_list: + attr_help += "

The following options are available but disabled.
" + \ + str(reject_list) + \ + "
If you believe this is a mistake, please contact your Galaxy admin." + attr['help'] = attr_help + + param_list.append(attr) + return param_list + + +def validate_by_group_workflow_parameters_mapper(by_group, workflow_resource_params): + valid = True + try: + if 'default' not in by_group: + raise Exception("'workflow_resource_params_mapper' YAML file is malformed, 'default' attribute not found!") + default_group = by_group['default'] + if 'groups' not in by_group: + raise Exception("'workflow_resource_params_mapper' YAML file is malformed, 'groups' attribute not found!") + if default_group not in by_group['groups']: + raise Exception("'workflow_resource_params_mapper' YAML file is malformed, default group with title '" + + default_group + "' not found in 'groups'!") + for group in by_group['groups']: + for attrib in by_group['groups'][group]: + if type(attrib) is dict: + if 'name' not in attrib: + raise Exception("'workflow_resource_params_mapper' YAML file is malformed, " + "'name' attribute not found in attribute of group '" + group + "'!") + if attrib['name'] not in workflow_resource_params: + raise Exception("'workflow_resource_params_mapper' YAML file is malformed, group with name '" + + attrib['name'] + "' not found in 'workflow_resource_params'!") + if 'options' not in attrib: + raise Exception("'workflow_resource_params_mapper' YAML file is malformed, " + "'options' attribute not found in attribute of group '" + group + "'!") + + valid_options = [] + for param_option in workflow_resource_params[attrib['name']]: + valid_options.append(param_option.attrib['value']) + for option in attrib['options']: + if option not in valid_options: + raise Exception("'workflow_resource_params_mapper' YAML file is malformed, '" + option + + "' in 'options' of '" + attrib['name'] + "' not found in attribute of group '" + group + "'!") + else: + if attrib not in workflow_resource_params: + raise Exception("'workflow_resource_params_mapper' YAML file is malformed, attribute with name " + "'" + attrib + "' not found in 'workflow_resource_params'!") + + except Exception as e: + log.exception(e) + valid = False + pass + + return valid + + +def _import_resource_mapping_function(qualified_function_path): + full_module_name, function_name = qualified_function_path.split(":", 1) + try: + __import__(full_module_name) + except ImportError: + raise Exception("Failed to find workflow resource mapper module %s" % full_module_name) + + module = sys.modules[full_module_name] + if hasattr(module, function_name): + return getattr(module, function_name) + else: + raise Exception("Failed to find workflow resource mapper function %s.%s" % (full_module_name, function_name)) + + +def _null_mapper_function(*args, **kwds): + return None diff --git a/lib/galaxy/workflow/resources/example.py.sample b/lib/galaxy/workflow/resources/example.py.sample new file mode 100644 index 00000000000..51454600ec0 --- /dev/null +++ b/lib/galaxy/workflow/resources/example.py.sample @@ -0,0 +1,22 @@ +import logging +log = logging.getLogger( __name__ ) + + +def admin_mapping(trans, stored_workflow, **kwds): + """ + This example workflow resource parameter mapping simply provides admins the ability to + specify priorities for workflows. To enable this setup ``workflow_resource_params_file`` + in the Galaxy configuration with a priority definition input called "priority" (such + as in the example), copy this file without the .sample extension, and set + ``workflow_resource_params_mapper`` to ``galaxy.workflow.resources.example:admin_mapping``. + """ + workflow_resource_params = kwds["workflow_resource_params"] + if trans.user_is_admin(): + priority_attrib = workflow_resource_params.get("priority").attrib + priority_attrib['data'] = [] + for child in workflow_resource_params.get('priority').getchildren(): + priority_attrib['data'].append(child.attrib) + time_attrib = workflow_resource_params.get("time").attrib + return [priority_attrib, time_attrib] + + return None diff --git a/lib/galaxy/workflow/run_request.py b/lib/galaxy/workflow/run_request.py index 6d67d332146..2d4b5ee6157 100644 --- a/lib/galaxy/workflow/run_request.py +++ b/lib/galaxy/workflow/run_request.py @@ -7,6 +7,7 @@ from galaxy import ( ) from galaxy.managers import histories from galaxy.tools.parameters.meta import expand_workflow_inputs +from galaxy.workflow.resources import get_resource_mapper_function INPUT_STEP_TYPES = ['data_input', 'data_collection_input', 'parameter_input'] @@ -47,12 +48,14 @@ class WorkflowRunConfig(object): inputs=None, param_map=None, allow_tool_state_corrections=False, - use_cached_job=False): + use_cached_job=False, + resource_params=None): self.target_history = target_history self.replacement_dict = replacement_dict self.copy_inputs_to_history = copy_inputs_to_history self.inputs = inputs or {} self.param_map = param_map or {} + self.resource_params = resource_params or {} self.allow_tool_state_corrections = allow_tool_state_corrections self.use_cached_job = use_cached_job @@ -305,6 +308,35 @@ def build_workflow_run_configs(trans, workflow, payload): normalized_inputs[key] = value['content'] else: normalized_inputs[key] = value + resource_params = payload.get('resource_params', {}) + if resource_params: + # quick attempt to validate parameters, just handle select options now since is what + # is needed for DTD - arbitrary plugins can define arbitrary logic at runtime in the + # destination function. In the future this should be extended to allow arbitrary + # pluggable validation. + resource_mapper_function = get_resource_mapper_function(trans.app) + # TODO: Do we need to do anything with the stored_workflow or can this be removed. + resource_parameters = resource_mapper_function(trans=trans, stored_workflow=None, workflow=workflow) + for resource_parameter in resource_parameters: + if resource_parameter.get("type") == "select": + name = resource_parameter.get("name") + if name in resource_params: + value = resource_params[name] + valid_option = False + # TODO: How should be handle the case where no selection is made by the user + # This can happen when there is a select on the page but the user has no options to select + # Here I have the validation pass it through. An alternative may be to remove the parameter if + # it is None. + if value is None: + valid_option = True + else: + for option_elem in resource_parameter.get('data'): + option_value = option_elem.get("value") + if value == option_value: + valid_option = True + if not valid_option: + raise exceptions.RequestParameterInvalidException("Invalid value for parameter '%s' found." % name) + run_configs.append(WorkflowRunConfig( target_history=history, replacement_dict=payload.get('replacement_params', {}), @@ -312,6 +344,7 @@ def build_workflow_run_configs(trans, workflow, payload): param_map=param_map, allow_tool_state_corrections=allow_tool_state_corrections, use_cached_job=use_cached_job, + resource_params=resource_params, )) return run_configs @@ -351,7 +384,8 @@ def workflow_run_config_to_request(trans, run_config, workflow): use_cached_job=run_config.use_cached_job, inputs={}, param_map={}, - allow_tool_state_corrections=run_config.allow_tool_state_corrections + allow_tool_state_corrections=run_config.allow_tool_state_corrections, + resource_params=run_config.resource_params ) subworkflow_invocation = workflow_run_config_to_request( trans, @@ -373,6 +407,9 @@ def workflow_run_config_to_request(trans, run_config, workflow): for step_id, content in run_config.inputs.items(): workflow_invocation.add_input(content, step_id) + resource_parameters = run_config.resource_params + for key, value in resource_parameters.items(): + add_parameter(key, value, param_types.RESOURCE_PARAMETERS) add_parameter("copy_inputs_to_history", "true" if run_config.copy_inputs_to_history else "false", param_types.META_PARAMETERS) add_parameter("use_cached_job", "true" if run_config.use_cached_job else "false", param_types.META_PARAMETERS) return workflow_invocation @@ -384,6 +421,7 @@ def workflow_request_to_run_config(work_request_context, workflow_invocation): replacement_dict = {} inputs = {} param_map = {} + resource_params = {} copy_inputs_to_history = None use_cached_job = False for parameter in workflow_invocation.input_parameters: @@ -396,6 +434,8 @@ def workflow_request_to_run_config(work_request_context, workflow_invocation): copy_inputs_to_history = (parameter.value == "true") if parameter.name == 'use_cached_job': use_cached_job = (parameter.value == 'true') + elif parameter_type == param_types.RESOURCE_PARAMETERS: + resource_params[parameter.name] = parameter.value for input_association in workflow_invocation.input_datasets: inputs[input_association.workflow_step_id] = input_association.dataset for input_association in workflow_invocation.input_dataset_collections: @@ -411,6 +451,7 @@ def workflow_request_to_run_config(work_request_context, workflow_invocation): param_map=param_map, copy_inputs_to_history=copy_inputs_to_history, use_cached_job=use_cached_job, + resource_params=resource_params, ) return workflow_run_config diff --git a/lib/galaxy_ext/metadata/set_metadata.py b/lib/galaxy_ext/metadata/set_metadata.py index d1236020456..cc7c16a2828 100644 --- a/lib/galaxy_ext/metadata/set_metadata.py +++ b/lib/galaxy_ext/metadata/set_metadata.py @@ -44,7 +44,7 @@ def set_meta_with_tool_provided(dataset_instance, file_dict, set_meta_kwds, data if extension == "_sniff_": try: from galaxy.datatypes import sniff - extension = sniff.handle_uploaded_dataset_file(dataset_instance.dataset.external_filename, datatypes_registry) + extension = sniff.handle_uploaded_dataset_file(dataset_instance.dataset.external_filename, datatypes_registry)[0] # We need to both set the extension so it is available to set_meta # and record it in the metadata so it can be reloaded on the server # side and the model updated (see MetadataCollection.{from,to}_JSON_dict) diff --git a/lib/tool_shed/util/hg_util.py b/lib/tool_shed/util/hg_util.py index 30b3e5e4d02..d2518227f9d 100644 --- a/lib/tool_shed/util/hg_util.py +++ b/lib/tool_shed/util/hg_util.py @@ -62,7 +62,7 @@ def clone_repository(repository_clone_url, repository_file_dir, ctx_rev): error_message = 'Error cloning repository: %s' % e if isinstance(e, subprocess.CalledProcessError): error_message += "\nOutput was:\n%s" % stdouterr - log.debug(error_message) + log.error(error_message) return False, error_message diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index 63dde0ce62f..cd82b331b1d 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -301,7 +301,7 @@ def get_repository_file_contents(app, file_path, repository_id, is_admin=False): return '
gzip compressed file
' elif checkers.is_bz2(file_path): return '
bz2 compressed file
' - elif checkers.check_zip(file_path): + elif checkers.is_zip(file_path): return '
zip compressed file
' elif checkers.check_binary(file_path): return '
Binary file
' diff --git a/lib/tool_shed/util/tool_util.py b/lib/tool_shed/util/tool_util.py index 3a6ac2069cb..556cf2137dc 100644 --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -190,7 +190,7 @@ def is_data_index_sample_file(file_path): return False if checkers.is_gzip(file_path): return False - if checkers.check_zip(file_path): + if checkers.is_zip(file_path): return False # Default to copying the file if none of the above are true. return True diff --git a/run_tests.sh b/run_tests.sh index 855c95609b1..5f2bca33031 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -289,7 +289,9 @@ then shift fi MY_UID=$(id -u) - DOCKER_RUN_EXTRA_ARGS="-e GALAXY_TEST_UID=${MY_UID} ${DOCKER_RUN_EXTRA_ARGS}" + # Skip client build process in the Docker container for all tests, the Jenkins task builds the client + # locally before testing - you will need to do this also if using this script for Selenium testing. + DOCKER_RUN_EXTRA_ARGS="-e GALAXY_TEST_UID=${MY_UID} -e GALAXY_SKIP_CLIENT_BUILD=1 ${DOCKER_RUN_EXTRA_ARGS}" echo "Launching docker container for testing with extra args ${DOCKER_RUN_EXTRA_ARGS}..." docker $DOCKER_EXTRA_ARGS run $DOCKER_RUN_EXTRA_ARGS -e "BUILD_NUMBER=$BUILD_NUMBER" -e "GALAXY_TEST_DATABASE_TYPE=$db_type" --rm -v `pwd`:/galaxy $DOCKER_IMAGE "$@" exit $? diff --git a/scripts/apply_tags.py b/scripts/apply_tags.py new file mode 100644 index 00000000000..1d118f19db0 --- /dev/null +++ b/scripts/apply_tags.py @@ -0,0 +1,167 @@ +""" Apply tags to the inherited history items of Galaxy """ + +import sys +import time + +from bioblend.galaxy import GalaxyInstance + + +class ApplyTagsHistory: + + @classmethod + def __init__(self, galaxy_url, galaxy_api_key, history_id=None): + self.galaxy_url = galaxy_url + self.galaxy_api_key = galaxy_api_key + self.history_id = history_id + + @classmethod + def read_galaxy_history(self): + """ + Read Galaxy's current history and inherit all the tags from a parent + to a child history item + """ + # connect to running Galaxy's instance + g_instance = GalaxyInstance(self.galaxy_url, self.galaxy_api_key, self.history_id) + history = g_instance.histories + job = g_instance.jobs + # if the history id is not supplied, then update tags for the most recently used history + if self.history_id is None: + update_history = history.get_most_recently_used_history() + else: + try: + update_history = history.show_history(self.history_id) + except Exception as exception: + print("Some problem occurred with history: %s" % self.history_id) + print(exception) + return + update_history_id = update_history["id"] + print("History name: %s" % update_history["name"]) + print("History id: %s" % update_history_id) + self.find_dataset_parents_update_tags(history, job, update_history_id) + + @classmethod + def find_dataset_parents_update_tags(self, history, job, history_id): + """ + Operate on datasets for a particular history and recursively find parents + for a dataset + """ + datasets_inheritance_chain = dict() + own_tags = dict() + parent_tags = dict() + count_datasets_updated = 0 + # get all datasets belonging to a history + all_datasets = history.show_history(history_id, contents=True) + print("Total datasets: %d. Updating their tags may take a while..." % len(all_datasets)) + for dataset in all_datasets: + try: + if dataset["deleted"] is False and dataset["state"] == 'ok': + parent_ids = list() + child_dataset_id = dataset["id"] + own_tags[child_dataset_id] = dataset["tags"] + # get information about the dataset like the job id + # used in its creation. One parameter "inputs" from the job details lists all the dataset id(s) + # used in creating the current dataset which is/are its parent datasets. + dataset_info = history.show_dataset_provenance(history_id, child_dataset_id, False) + job_details = job.show_job(dataset_info["job_id"], True) + if "inputs" in job_details: + # get all the inputs for the job that created this dataset. + # these inputs are the parent datasets of the current dataset + job_inputs = job_details["inputs"] + for item in job_inputs: + parent_id = job_inputs[item]["id"] + try: + if parent_id not in parent_tags: + parent_dataset = history.show_dataset(history_id, parent_id) + if not parent_dataset["deleted"]: + parent_tags[parent_id] = parent_dataset["tags"] + parent_ids.append(parent_id) + else: + parent_ids.append(parent_id) + except Exception: + pass + datasets_inheritance_chain[child_dataset_id] = parent_ids + except Exception: + pass + # collect all the parents for each dataset recursively + all_parents = self.collect_parent_ids(datasets_inheritance_chain) + # update tags + for dataset_id in all_parents: + parent_dataset_ids = all_parents[dataset_id] + # update history tags for a dataset taking all from its parents if there is a parent + if len(parent_dataset_ids) > 0: + is_updated = self.propagate_tags(history, history_id, parent_dataset_ids, dataset_id, parent_tags, own_tags) + if is_updated is True: + count_datasets_updated += 1 + print("Tags of %d datasets updated" % count_datasets_updated) + + @classmethod + def collect_parent_ids(self, datasets_inheritance_chain): + """ + Collect parent datasets for each dataset recursively + """ + recursive_parent_ids = dict() + for item in datasets_inheritance_chain: + recursive_parents = list() + + def find_parent_recursive(dataset_id): + if dataset_id in datasets_inheritance_chain: + # get parents of a dataset + dataset_parents = datasets_inheritance_chain[dataset_id] + # add all the parents to the recursive list + recursive_parents.extend(dataset_parents) + for parent in dataset_parents: + find_parent_recursive(parent) + find_parent_recursive(item) + # take unique parents + recursive_parent_ids[item] = list(set(recursive_parents)) + return recursive_parent_ids + + @classmethod + def collect_hash_tags(self, tags_list): + """ + Collect only hash tags and exclude others if any + """ + return [tag for tag in tags_list if len(tag.split(":")) > 1] + + @classmethod + def propagate_tags(self, history, current_history_id, parent_datasets_ids, dataset_id, parent_tags, own_tags): + """ + Propagate history tags from parent(s) to a child + """ + all_tags = list() + for parent_id in parent_datasets_ids: + # collect all the tags from the parent + all_tags.extend(parent_tags[parent_id]) + # take only hash tags + all_tags = self.collect_hash_tags(all_tags) + self_tags = self.collect_hash_tags(own_tags[dataset_id]) + # find unique tags from all parents + all_tags = set(all_tags) + self_tags_set = set(self_tags) + is_same = (all_tags == self_tags_set) + # update tags if there are new tags from parents + if is_same is False: + is_subset = all_tags.issubset(self_tags_set) + if is_subset is False: + # append the tags of the child itself + all_tags = list(all_tags) + all_tags.extend(self_tags) + # do a database update for the child dataset so that it reflects the tags from all parents + # take unique tags + history.update_dataset(current_history_id, dataset_id, tags=all_tags) + return True + + +if __name__ == "__main__": + + if len(sys.argv) < 3: + print("Usage: python apply_tags.py ") + exit(1) + start_time = time.time() + history_id = None + if len(sys.argv) > 3: + history_id = sys.argv[3] + history_tags = ApplyTagsHistory(sys.argv[1], sys.argv[2], history_id) + history_tags.read_galaxy_history() + end_time = time.time() + print("Program finished in %d seconds" % int(end_time - start_time)) diff --git a/scripts/bootstrap_history.py b/scripts/bootstrap_history.py index 7b3faa92731..e85f34f255a 100644 --- a/scripts/bootstrap_history.py +++ b/scripts/bootstrap_history.py @@ -112,7 +112,7 @@ To get a new Galaxy repository run: To update an existing Galaxy repository run: .. code-block:: shell - $$ git checkout release_${release} && git pull --ff-only origin release_${release} + $$ git fetch origin && git checkout release_${release} && git pull --ff-only origin release_${release} See the `community hub `__ for additional details regarding the source code locations. @@ -198,12 +198,12 @@ RELEASE_ISSUE_TEMPLATE = string.Template(""" - [ ] Ensure all [blocking milestone PRs](https://github.com/galaxyproject/galaxy/pulls?q=is%3Aopen+is%3Apr+milestone%3A${version}) have been merged or closed. make release-check-blocking-prs RELEASE_CURR=${version} - - [ ] Ensure previous release is merged into current. (TODO: Add Makefile target or this.) + - [ ] Ensure previous release is merged into current. [Github branch comparison](https://github.com/galaxyproject/galaxy/compare/release_${version}...release_${previous_version}) - [ ] Create and push release tag: make release-create RELEASE_CURR=${version} - - [ ] Switch Jenkins documentation build [branch specifier](https://jenkins.galaxyproject.org/job/Sphinx-Docs/configure) to `*/release_{version}`. + - [ ] Add the branch `*/release_{version}` to Jenkins documentation build [configuration matrix](https://jenkins.galaxyproject.org/job/galaxy-sphinx-by-branch/configure). - [ ] Trigger the documentation build. - [ ] **Do Docker Release** @@ -222,8 +222,8 @@ RELEASE_ISSUE_TEMPLATE = string.Template(""" - [ ] Review announcement in https://github.com/galaxyproject/galaxy/blob/dev/doc/source/releases/${version}_announce.rst - [ ] Stage annoucement content (Hub, Biostars, Bit.ly link) on annouce date to capture date tags. Note: all final content does not need to be completed to do this. - [ ] Create hub *highlights* and post to http://galaxyproject.org News (w/ RSS) and NewsBriefs. [An Example](https://galaxyproject.org/news/2016-04-galaxy-release). - - [ ] Tweet docs news *highlights* via bit.ly link to https://twitter.com/galaxyproject/ (As user ``galaxyproject``, password in Galaxy password store under ``twitter.com / galaxyproject`` ). [An Example](https://twitter.com/galaxyproject/status/733029921316986881). - - [ ] Post *highlights* type News to Galaxy Biostars https://biostar.usegalaxy.org. [An Example](https://biostar.usegalaxy.org/p/17712/). + - [ ] Tweet docs news *highlights* via bit.ly link to https://twitter.com/galaxyproject/. [An Example](https://twitter.com/galaxyproject/status/973646125633695744). + - [ ] Post *highlights* type News to Galaxy Biostars https://biostar.usegalaxy.org. [An Example](https://biostar.usegalaxy.org/p/27118/). - [ ] Email *highlights* to [galaxy-dev](http://dev.list.galaxyproject.org/) and [galaxy-announce](http://announce.list.galaxyproject.org/) @lists.galaxyproject.org. [An Example](http://dev.list.galaxyproject.org/The-Galaxy-release-16-04-is-out-tp4669419.html) - [ ] Adjust http://getgalaxy.org text and links to match current master branch by opening a PR for https://github.com/galaxyproject/galaxy-hub/ diff --git a/scripts/cleanup_datasets/admin_cleanup_datasets.py b/scripts/cleanup_datasets/admin_cleanup_datasets.py index c5d92e93897..ef3bddfcedd 100755 --- a/scripts/cleanup_datasets/admin_cleanup_datasets.py +++ b/scripts/cleanup_datasets/admin_cleanup_datasets.py @@ -53,6 +53,8 @@ from mako.template import Template from six.moves import configparser from sqlalchemy import and_, false +sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib'))) + import galaxy.config import galaxy.model.mapping import galaxy.util diff --git a/scripts/cleanup_datasets/update_dataset_size.py b/scripts/cleanup_datasets/update_dataset_size.py index 5cebec6e20e..25ad273ed91 100755 --- a/scripts/cleanup_datasets/update_dataset_size.py +++ b/scripts/cleanup_datasets/update_dataset_size.py @@ -10,6 +10,8 @@ import sys from six.moves import configparser +sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib'))) + import galaxy.app assert sys.version_info[:2] >= (2, 6) @@ -24,7 +26,7 @@ running. def main(): - if len(sys.argv) != 1 or sys.argv[1] == "-h" or sys.argv[1] == "--help": + if len(sys.argv) != 2 or sys.argv[1] == "-h" or sys.argv[1] == "--help": usage(sys.argv[0]) sys.exit() ini_file = sys.argv.pop(1) diff --git a/test/api/test_libraries.py b/test/api/test_libraries.py index ae0303a8074..6f30a981e60 100644 --- a/test/api/test_libraries.py +++ b/test/api/test_libraries.py @@ -1,4 +1,5 @@ import json +import unittest from base import api from base.populators import ( @@ -163,6 +164,7 @@ class LibrariesApiTestCase(api.ApiTestCase, TestsDatasets): dataset = self.library_populator.get_library_contents_with_path(library["id"], "/4.bed") assert dataset["file_size"] == 61, dataset + @unittest.skip # reference URLs changed, checksums now invalid. def test_fetch_bagit_archive_to_folder(self): history_id, library, destination = self._setup_fetch_to_folder("bagit_archive") example_bag_path = self.test_data_resolver.get_filename("example-bag.zip") diff --git a/test/api/test_tools_upload.py b/test/api/test_tools_upload.py index cb67b84507d..05e4f784263 100644 --- a/test/api/test_tools_upload.py +++ b/test/api/test_tools_upload.py @@ -83,31 +83,36 @@ class ToolsUploadTestCase(api.ApiTestCase): def test_rdata_not_decompressed(self): # Prevent regression of https://github.com/galaxyproject/galaxy/issues/753 rdata_path = TestDataResolver().get_filename("1.RData") - rdata_metadata = self._upload_and_get_details(open(rdata_path, "rb"), file_type="auto") + with open(rdata_path, "rb") as fh: + rdata_metadata = self._upload_and_get_details(fh, file_type="auto") self.assertEquals(rdata_metadata["file_ext"], "rdata") @skip_without_datatype("csv") def test_csv_upload(self): csv_path = TestDataResolver().get_filename("1.csv") - csv_metadata = self._upload_and_get_details(open(csv_path, "rb"), file_type="csv") + with open(csv_path, "rb") as fh: + csv_metadata = self._upload_and_get_details(fh, file_type="csv") self.assertEquals(csv_metadata["file_ext"], "csv") @skip_without_datatype("csv") def test_csv_upload_auto(self): csv_path = TestDataResolver().get_filename("1.csv") - csv_metadata = self._upload_and_get_details(open(csv_path, "rb"), file_type="auto") + with open(csv_path, "rb") as fh: + csv_metadata = self._upload_and_get_details(fh, file_type="auto") self.assertEquals(csv_metadata["file_ext"], "csv") @skip_without_datatype("csv") def test_csv_fetch(self): csv_path = TestDataResolver().get_filename("1.csv") - csv_metadata = self._upload_and_get_details(open(csv_path, "rb"), api="fetch", ext="csv", to_posix_lines=True) + with open(csv_path, "rb") as fh: + csv_metadata = self._upload_and_get_details(fh, api="fetch", ext="csv", to_posix_lines=True) self.assertEquals(csv_metadata["file_ext"], "csv") @skip_without_datatype("csv") def test_csv_sniff_fetch(self): csv_path = TestDataResolver().get_filename("1.csv") - csv_metadata = self._upload_and_get_details(open(csv_path, "rb"), api="fetch", ext="auto", to_posix_lines=True) + with open(csv_path, "rb") as fh: + csv_metadata = self._upload_and_get_details(fh, api="fetch", ext="auto", to_posix_lines=True) self.assertEquals(csv_metadata["file_ext"], "csv") @skip_without_datatype("velvet") diff --git a/test/api/test_workflows.py b/test/api/test_workflows.py index f40fad13620..e06ed6317a8 100644 --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -591,8 +591,8 @@ class WorkflowsApiTestCase(BaseWorkflowsApiTestCase): @skip_without_tool("multiple_versions") def test_run_versioned_tools(self): - history_01_id = self.dataset_populator.new_history() - workflow_version_01 = self._upload_yaml_workflow(""" + with self.dataset_populator.test_history() as history_01_id: + workflow_version_01 = self._upload_yaml_workflow(""" class: GalaxyWorkflow steps: - tool_id: multiple_versions @@ -600,11 +600,11 @@ steps: state: inttest: 0 """) - self.__invoke_workflow(history_01_id, workflow_version_01) - self.dataset_populator.wait_for_history(history_01_id, assert_ok=True) + self.__invoke_workflow(history_01_id, workflow_version_01) + self.dataset_populator.wait_for_history(history_01_id, assert_ok=True) - history_02_id = self.dataset_populator.new_history() - workflow_version_02 = self._upload_yaml_workflow(""" + with self.dataset_populator.test_history() as history_02_id: + workflow_version_02 = self._upload_yaml_workflow(""" class: GalaxyWorkflow steps: - tool_id: multiple_versions @@ -612,8 +612,8 @@ steps: state: inttest: 1 """) - self.__invoke_workflow(history_02_id, workflow_version_02) - self.dataset_populator.wait_for_history(history_02_id, assert_ok=True) + self.__invoke_workflow(history_02_id, workflow_version_02) + self.dataset_populator.wait_for_history(history_02_id, assert_ok=True) def __run_cat_workflow(self, inputs_by): workflow = self.workflow_populator.load_workflow(name="test_for_run") @@ -648,15 +648,15 @@ steps: f1: $link: split_up#paired_output """) - history_id = self.dataset_populator.new_history() - hda1 = self.dataset_populator.new_dataset(history_id, content="a\nb\nc\nd\n") - inputs = { - '0': self._ds_entry(hda1), - } - invocation_id = self.__invoke_workflow(history_id, workflow_id, inputs) - self.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id) - self.dataset_populator.wait_for_history(history_id, assert_ok=True) - self.assertEqual("a\nc\nb\nd\n", self.dataset_populator.get_history_dataset_content(history_id, hid=0)) + with self.dataset_populator.test_history() as history_id: + hda1 = self.dataset_populator.new_dataset(history_id, content="a\nb\nc\nd\n") + inputs = { + '0': self._ds_entry(hda1), + } + invocation_id = self.__invoke_workflow(history_id, workflow_id, inputs) + self.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id) + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + self.assertEqual("a\nc\nb\nd\n", self.dataset_populator.get_history_dataset_content(history_id, hid=0)) @skip_without_tool("job_properties") @skip_without_tool("identifier_multiple_in_conditional") @@ -677,22 +677,22 @@ steps: input1: $link: 0#out_file1 """) - history_id = self.dataset_populator.new_history() - invocation_id = self.__invoke_workflow(history_id, workflow_id) - self.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id, assert_ok=False) - failed_dataset_one = self.dataset_populator.get_history_dataset_details(history_id, hid=1, wait=True, assert_ok=False) - assert failed_dataset_one['state'] == 'error', failed_dataset_one - paused_dataset = self.dataset_populator.get_history_dataset_details(history_id, hid=5, wait=True, assert_ok=False) - assert paused_dataset['state'] == 'paused', paused_dataset - inputs = {"thebool": "false", - "failbool": "false", - "rerun_remap_job_id": failed_dataset_one['creating_job']} - self.dataset_populator.run_tool(tool_id='job_properties', - inputs=inputs, - history_id=history_id, - assert_ok=True) - unpaused_dataset = self.dataset_populator.get_history_dataset_details(history_id, hid=5, wait=True, assert_ok=False) - assert unpaused_dataset['state'] == 'ok' + with self.dataset_populator.test_history() as history_id: + invocation_id = self.__invoke_workflow(history_id, workflow_id) + self.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id, assert_ok=False) + failed_dataset_one = self.dataset_populator.get_history_dataset_details(history_id, hid=1, wait=True, assert_ok=False) + assert failed_dataset_one['state'] == 'error', failed_dataset_one + paused_dataset = self.dataset_populator.get_history_dataset_details(history_id, hid=5, wait=True, assert_ok=False) + assert paused_dataset['state'] == 'paused', paused_dataset + inputs = {"thebool": "false", + "failbool": "false", + "rerun_remap_job_id": failed_dataset_one['creating_job']} + self.dataset_populator.run_tool(tool_id='job_properties', + inputs=inputs, + history_id=history_id, + assert_ok=True) + unpaused_dataset = self.dataset_populator.get_history_dataset_details(history_id, hid=5, wait=True, assert_ok=False) + assert unpaused_dataset['state'] == 'ok' @skip_without_tool("job_properties") @skip_without_tool("identifier_multiple_in_conditional") @@ -796,21 +796,21 @@ steps: input1: $link: 2#out1 """) - history_id = self.dataset_populator.new_history() - hdca1 = self.dataset_collection_populator.create_list_in_history(history_id, contents=["a\nb\nc\nd\n", "e\nf\ng\nh\n"]).json() - self.dataset_populator.wait_for_history(history_id, assert_ok=True) - inputs = { - '0': self._ds_entry(hdca1), - } - invocation_id = self.__invoke_workflow(history_id, workflow_id, inputs) - self.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id) - self.dataset_populator.wait_for_history(history_id, assert_ok=True) - self.assertEqual("a\nc\nb\nd\ne\ng\nf\nh\n", self.dataset_populator.get_history_dataset_content(history_id, hid=0)) + with self.dataset_populator.test_history() as history_id: + hdca1 = self.dataset_collection_populator.create_list_in_history(history_id, contents=["a\nb\nc\nd\n", "e\nf\ng\nh\n"]).json() + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + inputs = { + '0': self._ds_entry(hdca1), + } + invocation_id = self.__invoke_workflow(history_id, workflow_id, inputs) + self.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id) + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + self.assertEqual("a\nc\nb\nd\ne\ng\nf\nh\n", self.dataset_populator.get_history_dataset_content(history_id, hid=0)) @skip_without_tool("collection_split_on_column") def test_workflow_run_dynamic_output_collections(self): - history_id = self.dataset_populator.new_history() - workflow_id = self._upload_yaml_workflow(""" + with self.dataset_populator.test_history() as history_id: + workflow_id = self._upload_yaml_workflow(""" class: GalaxyWorkflow steps: - label: text_input1 @@ -835,20 +835,20 @@ steps: input1: $link: split_up#split_output """) - hda1 = self.dataset_populator.new_dataset(history_id, content="samp1\t10.0\nsamp2\t20.0\n") - hda2 = self.dataset_populator.new_dataset(history_id, content="samp1\t30.0\nsamp2\t40.0\n") - self.dataset_populator.wait_for_history(history_id, assert_ok=True) - inputs = { - '0': self._ds_entry(hda1), - '1': self._ds_entry(hda2), - } - invocation_id = self.__invoke_workflow(history_id, workflow_id, inputs) - self.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id) - details = self.dataset_populator.get_history_dataset_details(history_id, hid=0) - last_item_hid = details["hid"] - assert last_item_hid == 7, "Expected 7 history items, got %s" % last_item_hid - content = self.dataset_populator.get_history_dataset_content(history_id, hid=0) - self.assertEqual("10.0\n30.0\n20.0\n40.0\n", content) + hda1 = self.dataset_populator.new_dataset(history_id, content="samp1\t10.0\nsamp2\t20.0\n") + hda2 = self.dataset_populator.new_dataset(history_id, content="samp1\t30.0\nsamp2\t40.0\n") + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + inputs = { + '0': self._ds_entry(hda1), + '1': self._ds_entry(hda2), + } + invocation_id = self.__invoke_workflow(history_id, workflow_id, inputs) + self.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id) + details = self.dataset_populator.get_history_dataset_details(history_id, hid=0) + last_item_hid = details["hid"] + assert last_item_hid == 7, "Expected 7 history items, got %s" % last_item_hid + content = self.dataset_populator.get_history_dataset_content(history_id, hid=0) + self.assertEqual("10.0\n30.0\n20.0\n40.0\n", content) @skip_without_tool("collection_split_on_column") @skip_without_tool("min_repeat") @@ -899,8 +899,8 @@ steps: @skip_without_tool("collection_split_on_column") def test_workflow_run_dynamic_output_collections_3(self): # Test a workflow that create a list:list:list followed by a mapping step. - history_id = self.dataset_populator.new_history() - workflow_id = self._upload_yaml_workflow(""" + with self.dataset_populator.test_history() as history_id: + workflow_id = self._upload_yaml_workflow(""" class: GalaxyWorkflow steps: - label: text_input1 @@ -930,23 +930,23 @@ steps: input1: $link: split_up_2#split_output """) - hda1 = self.dataset_populator.new_dataset(history_id, content="samp1\t10.0\nsamp2\t20.0\n") - hda2 = self.dataset_populator.new_dataset(history_id, content="samp1\t30.0\nsamp2\t40.0\n") - self.dataset_populator.wait_for_history(history_id, assert_ok=True) - inputs = { - '0': self._ds_entry(hda1), - '1': self._ds_entry(hda2), - } - invocation_id = self.__invoke_workflow(history_id, workflow_id, inputs) - self.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id) + hda1 = self.dataset_populator.new_dataset(history_id, content="samp1\t10.0\nsamp2\t20.0\n") + hda2 = self.dataset_populator.new_dataset(history_id, content="samp1\t30.0\nsamp2\t40.0\n") + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + inputs = { + '0': self._ds_entry(hda1), + '1': self._ds_entry(hda2), + } + invocation_id = self.__invoke_workflow(history_id, workflow_id, inputs) + self.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id) @skip_without_tool("mapper") @skip_without_tool("pileup") def test_workflow_metadata_validation_0(self): # Testing regression of # https://github.com/galaxyproject/galaxy/issues/1514 - history_id = self.dataset_populator.new_history() - self._run_jobs(""" + with self.dataset_populator.test_history() as history_id: + self._run_jobs(""" class: GalaxyWorkflow steps: - label: input_fastqs @@ -983,24 +983,24 @@ test_data: """, history_id=history_id) def test_run_subworkflow_simple(self): - history_id = self.dataset_populator.new_history() - workflow_run_description = """%s + with self.dataset_populator.test_history() as history_id: + workflow_run_description = """%s test_data: outer_input: value: 1.bed type: File """ % SIMPLE_NESTED_WORKFLOW_YAML - self._run_jobs(workflow_run_description, history_id=history_id) + self._run_jobs(workflow_run_description, history_id=history_id) - content = self.dataset_populator.get_history_dataset_content(history_id) - self.assertEqual("chr5\t131424298\t131424460\tCCDS4149.1_cds_0_0_chr5_131424299_f\t0\t+\nchr5\t131424298\t131424460\tCCDS4149.1_cds_0_0_chr5_131424299_f\t0\t+\n", content) + content = self.dataset_populator.get_history_dataset_content(history_id) + self.assertEqual("chr5\t131424298\t131424460\tCCDS4149.1_cds_0_0_chr5_131424299_f\t0\t+\nchr5\t131424298\t131424460\tCCDS4149.1_cds_0_0_chr5_131424299_f\t0\t+\n", content) @skip_without_tool("cat1") @skip_without_tool("collection_paired_test") def test_workflow_run_zip_collections(self): - history_id = self.dataset_populator.new_history() - workflow_id = self._upload_yaml_workflow(""" + with self.dataset_populator.test_history() as history_id: + workflow_id = self._upload_yaml_workflow(""" class: GalaxyWorkflow steps: - label: test_input_1 @@ -1025,21 +1025,21 @@ steps: f1: $link: zip_it#output """) - hda1 = self.dataset_populator.new_dataset(history_id, content="samp1\t10.0\nsamp2\t20.0\n") - hda2 = self.dataset_populator.new_dataset(history_id, content="samp1\t20.0\nsamp2\t40.0\n") - self.dataset_populator.wait_for_history(history_id, assert_ok=True) - inputs = { - '0': self._ds_entry(hda1), - '1': self._ds_entry(hda2), - } - invocation_id = self.__invoke_workflow(history_id, workflow_id, inputs) - self.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id) - content = self.dataset_populator.get_history_dataset_content(history_id) - self.assertEqual(content.strip(), "samp1\t10.0\nsamp2\t20.0\nsamp1\t20.0\nsamp2\t40.0") + hda1 = self.dataset_populator.new_dataset(history_id, content="samp1\t10.0\nsamp2\t20.0\n") + hda2 = self.dataset_populator.new_dataset(history_id, content="samp1\t20.0\nsamp2\t40.0\n") + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + inputs = { + '0': self._ds_entry(hda1), + '1': self._ds_entry(hda2), + } + invocation_id = self.__invoke_workflow(history_id, workflow_id, inputs) + self.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id) + content = self.dataset_populator.get_history_dataset_content(history_id) + self.assertEqual(content.strip(), "samp1\t10.0\nsamp2\t20.0\nsamp1\t20.0\nsamp2\t40.0") def test_filter_failed_mapping(self): - history_id = self.dataset_populator.new_history() - summary = self._run_jobs(""" + with self.dataset_populator.test_history() as history_id: + summary = self._run_jobs(""" class: GalaxyWorkflow inputs: - type: collection @@ -1070,16 +1070,16 @@ test_data: - identifier: i2 content: "1" """, history_id=history_id, wait=True, assert_ok=False) - jobs = summary.jobs + jobs = summary.jobs - def filter_jobs_by_tool(tool_id): - return [j for j in summary.jobs if j["tool_id"] == tool_id] + def filter_jobs_by_tool(tool_id): + return [j for j in summary.jobs if j["tool_id"] == tool_id] - assert len(filter_jobs_by_tool("upload1")) == 2, jobs - assert len(filter_jobs_by_tool("exit_code_from_file")) == 2, jobs - assert len(filter_jobs_by_tool("__FILTER_FAILED_DATASETS__")) == 1, jobs - # Follow proves one job was filtered out of the result of cat1 - assert len(filter_jobs_by_tool("cat1")) == 1, jobs + assert len(filter_jobs_by_tool("upload1")) == 2, jobs + assert len(filter_jobs_by_tool("exit_code_from_file")) == 2, jobs + assert len(filter_jobs_by_tool("__FILTER_FAILED_DATASETS__")) == 1, jobs + # Follow proves one job was filtered out of the result of cat1 + assert len(filter_jobs_by_tool("cat1")) == 1, jobs def test_workflow_request(self): workflow = self.workflow_populator.load_workflow(name="test_for_queue") @@ -1094,8 +1094,8 @@ test_data: self.dataset_populator.wait_for_history(history_id, assert_ok=True) def test_workflow_output_dataset(self): - history_id = self.dataset_populator.new_history() - summary = self._run_jobs(""" + with self.dataset_populator.test_history() as history_id: + summary = self._run_jobs(""" class: GalaxyWorkflow inputs: - id: input1 @@ -1112,20 +1112,20 @@ steps: test_data: input1: "hello world" """, history_id=history_id) - workflow_id = summary.workflow_id - invocation_id = summary.invocation_id - invocation_response = self._get("workflows/%s/invocations/%s" % (workflow_id, invocation_id)) - self._assert_status_code_is(invocation_response, 200) - invocation = invocation_response.json() - self._assert_has_keys(invocation , "id", "outputs", "output_collections") - assert len(invocation["output_collections"]) == 0 - assert len(invocation["outputs"]) == 1 - output_content = self.dataset_populator.get_history_dataset_content(history_id, dataset_id=invocation["outputs"]["wf_output_1"]["id"]) - assert "hello world" == output_content.strip() + workflow_id = summary.workflow_id + invocation_id = summary.invocation_id + invocation_response = self._get("workflows/%s/invocations/%s" % (workflow_id, invocation_id)) + self._assert_status_code_is(invocation_response, 200) + invocation = invocation_response.json() + self._assert_has_keys(invocation , "id", "outputs", "output_collections") + assert len(invocation["output_collections"]) == 0 + assert len(invocation["outputs"]) == 1 + output_content = self.dataset_populator.get_history_dataset_content(history_id, dataset_id=invocation["outputs"]["wf_output_1"]["id"]) + assert "hello world" == output_content.strip() def test_workflow_output_dataset_collection(self): - history_id = self.dataset_populator.new_history() - summary = self._run_jobs(""" + with self.dataset_populator.test_history() as history_id: + summary = self._run_jobs(""" class: GalaxyWorkflow inputs: - id: input1 @@ -1149,25 +1149,25 @@ test_data: value: 1.fastq type: File """, history_id=history_id) - workflow_id = summary.workflow_id - invocation_id = summary.invocation_id - invocation_response = self._get("workflows/%s/invocations/%s" % (workflow_id, invocation_id)) - self._assert_status_code_is(invocation_response, 200) - invocation = invocation_response.json() - self._assert_has_keys(invocation , "id", "outputs", "output_collections") - assert len(invocation["output_collections"]) == 1 - assert len(invocation["outputs"]) == 0 - output_content = self.dataset_populator.get_history_collection_details(history_id, content_id=invocation["output_collections"]["wf_output_1"]["id"]) - self._assert_has_keys(output_content , "id", "elements") - assert output_content["collection_type"] == "list" - elements = output_content["elements"] - assert len(elements) == 1 - elements0 = elements[0] - assert elements0["element_identifier"] == "el1" + workflow_id = summary.workflow_id + invocation_id = summary.invocation_id + invocation_response = self._get("workflows/%s/invocations/%s" % (workflow_id, invocation_id)) + self._assert_status_code_is(invocation_response, 200) + invocation = invocation_response.json() + self._assert_has_keys(invocation , "id", "outputs", "output_collections") + assert len(invocation["output_collections"]) == 1 + assert len(invocation["outputs"]) == 0 + output_content = self.dataset_populator.get_history_collection_details(history_id, content_id=invocation["output_collections"]["wf_output_1"]["id"]) + self._assert_has_keys(output_content , "id", "elements") + assert output_content["collection_type"] == "list" + elements = output_content["elements"] + assert len(elements) == 1 + elements0 = elements[0] + assert elements0["element_identifier"] == "el1" def test_worklfow_input_mapping(self): - history_id = self.dataset_populator.new_history() - summary = self._run_jobs(""" + with self.dataset_populator.test_history() as history_id: + summary = self._run_jobs(""" class: GalaxyWorkflow inputs: - id: input1 @@ -1192,25 +1192,25 @@ test_data: value: 1.fastq type: File """, history_id=history_id) - workflow_id = summary.workflow_id - invocation_id = summary.invocation_id - invocation_response = self._get("workflows/%s/invocations/%s" % (workflow_id, invocation_id)) - self._assert_status_code_is(invocation_response, 200) - invocation = invocation_response.json() - self._assert_has_keys(invocation , "id", "outputs", "output_collections") - assert len(invocation["output_collections"]) == 1 - assert len(invocation["outputs"]) == 0 - output_content = self.dataset_populator.get_history_collection_details(history_id, content_id=invocation["output_collections"]["wf_output_1"]["id"]) - self._assert_has_keys(output_content , "id", "elements") - elements = output_content["elements"] - assert len(elements) == 2 - elements0 = elements[0] - assert elements0["element_identifier"] == "el1" + workflow_id = summary.workflow_id + invocation_id = summary.invocation_id + invocation_response = self._get("workflows/%s/invocations/%s" % (workflow_id, invocation_id)) + self._assert_status_code_is(invocation_response, 200) + invocation = invocation_response.json() + self._assert_has_keys(invocation , "id", "outputs", "output_collections") + assert len(invocation["output_collections"]) == 1 + assert len(invocation["outputs"]) == 0 + output_content = self.dataset_populator.get_history_collection_details(history_id, content_id=invocation["output_collections"]["wf_output_1"]["id"]) + self._assert_has_keys(output_content , "id", "elements") + elements = output_content["elements"] + assert len(elements) == 2 + elements0 = elements[0] + assert elements0["element_identifier"] == "el1" @skip_without_tool("collection_creates_pair") def test_workflow_run_input_mapping_with_output_collections(self): - history_id = self.dataset_populator.new_history() - summary = self._run_jobs(""" + with self.dataset_populator.test_history() as history_id: + summary = self._run_jobs(""" class: GalaxyWorkflow outputs: - id: wf_output_1 @@ -1235,21 +1235,21 @@ test_data: value: 1.fastq type: File """, history_id=history_id) - workflow_id = summary.workflow_id - invocation_id = summary.invocation_id - invocation_response = self._get("workflows/%s/invocations/%s" % (workflow_id, invocation_id)) - self._assert_status_code_is(invocation_response, 200) - invocation = invocation_response.json() - self._assert_has_keys(invocation , "id", "outputs", "output_collections") - assert len(invocation["output_collections"]) == 1 - assert len(invocation["outputs"]) == 0 - output_content = self.dataset_populator.get_history_collection_details(history_id, content_id=invocation["output_collections"]["wf_output_1"]["id"]) - self._assert_has_keys(output_content , "id", "elements") - assert output_content["collection_type"] == "list:paired", output_content - elements = output_content["elements"] - assert len(elements) == 2 - elements0 = elements[0] - assert elements0["element_identifier"] == "el1" + workflow_id = summary.workflow_id + invocation_id = summary.invocation_id + invocation_response = self._get("workflows/%s/invocations/%s" % (workflow_id, invocation_id)) + self._assert_status_code_is(invocation_response, 200) + invocation = invocation_response.json() + self._assert_has_keys(invocation , "id", "outputs", "output_collections") + assert len(invocation["output_collections"]) == 1 + assert len(invocation["outputs"]) == 0 + output_content = self.dataset_populator.get_history_collection_details(history_id, content_id=invocation["output_collections"]["wf_output_1"]["id"]) + self._assert_has_keys(output_content , "id", "elements") + assert output_content["collection_type"] == "list:paired", output_content + elements = output_content["elements"] + assert len(elements) == 2 + elements0 = elements[0] + assert elements0["element_identifier"] == "el1" def test_workflow_run_input_mapping_with_subworkflows(self): with self.dataset_populator.test_history() as history_id: @@ -1687,8 +1687,8 @@ test_data: assert invocation['state'] == 'cancelled' def test_run_with_implicit_connection(self): - history_id = self.dataset_populator.new_history() - run_summary = self._run_jobs(""" + with self.dataset_populator.test_history() as history_id: + run_summary = self._run_jobs(""" class: GalaxyWorkflow steps: - label: test_input @@ -1722,25 +1722,25 @@ steps: test_data: test_input: "hello world" """, history_id=history_id, wait=False) - history_id = run_summary.history_id - workflow_id = run_summary.workflow_id - invocation_id = run_summary.invocation_id - # Wait for first two jobs to be scheduled - upload and first cat. - wait_on(lambda: len(self._history_jobs(history_id)) >= 2 or None, "history jobs") - self.dataset_populator.wait_for_history(history_id, assert_ok=True) - invocation = self._invocation_details(workflow_id, invocation_id) - assert invocation['state'] != 'scheduled', invocation - # Expect two jobs - the upload and first cat. randomlines shouldn't run - # it is implicitly dependent on second cat. - self._assert_history_job_count(history_id, 2) + history_id = run_summary.history_id + workflow_id = run_summary.workflow_id + invocation_id = run_summary.invocation_id + # Wait for first two jobs to be scheduled - upload and first cat. + wait_on(lambda: len(self._history_jobs(history_id)) >= 2 or None, "history jobs") + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + invocation = self._invocation_details(workflow_id, invocation_id) + assert invocation['state'] != 'scheduled', invocation + # Expect two jobs - the upload and first cat. randomlines shouldn't run + # it is implicitly dependent on second cat. + self._assert_history_job_count(history_id, 2) - self.__review_paused_steps(workflow_id, invocation_id, order_index=2, action=True) - self.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id) - self._assert_history_job_count(history_id, 4) + self.__review_paused_steps(workflow_id, invocation_id, order_index=2, action=True) + self.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id) + self._assert_history_job_count(history_id, 4) def test_run_with_validated_parameter_connection_valid(self): - history_id = self.dataset_populator.new_history() - run_summary = self._run_jobs(""" + with self.dataset_populator.test_history() as history_id: + run_summary = self._run_jobs(""" class: GalaxyWorkflow inputs: - label: text_input @@ -1756,14 +1756,14 @@ test_data: value: "abd" type: raw """, history_id=history_id, wait=True) - time.sleep(10) - self.workflow_populator.wait_for_invocation(run_summary.workflow_id, run_summary.invocation_id) - jobs = self._history_jobs(history_id) - assert len(jobs) == 1 + time.sleep(10) + self.workflow_populator.wait_for_invocation(run_summary.workflow_id, run_summary.invocation_id) + jobs = self._history_jobs(history_id) + assert len(jobs) == 1 def test_run_with_validated_parameter_connection_invalid(self): - history_id = self.dataset_populator.new_history() - self._run_jobs(""" + with self.dataset_populator.test_history() as history_id: + self._run_jobs(""" class: GalaxyWorkflow inputs: - label: text_input @@ -1781,8 +1781,8 @@ test_data: """, history_id=history_id, wait=True, assert_ok=False) def test_run_with_text_connection(self): - history_id = self.dataset_populator.new_history() - self._run_jobs(""" + with self.dataset_populator.test_history() as history_id: + self._run_jobs(""" class: GalaxyWorkflow inputs: - label: data_input @@ -1809,9 +1809,9 @@ test_data: type: raw """, history_id=history_id) - self.dataset_populator.wait_for_history(history_id, assert_ok=True) - content = self.dataset_populator.get_history_dataset_content(history_id) - self.assertEqual("chr5\t131424298\t131424460\tCCDS4149.1_cds_0_0_chr5_131424299_f\t0\t+\n", content) + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + content = self.dataset_populator.get_history_dataset_content(history_id) + self.assertEqual("chr5\t131424298\t131424460\tCCDS4149.1_cds_0_0_chr5_131424299_f\t0\t+\n", content) def wait_for_invocation_and_jobs(self, history_id, workflow_id, invocation_id, assert_ok=True): state = self.workflow_populator.wait_for_invocation(workflow_id, invocation_id) @@ -1845,7 +1845,7 @@ test_data: first_wf_output = self._get("datasets/%s" % run_workflow_response['outputs'][0]).json() second_wf_output = self._get("datasets/%s" % new_workflow_response['outputs'][0]).json() assert first_wf_output['file_name'] == second_wf_output['file_name'], \ - "first output :\n%s\nsecond output: %s" % (first_wf_output, second_wf_output) + "first output:\n%s\nsecond output:\n%s" % (first_wf_output, second_wf_output) @skip_without_tool('cat1') def test_nested_workflow_rerun_with_use_cached_job(self): @@ -1860,7 +1860,7 @@ test_data: run_jobs_summary = self._run_jobs(workflow_run_description, history_id=history_id_one) self.dataset_populator.wait_for_history(history_id_one, assert_ok=True) workflow_request = run_jobs_summary.workflow_request - # We copy the inputs to a new history and re-reun the workflow + # We copy the inputs to a new history and re-run the workflow inputs = json.loads(workflow_request['inputs']) dataset_type = inputs['outer_input']['src'] dataset_id = inputs['outer_input']['id'] @@ -1915,20 +1915,20 @@ test_data: def test_workflow_run_with_matching_lists(self): workflow = self.workflow_populator.load_workflow_from_resource("test_workflow_matching_lists") workflow_id = self.workflow_populator.create_workflow(workflow) - history_id = self.dataset_populator.new_history() - hdca1 = self.dataset_collection_populator.create_list_in_history(history_id, contents=[("sample1-1", "1 2 3"), ("sample2-1", "7 8 9")]).json() - hdca2 = self.dataset_collection_populator.create_list_in_history(history_id, contents=[("sample1-2", "4 5 6"), ("sample2-2", "0 a b")]).json() - self.dataset_populator.wait_for_history(history_id, assert_ok=True) - label_map = {"list1": self._ds_entry(hdca1), "list2": self._ds_entry(hdca2)} - workflow_request = dict( - history="hist_id=%s" % history_id, - workflow_id=workflow_id, - ds_map=self._build_ds_map(workflow_id, label_map), - ) - run_workflow_response = self._post("workflows", data=workflow_request) - self._assert_status_code_is(run_workflow_response, 200) - self.dataset_populator.wait_for_history(history_id, assert_ok=True) - self.assertEqual("1 2 3\n4 5 6\n7 8 9\n0 a b\n", self.dataset_populator.get_history_dataset_content(history_id)) + with self.dataset_populator.test_history() as history_id: + hdca1 = self.dataset_collection_populator.create_list_in_history(history_id, contents=[("sample1-1", "1 2 3"), ("sample2-1", "7 8 9")]).json() + hdca2 = self.dataset_collection_populator.create_list_in_history(history_id, contents=[("sample1-2", "4 5 6"), ("sample2-2", "0 a b")]).json() + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + label_map = {"list1": self._ds_entry(hdca1), "list2": self._ds_entry(hdca2)} + workflow_request = dict( + history="hist_id=%s" % history_id, + workflow_id=workflow_id, + ds_map=self._build_ds_map(workflow_id, label_map), + ) + run_workflow_response = self._post("workflows", data=workflow_request) + self._assert_status_code_is(run_workflow_response, 200) + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + self.assertEqual("1 2 3\n4 5 6\n7 8 9\n0 a b\n", self.dataset_populator.get_history_dataset_content(history_id)) def test_workflow_stability(self): # Run this index stability test with following command: @@ -2001,8 +2001,8 @@ test_data: {} @skip_without_tool("cat") def test_run_rename_on_mapped_over_collection(self): - history_id = self.dataset_populator.new_history() - self._run_jobs(""" + with self.dataset_populator.test_history() as history_id: + self._run_jobs(""" class: GalaxyWorkflow inputs: - id: input1 @@ -2026,19 +2026,19 @@ test_data: value: 1.fastq type: File """, history_id=history_id) - content = self.dataset_populator.get_history_dataset_details(history_id, hid=4, wait=True, assert_ok=True) - name = content["name"] - assert name == "my new name", name - assert content["history_content_type"] == "dataset" - content = self.dataset_populator.get_history_collection_details(history_id, hid=3, wait=True, assert_ok=True) - name = content["name"] - assert content["history_content_type"] == "dataset_collection", content - assert name == "my new name", name + content = self.dataset_populator.get_history_dataset_details(history_id, hid=4, wait=True, assert_ok=True) + name = content["name"] + assert name == "my new name", name + assert content["history_content_type"] == "dataset" + content = self.dataset_populator.get_history_collection_details(history_id, hid=3, wait=True, assert_ok=True) + name = content["name"] + assert content["history_content_type"] == "dataset_collection", content + assert name == "my new name", name @skip_without_tool("cat") def test_run_rename_based_on_inputs_on_mapped_over_collection(self): - history_id = self.dataset_populator.new_history() - self._run_jobs(""" + with self.dataset_populator.test_history() as history_id: + self._run_jobs(""" class: GalaxyWorkflow inputs: - id: input1 @@ -2062,10 +2062,10 @@ test_data: value: 1.fastq type: File """, history_id=history_id) - content = self.dataset_populator.get_history_collection_details(history_id, hid=3, wait=True, assert_ok=True) - name = content["name"] - assert content["history_content_type"] == "dataset_collection", content - assert name == "the_dataset_list suffix", name + content = self.dataset_populator.get_history_collection_details(history_id, hid=3, wait=True, assert_ok=True) + name = content["name"] + assert content["history_content_type"] == "dataset_collection", content + assert name == "the_dataset_list suffix", name @skip_without_tool("collection_creates_pair") def test_run_rename_collection_output(self): @@ -2118,8 +2118,8 @@ test_data: {} @skip_without_tool("cat") def test_run_rename_based_on_input(self): - history_id = self.dataset_populator.new_history() - self._run_jobs(""" + with self.dataset_populator.test_history() as history_id: + self._run_jobs(""" class: GalaxyWorkflow inputs: - id: input1 @@ -2138,9 +2138,9 @@ test_data: type: File name: fasta1 """, history_id=history_id) - content = self.dataset_populator.get_history_dataset_details(history_id, wait=True, assert_ok=True) - name = content["name"] - assert name == "fasta1 suffix", name + content = self.dataset_populator.get_history_dataset_details(history_id, wait=True, assert_ok=True) + name = content["name"] + assert name == "fasta1 suffix", name @skip_without_tool("fail_identifier") @skip_without_tool("cat") @@ -2193,8 +2193,8 @@ test_data: @skip_without_tool("cat") def test_run_rename_based_on_input_recursive(self): - history_id = self.dataset_populator.new_history() - self._run_jobs(""" + with self.dataset_populator.test_history() as history_id: + self._run_jobs(""" class: GalaxyWorkflow inputs: - id: input1 @@ -2213,14 +2213,14 @@ test_data: type: File name: '#{input1}' """, history_id=history_id) - content = self.dataset_populator.get_history_dataset_details(history_id, wait=True, assert_ok=True) - name = content["name"] - assert name == "#{input1} #{INPUT1} suffix", name + content = self.dataset_populator.get_history_dataset_details(history_id, wait=True, assert_ok=True) + name = content["name"] + assert name == "#{input1} #{INPUT1} suffix", name @skip_without_tool("cat") def test_run_rename_based_on_input_repeat(self): - history_id = self.dataset_populator.new_history() - self._run_jobs(""" + with self.dataset_populator.test_history() as history_id: + self._run_jobs(""" class: GalaxyWorkflow inputs: - id: input1 @@ -2247,14 +2247,14 @@ test_data: type: File name: fasta2 """, history_id=history_id) - content = self.dataset_populator.get_history_dataset_details(history_id, wait=True, assert_ok=True) - name = content["name"] - assert name == "fasta2 suffix", name + content = self.dataset_populator.get_history_dataset_details(history_id, wait=True, assert_ok=True) + name = content["name"] + assert name == "fasta2 suffix", name @skip_without_tool("mapper2") def test_run_rename_based_on_input_conditional(self): - history_id = self.dataset_populator.new_history() - self._run_jobs(""" + with self.dataset_populator.test_history() as history_id: + self._run_jobs(""" class: GalaxyWorkflow inputs: - id: fasta_input @@ -2285,14 +2285,14 @@ test_data: name: fastq1 file_type: fastqsanger """, history_id=history_id) - content = self.dataset_populator.get_history_dataset_details(history_id, wait=True, assert_ok=True) - name = content["name"] - assert name == "fastq1 suffix", name + content = self.dataset_populator.get_history_dataset_details(history_id, wait=True, assert_ok=True) + name = content["name"] + assert name == "fastq1 suffix", name @skip_without_tool("mapper2") def test_run_rename_based_on_input_collection(self): - history_id = self.dataset_populator.new_history() - self._run_jobs(""" + with self.dataset_populator.test_history() as history_id: + self._run_jobs(""" class: GalaxyWorkflow inputs: - id: fasta_input @@ -2328,9 +2328,9 @@ test_data: value: 1.fastq type: File """, history_id=history_id) - content = self.dataset_populator.get_history_dataset_details(history_id, wait=True, assert_ok=True) - name = content["name"] - assert name == "the_dataset_pair suffix", name + content = self.dataset_populator.get_history_dataset_details(history_id, wait=True, assert_ok=True) + name = content["name"] + assert name == "the_dataset_pair suffix", name @skip_without_tool("collection_creates_pair") def test_run_hide_on_collection_output(self): @@ -2360,8 +2360,8 @@ test_data: @skip_without_tool("cat") def test_run_hide_on_mapped_over_collection(self): - history_id = self.dataset_populator.new_history() - self._run_jobs(""" + with self.dataset_populator.test_history() as history_id: + self._run_jobs(""" class: GalaxyWorkflow inputs: - id: input1 @@ -2386,13 +2386,13 @@ test_data: type: File """, history_id=history_id) - content = self.dataset_populator.get_history_dataset_details(history_id, hid=4, wait=True, assert_ok=True) - assert content["history_content_type"] == "dataset" - assert content["visible"] is False + content = self.dataset_populator.get_history_dataset_details(history_id, hid=4, wait=True, assert_ok=True) + assert content["history_content_type"] == "dataset" + assert content["visible"] is False - content = self.dataset_populator.get_history_collection_details(history_id, hid=3, wait=True, assert_ok=True) - assert content["history_content_type"] == "dataset_collection", content - assert content["visible"] is False + content = self.dataset_populator.get_history_collection_details(history_id, hid=3, wait=True, assert_ok=True) + assert content["history_content_type"] == "dataset_collection", content + assert content["visible"] is False @skip_without_tool("collection_creates_pair") def test_run_add_tag_on_collection_output(self): @@ -2552,43 +2552,42 @@ steps: $link: the_pause """) downloaded_workflow = self._download_workflow(workflow_id) - print(downloaded_workflow) uuid_dict = dict((int(index), step["uuid"]) for index, step in downloaded_workflow["steps"].items()) - history_id = self.dataset_populator.new_history() - hda = self.dataset_populator.new_dataset(history_id, content="1 2 3") - self.dataset_populator.wait_for_history(history_id) - inputs = { - '0': self._ds_entry(hda), - } - print(inputs) - uuid2 = uuid_dict[3] - workflow_request = {} - workflow_request["replacement_params"] = dumps(dict(replaceme="was replaced")) - pja_map = { - "RenameDatasetActionout_file1": dict( - action_type="RenameDatasetAction", - output_name="out_file1", - action_arguments=dict(newname="foo ${replaceme}"), - ) - } - workflow_request["parameters"] = dumps({ - uuid2: {"__POST_JOB_ACTIONS__": pja_map} - }) - invocation_id = self.__invoke_workflow(history_id, workflow_id, inputs=inputs, request=workflow_request) + with self.dataset_populator.test_history() as history_id: + hda = self.dataset_populator.new_dataset(history_id, content="1 2 3") + self.dataset_populator.wait_for_history(history_id) + inputs = { + '0': self._ds_entry(hda), + } + print(inputs) + uuid2 = uuid_dict[3] + workflow_request = {} + workflow_request["replacement_params"] = dumps(dict(replaceme="was replaced")) + pja_map = { + "RenameDatasetActionout_file1": dict( + action_type="RenameDatasetAction", + output_name="out_file1", + action_arguments=dict(newname="foo ${replaceme}"), + ) + } + workflow_request["parameters"] = dumps({ + uuid2: {"__POST_JOB_ACTIONS__": pja_map} + }) + invocation_id = self.__invoke_workflow(history_id, workflow_id, inputs=inputs, request=workflow_request) - time.sleep(2) - self.dataset_populator.wait_for_history(history_id) - self.__review_paused_steps(workflow_id, invocation_id, order_index=2, action=True) + time.sleep(2) + self.dataset_populator.wait_for_history(history_id) + self.__review_paused_steps(workflow_id, invocation_id, order_index=2, action=True) - self.workflow_populator.wait_for_workflow(workflow_id, invocation_id, history_id) - time.sleep(1) - content = self.dataset_populator.get_history_dataset_details(history_id) - assert content["name"] == "foo was replaced", content["name"] + self.workflow_populator.wait_for_workflow(workflow_id, invocation_id, history_id) + time.sleep(1) + content = self.dataset_populator.get_history_dataset_details(history_id) + assert content["name"] == "foo was replaced", content["name"] @skip_without_tool("cat1") def test_delete_intermediate_datasets_pja_1(self): - history_id = self.dataset_populator.new_history() - self._run_jobs(""" + with self.dataset_populator.test_history() as history_id: + self._run_jobs(""" class: GalaxyWorkflow inputs: - id: input1 @@ -2617,17 +2616,17 @@ steps: test_data: input1: "hello world" """, history_id=history_id) - hda1 = self.dataset_populator.get_history_dataset_details(history_id, hid=1) - hda2 = self.dataset_populator.get_history_dataset_details(history_id, hid=2) - hda3 = self.dataset_populator.get_history_dataset_details(history_id, hid=3) - hda4 = self.dataset_populator.get_history_dataset_details(history_id, hid=4) - assert not hda1["deleted"] - assert hda2["deleted"] - # I think hda3 should be deleted, but the inputs to - # steps with workflow outputs are not deleted. - # assert hda3["deleted"] - print(hda3["deleted"]) - assert not hda4["deleted"] + hda1 = self.dataset_populator.get_history_dataset_details(history_id, hid=1) + hda2 = self.dataset_populator.get_history_dataset_details(history_id, hid=2) + hda3 = self.dataset_populator.get_history_dataset_details(history_id, hid=3) + hda4 = self.dataset_populator.get_history_dataset_details(history_id, hid=4) + assert not hda1["deleted"] + assert hda2["deleted"] + # I think hda3 should be deleted, but the inputs to + # steps with workflow outputs are not deleted. + # assert hda3["deleted"] + print(hda3["deleted"]) + assert not hda4["deleted"] @skip_without_tool("random_lines1") def test_run_replace_params_by_tool(self): @@ -2659,39 +2658,39 @@ test_data: def test_run_batch(self): workflow = self.workflow_populator.load_workflow_from_resource("test_workflow_batch") workflow_id = self.workflow_populator.create_workflow(workflow) - history_id = self.dataset_populator.new_history() - hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3") - hda2 = self.dataset_populator.new_dataset(history_id, content="4 5 6") - hda3 = self.dataset_populator.new_dataset(history_id, content="7 8 9") - hda4 = self.dataset_populator.new_dataset(history_id, content="10 11 12") - parameters = { - "0": {"input": {"batch": True, "values": [{"id" : hda1.get("id"), "hid": hda1.get("hid"), "src": "hda"}, - {"id" : hda2.get("id"), "hid": hda2.get("hid"), "src": "hda"}, - {"id" : hda3.get("id"), "hid": hda2.get("hid"), "src": "hda"}, - {"id" : hda4.get("id"), "hid": hda2.get("hid"), "src": "hda"}]}}, - "1": {"input": {"batch": False, "values": [{"id" : hda1.get("id"), "hid": hda1.get("hid"), "src": "hda"}]}, "exp": "2"}} - workflow_request = { - "history_id" : history_id, - "batch" : True, - "parameters_normalized": True, - "parameters" : dumps(parameters), - } - invocation_response = self._post("workflows/%s/usage" % workflow_id, data=workflow_request) - self._assert_status_code_is(invocation_response, 200) - time.sleep(5) - self.dataset_populator.wait_for_history(history_id, assert_ok=True) - r1 = "1 2 3\t1\n1 2 3\t2\n" - r2 = "4 5 6\t1\n1 2 3\t2\n" - r3 = "7 8 9\t1\n1 2 3\t2\n" - r4 = "10 11 12\t1\n1 2 3\t2\n" - t1 = self.dataset_populator.get_history_dataset_content(history_id, hid=7) - t2 = self.dataset_populator.get_history_dataset_content(history_id, hid=10) - t3 = self.dataset_populator.get_history_dataset_content(history_id, hid=13) - t4 = self.dataset_populator.get_history_dataset_content(history_id, hid=16) - self.assertEqual(r1, t1) - self.assertEqual(r2, t2) - self.assertEqual(r3, t3) - self.assertEqual(r4, t4) + with self.dataset_populator.test_history() as history_id: + hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3") + hda2 = self.dataset_populator.new_dataset(history_id, content="4 5 6") + hda3 = self.dataset_populator.new_dataset(history_id, content="7 8 9") + hda4 = self.dataset_populator.new_dataset(history_id, content="10 11 12") + parameters = { + "0": {"input": {"batch": True, "values": [{"id" : hda1.get("id"), "hid": hda1.get("hid"), "src": "hda"}, + {"id" : hda2.get("id"), "hid": hda2.get("hid"), "src": "hda"}, + {"id" : hda3.get("id"), "hid": hda2.get("hid"), "src": "hda"}, + {"id" : hda4.get("id"), "hid": hda2.get("hid"), "src": "hda"}]}}, + "1": {"input": {"batch": False, "values": [{"id" : hda1.get("id"), "hid": hda1.get("hid"), "src": "hda"}]}, "exp": "2"}} + workflow_request = { + "history_id" : history_id, + "batch" : True, + "parameters_normalized": True, + "parameters" : dumps(parameters), + } + invocation_response = self._post("workflows/%s/usage" % workflow_id, data=workflow_request) + self._assert_status_code_is(invocation_response, 200) + time.sleep(5) + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + r1 = "1 2 3\t1\n1 2 3\t2\n" + r2 = "4 5 6\t1\n1 2 3\t2\n" + r3 = "7 8 9\t1\n1 2 3\t2\n" + r4 = "10 11 12\t1\n1 2 3\t2\n" + t1 = self.dataset_populator.get_history_dataset_content(history_id, hid=7) + t2 = self.dataset_populator.get_history_dataset_content(history_id, hid=10) + t3 = self.dataset_populator.get_history_dataset_content(history_id, hid=13) + t4 = self.dataset_populator.get_history_dataset_content(history_id, hid=16) + self.assertEqual(r1, t1) + self.assertEqual(r2, t2) + self.assertEqual(r3, t3) + self.assertEqual(r4, t4) @skip_without_tool("validation_default") def test_parameter_substitution_sanitization(self): @@ -2703,8 +2702,8 @@ test_data: @skip_without_tool("validation_repeat") def test_parameter_substitution_validation_value_errors_0(self): - history_id = self.dataset_populator.new_history() - workflow_id = self._upload_yaml_workflow(""" + with self.dataset_populator.test_history() as history_id: + workflow_id = self._upload_yaml_workflow(""" class: GalaxyWorkflow steps: - tool_id: validation_repeat @@ -2712,14 +2711,14 @@ steps: r2: - text: "abd" """) - workflow_request = dict( - history="hist_id=%s" % history_id, - parameters=dumps(dict(validation_repeat={"r2_0|text": ""})) - ) - url = "workflows/%s/invocations" % workflow_id - invocation_response = self._post(url, data=workflow_request) - # Take a valid stat and make it invalid, assert workflow won't run. - self._assert_status_code_is(invocation_response, 400) + workflow_request = dict( + history="hist_id=%s" % history_id, + parameters=dumps(dict(validation_repeat={"r2_0|text": ""})) + ) + url = "workflows/%s/invocations" % workflow_id + invocation_response = self._post(url, data=workflow_request) + # Take a valid stat and make it invalid, assert workflow won't run. + self._assert_status_code_is(invocation_response, 400) @skip_without_tool("validation_default") def test_parameter_substitution_validation_value_errors_1(self): @@ -2730,8 +2729,8 @@ steps: @skip_without_tool("validation_repeat") def test_workflow_import_state_validation_1(self): - history_id = self.dataset_populator.new_history() - self._run_jobs(""" + with self.dataset_populator.test_history() as history_id: + self._run_jobs(""" class: GalaxyWorkflow steps: - tool_id: validation_repeat diff --git a/test/base/populators.py b/test/base/populators.py index 22d1d15d87d..733270aae7f 100644 --- a/test/base/populators.py +++ b/test/base/populators.py @@ -2,6 +2,7 @@ import contextlib import json import os import time +import unittest from functools import wraps from operator import itemgetter @@ -33,10 +34,11 @@ def flakey(method): def wrapped_method(test_case, *args, **kwargs): try: method(test_case, *args, **kwargs) + except unittest.SkipTest: + raise except Exception: if SKIP_FLAKEY_TESTS_ON_ERROR: - from nose.plugins.skip import SkipTest - raise SkipTest() + raise unittest.SkipTest("Error encountered during test marked as @flakey.") else: raise @@ -46,7 +48,7 @@ def flakey(method): def skip_without_tool(tool_id): """Decorate an API test method as requiring a specific tool. - Have test framework skip the test case is the tool is unavailable. + Have test framework skip the test case if the tool is unavailable. """ def method_wrapper(method): @@ -71,7 +73,7 @@ def skip_without_tool(tool_id): def skip_without_datatype(extension): """Decorate an API test method as requiring a specific datatype. - Have test framework skip the test case is the tool is unavailable. + Have test framework skip the test case if the datatype is unavailable. """ def has_datatype(api_test_case): @@ -132,7 +134,7 @@ class TestsDatasets: class BaseDatasetPopulator(object): """ Abstract description of API operations optimized for testing - Galaxy - implementations must implement _get and _post. + Galaxy - implementations must implement _get, _post and _delete. """ def new_dataset(self, history_id, content=None, wait=False, **kwds): @@ -174,7 +176,7 @@ class BaseDatasetPopulator(object): def wait_for_history(self, history_id, assert_ok=False, timeout=DEFAULT_TIMEOUT): try: - return wait_on_state(lambda: self._get("histories/%s" % history_id), assert_ok=assert_ok, timeout=timeout) + return wait_on_state(lambda: self._get("histories/%s" % history_id), desc="history state", assert_ok=assert_ok, timeout=timeout) except AssertionError: self._summarize_history(history_id) raise @@ -182,22 +184,32 @@ class BaseDatasetPopulator(object): def wait_for_history_jobs(self, history_id, assert_ok=False, timeout=DEFAULT_TIMEOUT): query_params = {"history_id": history_id} - def has_active_jobs(): + def get_jobs(): jobs_response = self._get("jobs", query_params) assert jobs_response.status_code == 200 - active_jobs = [j for j in jobs_response.json() if j["state"] in ["new", "upload", "waiting", "queued", "running"]] + return jobs_response.json() + + def has_active_jobs(): + jobs = get_jobs() + active_jobs = [j for j in jobs if j["state"] in ["new", "upload", "waiting", "queued", "running"]] if len(active_jobs) == 0: return True else: return None - wait_on(has_active_jobs, "active jobs", timeout=timeout) + try: + wait_on(has_active_jobs, "active jobs", timeout=timeout) + except TimeoutAssertionError as e: + jobs = get_jobs() + message = "Failed waiting on active jobs to complete, current jobs are [%s]. %s" % (jobs, e.message) + raise TimeoutAssertionError(message) + if assert_ok: return self.wait_for_history(history_id, assert_ok=True, timeout=timeout) def wait_for_job(self, job_id, assert_ok=False, timeout=DEFAULT_TIMEOUT): - return wait_on_state(lambda: self.get_job_details(job_id), assert_ok=assert_ok, timeout=timeout) + return wait_on_state(lambda: self.get_job_details(job_id), desc="job state", assert_ok=assert_ok, timeout=timeout) def get_job_details(self, job_id, full=False): return self._get("jobs/%s?full=%s" % (job_id, full)) @@ -378,7 +390,7 @@ class DatasetPopulator(BaseDatasetPopulator): self.galaxy_interactor._summarize_history(history_id) def wait_for_dataset(self, history_id, dataset_id, assert_ok=False, timeout=DEFAULT_TIMEOUT): - return wait_on_state(lambda: self._get("histories/%s/contents/%s" % (history_id, dataset_id)), assert_ok=assert_ok, timeout=timeout) + return wait_on_state(lambda: self._get("histories/%s/contents/%s" % (history_id, dataset_id)), desc="dataset state", assert_ok=assert_ok, timeout=timeout) class BaseWorkflowPopulator(object): @@ -427,7 +439,7 @@ class BaseWorkflowPopulator(object): def wait_for_invocation(self, workflow_id, invocation_id, timeout=DEFAULT_TIMEOUT): url = "workflows/%s/usage/%s" % (workflow_id, invocation_id) - return wait_on_state(lambda: self._get(url), timeout=timeout) + return wait_on_state(lambda: self._get(url), desc="workflow invocation state", timeout=timeout) def wait_for_workflow(self, workflow_id, invocation_id, history_id, assert_ok=True, timeout=DEFAULT_TIMEOUT): """ Wait for a workflow invocation to completely schedule and then history @@ -782,7 +794,7 @@ class DatasetCollectionPopulator(BaseDatasetCollectionPopulator): return create_response -def wait_on_state(state_func, skip_states=["running", "queued", "new", "ready"], assert_ok=False, timeout=DEFAULT_TIMEOUT): +def wait_on_state(state_func, desc="state", skip_states=["running", "queued", "new", "ready"], assert_ok=False, timeout=DEFAULT_TIMEOUT): def get_state(): response = state_func() assert response.status_code == 200, "Failed to fetch state update while waiting." @@ -793,7 +805,11 @@ def wait_on_state(state_func, skip_states=["running", "queued", "new", "ready"], if assert_ok: assert state == "ok", "Final state - %s - not okay." % state return state - return wait_on(get_state, desc="state", timeout=timeout) + try: + return wait_on(get_state, desc=desc, timeout=timeout) + except TimeoutAssertionError as e: + response = state_func() + raise TimeoutAssertionError("%s Current response containing state [%s]." % (e.message, response.json())) class GiPostGetMixin: @@ -858,9 +874,15 @@ def wait_on(function, desc, timeout=DEFAULT_TIMEOUT): timeout_message = "Timed out after %s seconds waiting on %s." % ( total_wait, desc ) - assert False, timeout_message + raise TimeoutAssertionError(timeout_message) iteration += 1 value = function() if value is not None: return value time.sleep(delta) + + +class TimeoutAssertionError(AssertionError): + + def __init__(self, message): + super(TimeoutAssertionError, self).__init__(message) diff --git a/test/functional/tools/sample_datatypes_conf.xml b/test/functional/tools/sample_datatypes_conf.xml index 3b8b0dc0d58..8010f26e561 100644 --- a/test/functional/tools/sample_datatypes_conf.xml +++ b/test/functional/tools/sample_datatypes_conf.xml @@ -45,5 +45,7 @@ + + diff --git a/test/galaxy_selenium/navigates_galaxy.py b/test/galaxy_selenium/navigates_galaxy.py index 03727b15221..92387fe7ce4 100644 --- a/test/galaxy_selenium/navigates_galaxy.py +++ b/test/galaxy_selenium/navigates_galaxy.py @@ -43,7 +43,7 @@ WAIT_TYPES = Bunch( # Fade in, fade out, etc... UX_TRANSITION=WaitType("ux_transition", 5), # Toastr popup and dismissal, etc... - UX_POPUP=WaitType("ux_popup", 10), + UX_POPUP=WaitType("ux_popup", 15), # Creating a new history and loading it into the panel. DATABASE_OPERATION=WaitType("database_operation", 10), # Wait time for jobs to complete in default environment. @@ -711,8 +711,8 @@ class NavigatesGalaxy(HasDriver): def wait_for_overlays_cleared(self): """Wait for modals and Toast notifications to disappear.""" - self.wait_for_selector_absent_or_hidden(".ui-modal") - self.wait_for_selector_absent_or_hidden(".toast") + self.wait_for_selector_absent_or_hidden(".ui-modal", wait_type=WAIT_TYPES.UX_POPUP) + self.wait_for_selector_absent_or_hidden(".toast", wait_type=WAIT_TYPES.UX_POPUP) def workflow_index_open(self): self.home() diff --git a/test/integration/test_upload_configuration_options.py b/test/integration/test_upload_configuration_options.py index fa9548c230a..5aeb25deeac 100644 --- a/test/integration/test_upload_configuration_options.py +++ b/test/integration/test_upload_configuration_options.py @@ -117,9 +117,9 @@ class NonAdminsCannotPasteFilePathTestCase(BaseUploadContentConfigurationTestCas def test_disallowed_for_primary_file(self): payload = self.dataset_populator.upload_payload( - self.history_id, 'file://%s/1.RData' % TEST_DATA_DIRECTORY, ext="binary" + self.history_id, 'file://%s/1.RData' % TEST_DATA_DIRECTORY, file_type="binary" ) - create_response = self._post("tools", data=payload) + create_response = self.dataset_populator.tools_post(payload) # Ideally this would be 403 but the tool API endpoint isn't using # the newer API decorator that handles those details. assert create_response.status_code >= 400 @@ -139,7 +139,7 @@ class NonAdminsCannotPasteFilePathTestCase(BaseUploadContentConfigurationTestCas "files_2|type": "upload_dataset", }, ) - create_response = self._post("tools", data=payload) + create_response = self.dataset_populator.tools_post(payload) # Ideally this would be 403 but the tool API endpoint isn't using # the newer API decorator that handles those details. assert create_response.status_code >= 400 @@ -193,7 +193,7 @@ class AdminsCanPasteFilePathsTestCase(BaseUploadContentConfigurationTestCase): payload = self.dataset_populator.upload_payload( self.history_id, 'file://%s/random-file' % TEST_DATA_DIRECTORY, ) - create_response = self._post("tools", data=payload) + create_response = self.dataset_populator.tools_post(payload) # Is admin - so this should work fine! assert create_response.status_code == 200 @@ -246,7 +246,7 @@ class DefaultBinaryContentFiltersTestCase(BaseUploadContentConfigurationTestCase self.history_id, 'file://%s/random-file' % TEST_DATA_DIRECTORY, file_type="auto", wait=True ) dataset = self.dataset_populator.get_history_dataset_details(self.history_id, dataset=dataset) - assert dataset["file_ext"] == "data", dataset + assert dataset["file_ext"] == "binary", dataset def test_gzipped_html_content_blocked_by_default(self): dataset = self.dataset_populator.new_dataset( @@ -287,7 +287,7 @@ class AutoDecompressTestCase(BaseUploadContentConfigurationTestCase): self.history_id, 'file://%s/1.sam.gz' % TEST_DATA_DIRECTORY, file_type="auto", auto_decompress=False, wait=True ) dataset = self.dataset_populator.get_history_dataset_details(self.history_id, dataset=dataset) - assert dataset["file_ext"] == "data", dataset + assert dataset["file_ext"] == "binary", dataset def test_auto_decompress_on(self): dataset = self.dataset_populator.new_dataset( @@ -301,9 +301,9 @@ class LocalAddressWhitelisting(BaseUploadContentConfigurationTestCase): def test_blocked_url_for_primary_file(self): payload = self.dataset_populator.upload_payload( - self.history_id, 'http://localhost/', ext="txt" + self.history_id, 'http://localhost/', file_type="txt" ) - create_response = self._post("tools", data=payload) + create_response = self.dataset_populator.tools_post(payload) # Ideally this would be 403 but the tool API endpoint isn't using # the newer API decorator that handles those details. assert create_response.status_code >= 400 @@ -321,7 +321,7 @@ class LocalAddressWhitelisting(BaseUploadContentConfigurationTestCase): "files_2|type": "upload_dataset", }, ) - create_response = self._post("tools", data=payload) + create_response = self.dataset_populator.tools_post(payload) # Ideally this would be 403 but the tool API endpoint isn't using # the newer API decorator that handles those details. assert create_response.status_code >= 400 @@ -497,13 +497,12 @@ class AdvancedFtpUploadFetchTestCase(BaseFtpUploadConfigurationTestCase): "ftp_path": "subdir", "collection_type": "list", } - response = self.fetch_target(target) - self._assert_status_code_is(response, 200) + self.fetch_target(target, assert_ok=True) hdca = self.dataset_populator.get_history_collection_details(self.history_id, hid=1) assert len(hdca["elements"]) == 3, hdca element0 = hdca["elements"][0] - assert element0["element_identifier"] == "1" - assert element0["object"]["file_size"] == 9 + assert element0["element_identifier"] == "1", hdca + assert element0["object"]["file_size"] == 9, element0 def test_fetch_nested_elements_from(self): dir_path = self._get_user_ftp_path() @@ -519,8 +518,7 @@ class AdvancedFtpUploadFetchTestCase(BaseFtpUploadConfigurationTestCase): "elements": elements, "collection_type": "list:list", } - response = self.fetch_target(target) - self._assert_status_code_is(response, 200) + self.fetch_target(target, assert_ok=True) hdca = self.dataset_populator.get_history_collection_details(self.history_id, hid=1) assert len(hdca["elements"]) == 2, hdca element0 = hdca["elements"][0] diff --git a/test/selenium_tests/framework.py b/test/selenium_tests/framework.py index 1eb2f164803..1b93fad5a4c 100644 --- a/test/selenium_tests/framework.py +++ b/test/selenium_tests/framework.py @@ -110,33 +110,33 @@ def dump_test_information(self, name_prefix): buf.write(content.encode("utf-8") if not raw else content) os.makedirs(target_directory) - self.driver.save_screenshot(os.path.join(target_directory, "last.png")) - write_file("page_source.txt", self.driver.page_source) - write_file("DOM.txt", self.driver.execute_script("return document.documentElement.outerHTML")) write_file("stacktrace.txt", traceback.format_exc()) - for snapshot in getattr(self, "snapshots", []): snapshot.write_to_error_directory(write_file) + # Try to use the Selenium driver to recover more debug information, but don't + # throw an exception if the connection is broken in some way. + try: + self.driver.save_screenshot(os.path.join(target_directory, "last.png")) + write_file("page_source.txt", self.driver.page_source) + write_file("DOM.txt", self.driver.execute_script("return document.documentElement.outerHTML")) + except Exception: + print("Failed to use test driver to recover debug information from Selenium.") + write_file("selenium_exception.txt", traceback.format_exc()) + for log_type in ["browser", "driver"]: - full_log = self.driver.get_log(log_type) - trimmed_log = [l for l in full_log if l["level"] not in ["DEBUG", "INFO"]] try: + full_log = self.driver.get_log(log_type) + trimmed_log = [l for l in full_log if l["level"] not in ["DEBUG", "INFO"]] write_file("%s.log.json" % log_type, json.dumps(trimmed_log, indent=True)) write_file("%s.log.verbose.json" % log_type, json.dumps(full_log, indent=True)) except Exception: continue - iframes = self.driver.find_elements_by_css_selector("iframe") - for iframe in iframes: - pass - # TODO: Dump content out for debugging in the future. - # iframe_id = iframe.get_attribute("id") - # if iframe_id: - # write_file("iframe_%s" % iframe_id, "My content") @nottest def selenium_test(f): + test_name = f.__name__ @wraps(f) def func_wrapper(self, *args, **kwds): @@ -146,10 +146,15 @@ def selenium_test(f): self.reset_driver_and_session() try: return f(self, *args, **kwds) + except unittest.SkipTest: + dump_test_information(self, test_name) + # Don't retry if we have purposely decided to skip the test. + raise except Exception: - dump_test_information(self, f.__name__) + dump_test_information(self, test_name) if retry_attempts < GALAXY_TEST_SELENIUM_RETRIES: retry_attempts += 1 + print("Test function [%s] threw an exception, retrying. Failed attempts - %s." % (test_name, retry_attempts)) else: raise diff --git a/test/selenium_tests/test_collection_builders.py b/test/selenium_tests/test_collection_builders.py index 960ca4cc873..009defe2670 100644 --- a/test/selenium_tests/test_collection_builders.py +++ b/test/selenium_tests/test_collection_builders.py @@ -41,8 +41,8 @@ class CollectionBuildersTestCase(SeleniumTestCase): def test_build_pair_simple(self): self.perform_upload(self.get_filename("1.tabular")) self.perform_upload(self.get_filename("2.tabular")) - self.history_panel_wait_for_hid_visible(1) - self.history_panel_wait_for_hid_visible(2) + self._wait_for_hid_visible(1) + self._wait_for_hid_visible(2) self.history_panel_multi_operations_show() self.history_panel_muli_operation_select_hid(1) self.history_panel_muli_operation_select_hid(2) @@ -56,8 +56,8 @@ class CollectionBuildersTestCase(SeleniumTestCase): def test_build_paired_list_simple(self): self.perform_upload(self.get_filename("1.tabular")) self.perform_upload(self.get_filename("2.tabular")) - self.history_panel_wait_for_hid_visible(1) - self.history_panel_wait_for_hid_visible(2) + self._wait_for_hid_visible(1) + self._wait_for_hid_visible(2) self.history_panel_multi_operations_show() self.history_panel_muli_operation_select_hid(1) self.history_panel_muli_operation_select_hid(2) @@ -75,8 +75,8 @@ class CollectionBuildersTestCase(SeleniumTestCase): def test_build_paired_list_hide_original(self): self.perform_upload(self.get_filename("1.tabular")) self.perform_upload(self.get_filename("2.tabular")) - self.history_panel_wait_for_hid_visible(1) - self.history_panel_wait_for_hid_visible(2) + self._wait_for_hid_visible(1) + self._wait_for_hid_visible(2) self.history_panel_multi_operations_show() self.history_panel_muli_operation_select_hid(1) self.history_panel_muli_operation_select_hid(2) @@ -99,3 +99,6 @@ class CollectionBuildersTestCase(SeleniumTestCase): self.history_panel_wait_for_hid_ok(3) self.history_panel_wait_for_hid_hidden(1) self.history_panel_wait_for_hid_hidden(2) + + def _wait_for_hid_visible(self, hid): + self.history_panel_wait_for_hid_visible(hid, allowed_force_refreshes=1) diff --git a/test/selenium_tests/test_jupyter.py b/test/selenium_tests/test_jupyter.py index 7a8ee09cfbe..e92a7065d1b 100644 --- a/test/selenium_tests/test_jupyter.py +++ b/test/selenium_tests/test_jupyter.py @@ -13,8 +13,8 @@ class JupyterTestCase(SeleniumTestCase): ensure_registered = True - @flakey @selenium_test + @flakey @managed_history def test_jupyter_launch(self): self._stage_test_data_and_launch() @@ -29,8 +29,8 @@ class JupyterTestCase(SeleniumTestCase): finally: self.driver.switch_to.default_content() - @flakey @selenium_test + @flakey @managed_history def test_jupyter_interaction(self): self._stage_test_data_and_launch() diff --git a/test/selenium_tests/test_library_to_collections.py b/test/selenium_tests/test_library_to_collections.py index b2314f6db1f..ef32233c14a 100644 --- a/test/selenium_tests/test_library_to_collections.py +++ b/test/selenium_tests/test_library_to_collections.py @@ -1,16 +1,37 @@ +import unittest + from .framework import ( selenium_test, - SharedStateSeleniumTestCase, + SeleniumTestCase, ) -class LibraryToCollectionsTestCase(SharedStateSeleniumTestCase): +@unittest.skip +class LibraryToCollectionsTestCase(SeleniumTestCase): requires_admin = True @selenium_test def test_list_creation(self): self.admin_login() + self.perform_upload(self.get_filename("1.bed")) + self.history_panel_wait_for_hid_ok(1, allowed_force_refreshes=1) + self.perform_upload(self.get_filename("2.bed")) + self.history_panel_wait_for_hid_ok(2, allowed_force_refreshes=1) + + self.name = self._get_random_name(prefix="testcontents") + + self.libraries_open() + self.libraries_index_create(self.name) + self.libraries_open_with_name(self.name) + + self.libraries_dataset_import_from_history() + self.libraries_dataset_import_from_history_select(["1.bed", "2.bed"]) + self.sleep_for(self.wait_types.UX_RENDER) + self.libraries_dataset_import_from_history_click_ok() + + self.home() + self.history_panel_create_new_with_name("new_history_for_library_list") self.libraries_open_with_name(self.name) self.sleep_for(self.wait_types.UX_RENDER) @@ -26,21 +47,3 @@ class LibraryToCollectionsTestCase(SharedStateSeleniumTestCase): self.collection_builder_create() self.home() self.history_panel_wait_for_hid_ok(3) - - def setup_shared_state(self): - self.admin_login() - self.perform_upload(self.get_filename("1.bed")) - self.wait_for_history() - self.perform_upload(self.get_filename("2.bed")) - self.wait_for_history() - - self.name = self._get_random_name(prefix="testcontents") - - self.libraries_open() - self.libraries_index_create(self.name) - self.libraries_open_with_name(self.name) - - self.libraries_dataset_import_from_history() - self.libraries_dataset_import_from_history_select(["1.bed", "2.bed"]) - self.sleep_for(self.wait_types.UX_RENDER) - self.libraries_dataset_import_from_history_click_ok() diff --git a/test/selenium_tests/test_tool_form.py b/test/selenium_tests/test_tool_form.py index d880f682beb..0ba330b0f43 100644 --- a/test/selenium_tests/test_tool_form.py +++ b/test/selenium_tests/test_tool_form.py @@ -85,8 +85,8 @@ class ToolFormTestCase(SeleniumTestCase, UsesHistoryItemAssertions): self.history_panel_wait_for_hid_ok(2) self._check_dataset_details_for_inttest_value(2) - @flakey @selenium_test + @flakey def test_run_data(self): test_path = self.get_filename("1.fasta") test_path_decoy = self.get_filename("1.txt") diff --git a/test/shed_functional/test_data/emboss/datatypes/datatypes_conf.xml b/test/shed_functional/test_data/emboss/datatypes/datatypes_conf.xml index 5e9b8bc4717..097b9079c5a 100644 --- a/test/shed_functional/test_data/emboss/datatypes/datatypes_conf.xml +++ b/test/shed_functional/test_data/emboss/datatypes/datatypes_conf.xml @@ -97,5 +97,7 @@ + + diff --git a/test/unit/jobs/dynamic_tool_destination/data/job_conf.xml b/test/unit/jobs/dynamic_tool_destination/data/job_conf.xml new file mode 100644 index 00000000000..5fcf023f8e9 --- /dev/null +++ b/test/unit/jobs/dynamic_tool_destination/data/job_conf.xml @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/unit/jobs/dynamic_tool_destination/data/priority_tool_destination.yml b/test/unit/jobs/dynamic_tool_destination/data/priority_tool_destination.yml index 10238c88213..7198610ff9a 100644 --- a/test/unit/jobs/dynamic_tool_destination/data/priority_tool_destination.yml +++ b/test/unit/jobs/dynamic_tool_destination/data/priority_tool_destination.yml @@ -116,9 +116,9 @@ tools: default_destination: priority: - low: waffles_default_low - med: waffles_default_med - high: waffles_default_high + low: cluster_default_low + med: cluster_default_med + high: cluster_default_high users: user@email.com: priority: high diff --git a/test/unit/jobs/dynamic_tool_destination/data/test_no_verbose.yml b/test/unit/jobs/dynamic_tool_destination/data/test_no_verbose.yml index 4cacdaf700a..c5008efb98b 100644 --- a/test/unit/jobs/dynamic_tool_destination/data/test_no_verbose.yml +++ b/test/unit/jobs/dynamic_tool_destination/data/test_no_verbose.yml @@ -6,6 +6,6 @@ tools: lower_bound: 1 KB upper_bound: Infinity destination: Destination1 - default_destination: waffles_default -default_destination: waffles_default + default_destination: cluster_default +default_destination: cluster_default verbose: False diff --git a/test/unit/jobs/dynamic_tool_destination/data/test_num_input_datasets.yml b/test/unit/jobs/dynamic_tool_destination/data/test_num_input_datasets.yml index f15a4621c5d..78598cf2fca 100644 --- a/test/unit/jobs/dynamic_tool_destination/data/test_num_input_datasets.yml +++ b/test/unit/jobs/dynamic_tool_destination/data/test_num_input_datasets.yml @@ -1,6 +1,6 @@ tools: spades: - default_destination: waffles_default + default_destination: cluster_default smalt: rules: - rule_type: num_input_datasets @@ -13,5 +13,5 @@ tools: lower_bound: 200 upper_bound: Infinity destination: cluster_high_32 -default_destination: waffles_low +default_destination: cluster_low verbose: True diff --git a/test/unit/jobs/dynamic_tool_destination/data/tool_destination.yml b/test/unit/jobs/dynamic_tool_destination/data/tool_destination.yml index 3b78a4acbc2..82a2a14740b 100644 --- a/test/unit/jobs/dynamic_tool_destination/data/tool_destination.yml +++ b/test/unit/jobs/dynamic_tool_destination/data/tool_destination.yml @@ -22,7 +22,7 @@ tools: lower_bound: 5 upper_bound: Infinity destination: Destination3 - default_destination: waffles_default + default_destination: cluster_default test_overlap: rules: @@ -46,7 +46,7 @@ tools: lower_bound: 5 upper_bound: Infinity destination: Destination5 - default_destination: waffles_default + default_destination: cluster_default test_db: rules: @@ -61,7 +61,7 @@ tools: lower_bound: 0 upper_bound: 1 KB destination: Destination4 - default_destination: waffles_default + default_destination: cluster_default test_db_high: rules: @@ -70,7 +70,7 @@ tools: lower_bound: 0 upper_bound: Infinity destination: Destination5 - default_destination: waffles_default + default_destination: cluster_default test_arguments: rules: @@ -79,7 +79,7 @@ tools: arguments: careful: true destination: Destination6 - default_destination: waffles_default + default_destination: cluster_default -default_destination: waffles_default +default_destination: cluster_default verbose: True diff --git a/test/unit/jobs/dynamic_tool_destination/mockGalaxy.py b/test/unit/jobs/dynamic_tool_destination/mockGalaxy.py index 04a33c672c8..49b5fda5c66 100644 --- a/test/unit/jobs/dynamic_tool_destination/mockGalaxy.py +++ b/test/unit/jobs/dynamic_tool_destination/mockGalaxy.py @@ -7,6 +7,7 @@ class Job(object): self.input_datasets = [] self.input_library_datasets = [] self.param_values = dict() + self.parameters = [] def get_param_values(self, app, ignore_errors=False): return self.param_values @@ -17,6 +18,9 @@ class Job(object): def add_input_dataset(self, dataset): self.input_datasets.append(dataset) + def get_parameters(self): + return self.parameters + class InputDataset(object): def __init__(self, name, dataset): @@ -75,14 +79,22 @@ class JobConfig(object): self.info = namedtuple('info', ['id', 'nativeSpec', 'runner']) self.tool_id = tool_id self.nativeSpec = params - self.default_id = "waffles_default" + self.default_id = "cluster_default" self.defNativeSpec = "-q test.q" self.defRunner = "drmaa" self.keys = {tool_id: self.info(self.tool_id, self.nativeSpec, self.defRunner), - "waffles_default": self.info(self.default_id, self.defNativeSpec, self.defRunner), } + "cluster_default": self.info(self.default_id, self.defNativeSpec, self.defRunner), } def get_destination(self, tool_id): - return self.keys[tool_id] + invalid_destinations = ["cluster-kow", "destinationf", "thig", + "not_true_destination", "cluster_kow", + "Destination_3_med", "fake_destination", + "cluster_defaut", "even_lamerr_cluster", + "no_such_dest"] + if tool_id in invalid_destinations: + return None + else: + return tool_id # JobMappingException mock======================================= diff --git a/test/unit/jobs/dynamic_tool_destination/test_dynamic_tool_destination.py b/test/unit/jobs/dynamic_tool_destination/test_dynamic_tool_destination.py index 7c9bfe5a7ef..a46dc566b64 100644 --- a/test/unit/jobs/dynamic_tool_destination/test_dynamic_tool_destination.py +++ b/test/unit/jobs/dynamic_tool_destination/test_dynamic_tool_destination.py @@ -10,7 +10,7 @@ from galaxy.jobs.mapper import JobMappingException from . import mockGalaxy as mg from . import ymltests as yt -theApp = mg.App("waffles_default", "test_spec") +theApp = mg.App("cluster_default", "test_spec") script_dir = os.path.dirname(__file__) # ======================Jobs==================================== @@ -65,13 +65,14 @@ usersTool = mg.Tool('test_users') numinputsTool = mg.Tool('test_num_input_datasets') -# =======================YML file================================ +# =======================Configuration files================================ path = script_dir + "/data/tool_destination.yml" priority_path = script_dir + "/data/priority_tool_destination.yml" broken_default_dest_path = script_dir + "/data/dest_fail.yml" no_verbose_path = script_dir + "/data/test_no_verbose.yml" users_test_path = script_dir + "/data/test_users.yml" num_input_datasets_test_path = script_dir + "/data/test_num_input_datasets.yml" +job_conf_path = script_dir + "/data/job_conf.xml" # ======================Test Variables========================= value = 1 @@ -94,20 +95,20 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_brokenDestYML(self, l): - self.assertRaises(JobMappingException, map_tool_to_destination, runJob, theApp, vanillaTool, "user@email.com", True, broken_default_dest_path) + self.assertRaises(JobMappingException, map_tool_to_destination, runJob, theApp, vanillaTool, "user@email.com", True, broken_default_dest_path, job_conf_path) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'No global default destination specified in config!'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Loading file: input1' + script_dir + '/data/test3.full'), - ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Total size: 3.23 KB') + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Total size: 3.23 KB'), ) @log_capture() def test_filesize_empty(self, l): - self.assertRaises(JobMappingException, map_tool_to_destination, emptyJob, theApp, vanillaTool, "user@email.com", True, path) - self.assertRaises(JobMappingException, map_tool_to_destination, emptyJob, theApp, vanillaTool, "user@email.com", True, priority_path) + self.assertRaises(JobMappingException, map_tool_to_destination, emptyJob, theApp, vanillaTool, "user@email.com", True, path, job_conf_path) + self.assertRaises(JobMappingException, map_tool_to_destination, emptyJob, theApp, vanillaTool, "user@email.com", True, priority_path, job_conf_path) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), @@ -116,6 +117,7 @@ class TestDynamicToolDestination(unittest.TestCase): ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Total size: 0.00 B'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Total number of files: 1'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No default_priority section found in config. Setting 'med' as default priority."), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Loading file: input1' + script_dir + '/data/test.empty'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Total size: 0.00 B'), @@ -124,8 +126,8 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_filesize_zero(self, l): - self.assertRaises(JobMappingException, map_tool_to_destination, zeroJob, theApp, vanillaTool, "user@email.com", True, path) - self.assertRaises(JobMappingException, map_tool_to_destination, zeroJob, theApp, vanillaTool, "user@email.com", True, priority_path) + self.assertRaises(JobMappingException, map_tool_to_destination, zeroJob, theApp, vanillaTool, "user@email.com", True, path, job_conf_path) + self.assertRaises(JobMappingException, map_tool_to_destination, zeroJob, theApp, vanillaTool, "user@email.com", True, priority_path, job_conf_path) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), @@ -133,6 +135,7 @@ class TestDynamicToolDestination(unittest.TestCase): ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Total size: 0.00 B'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Total number of files: 0'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No default_priority section found in config. Setting 'med' as default priority."), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Total size: 0.00 B'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Total number of files: 0') @@ -140,8 +143,8 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_filesize_fail(self, l): - self.assertRaises(JobMappingException, map_tool_to_destination, failJob, theApp, vanillaTool, "user@email.com", True, path) - self.assertRaises(JobMappingException, map_tool_to_destination, failJob, theApp, vanillaTool, "user@email.com", True, priority_path) + self.assertRaises(JobMappingException, map_tool_to_destination, failJob, theApp, vanillaTool, "user@email.com", True, path, job_conf_path) + self.assertRaises(JobMappingException, map_tool_to_destination, failJob, theApp, vanillaTool, "user@email.com", True, priority_path, job_conf_path) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), @@ -150,6 +153,7 @@ class TestDynamicToolDestination(unittest.TestCase): ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Total size: 293.00 B'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Total number of files: 1'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No default_priority section found in config. Setting 'med' as default priority."), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Loading file: input1' + script_dir + '/data/test1.full'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Total size: 293.00 B'), @@ -158,9 +162,9 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_filesize_run(self, l): - job = map_tool_to_destination(runJob, theApp, vanillaTool, "user@email.com", True, path) + job = map_tool_to_destination(runJob, theApp, vanillaTool, "user@email.com", True, path, job_conf_path) self.assertEquals(job, 'Destination1') - priority_job = map_tool_to_destination(runJob, theApp, vanillaTool, "user@email.com", True, priority_path) + priority_job = map_tool_to_destination(runJob, theApp, vanillaTool, "user@email.com", True, priority_path, job_conf_path) self.assertEquals(priority_job, 'Destination1_high') l.check( @@ -171,6 +175,7 @@ class TestDynamicToolDestination(unittest.TestCase): ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Total number of files: 1'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Running 'test' with 'Destination1'."), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No default_priority section found in config. Setting 'med' as default priority."), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Loading file: input1' + script_dir + '/data/test3.full'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Total size: 3.23 KB'), @@ -180,27 +185,28 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_default_tool(self, l): - job = map_tool_to_destination(runJob, theApp, defaultTool, "user@email.com", True, path) - self.assertEquals(job, 'waffles_default') - priority_job = map_tool_to_destination(runJob, theApp, defaultTool, "user@email.com", True, priority_path) - self.assertEquals(priority_job, 'waffles_default_high') + job = map_tool_to_destination(runJob, theApp, defaultTool, "user@email.com", True, path, job_conf_path) + self.assertEquals(job, 'cluster_default') + priority_job = map_tool_to_destination(runJob, theApp, defaultTool, "user@email.com", True, priority_path, job_conf_path) + self.assertEquals(priority_job, 'cluster_default_high') l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Tool 'test_tooldefault' not specified in config. Using default destination."), - ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Running 'test_tooldefault' with 'waffles_default'."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Running 'test_tooldefault' with 'cluster_default'."), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No default_priority section found in config. Setting 'med' as default priority."), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Tool 'test_tooldefault' not specified in config. Using default destination."), - ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Running 'test_tooldefault' with 'waffles_default_high'.") + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Running 'test_tooldefault' with 'cluster_default_high'.") ) @log_capture() def test_arguments_tool(self, l): - job = map_tool_to_destination(argJob, theApp, argTool, "user@email.com", True, path) + job = map_tool_to_destination(argJob, theApp, argTool, "user@email.com", True, path, job_conf_path) self.assertEquals(job, 'Destination6') - priority_job = map_tool_to_destination(argJob, theApp, argTool, "user@email.com", True, priority_path) + priority_job = map_tool_to_destination(argJob, theApp, argTool, "user@email.com", True, priority_path, job_conf_path) self.assertEquals(priority_job, 'Destination6_med') l.check( @@ -208,49 +214,52 @@ class TestDynamicToolDestination(unittest.TestCase): ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Running 'test_arguments' with 'Destination6'."), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No default_priority section found in config. Setting 'med' as default priority."), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Running 'test_arguments' with 'Destination6_med'.") ) @log_capture() def test_arguments_arg_not_found(self, l): - job = map_tool_to_destination(argNotFoundJob, theApp, argTool, "user@email.com", True, path) - self.assertEquals(job, 'waffles_default') - priority_job = map_tool_to_destination(argNotFoundJob, theApp, argTool, "user@email.com", True, priority_path) - self.assertEquals(priority_job, 'waffles_default_high') + job = map_tool_to_destination(argNotFoundJob, theApp, argTool, "user@email.com", True, path, job_conf_path) + self.assertEquals(job, 'cluster_default') + priority_job = map_tool_to_destination(argNotFoundJob, theApp, argTool, "user@email.com", True, priority_path, job_conf_path) + self.assertEquals(priority_job, 'cluster_default_high') l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.'), - ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Running 'test_arguments' with 'waffles_default'."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Running 'test_arguments' with 'cluster_default'."), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No default_priority section found in config. Setting 'med' as default priority."), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.'), - ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Running 'test_arguments' with 'waffles_default_high'.") + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Running 'test_arguments' with 'cluster_default_high'.") ) @log_capture() def test_tool_not_found(self, l): - job = map_tool_to_destination(runJob, theApp, unTool, "user@email.com", True, path) - self.assertEquals(job, 'waffles_default') - priority_job = map_tool_to_destination(runJob, theApp, unTool, "user@email.com", True, priority_path) - self.assertEquals(priority_job, 'waffles_default_high') + job = map_tool_to_destination(runJob, theApp, unTool, "user@email.com", True, path, job_conf_path) + self.assertEquals(job, 'cluster_default') + priority_job = map_tool_to_destination(runJob, theApp, unTool, "user@email.com", True, priority_path, job_conf_path) + self.assertEquals(priority_job, 'cluster_default_high') l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Tool 'unregistered' not specified in config. Using default destination."), - ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Running 'unregistered' with 'waffles_default'."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Running 'unregistered' with 'cluster_default'."), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No default_priority section found in config. Setting 'med' as default priority."), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Tool 'unregistered' not specified in config. Using default destination."), - ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Running 'unregistered' with 'waffles_default_high'.") + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Running 'unregistered' with 'cluster_default_high'.") ) @log_capture() def test_fasta(self, l): - job = map_tool_to_destination(dbJob, theApp, dbTool, "user@email.com", True, path) + job = map_tool_to_destination(dbJob, theApp, dbTool, "user@email.com", True, path, job_conf_path) self.assertEquals(job, 'Destination4') - priority_job = map_tool_to_destination(dbJob, theApp, dbTool, "user@email.com", True, priority_path) + priority_job = map_tool_to_destination(dbJob, theApp, dbTool, "user@email.com", True, priority_path, job_conf_path) self.assertEquals(priority_job, 'Destination4_high') l.check( @@ -260,6 +269,7 @@ class TestDynamicToolDestination(unittest.TestCase): ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Total amount of records: 10'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Running 'test_db' with 'Destination4'."), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No default_priority section found in config. Setting 'med' as default priority."), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Loading file: input1' + script_dir + '/data/test.fasta'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Total amount of records: 10'), @@ -268,9 +278,9 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_fasta_count(self, l): - job = map_tool_to_destination(dbcountJob, theApp, dbTool, "user@email.com", True, path) + job = map_tool_to_destination(dbcountJob, theApp, dbTool, "user@email.com", True, path, job_conf_path) self.assertEquals(job, 'Destination4') - priority_job = map_tool_to_destination(dbcountJob, theApp, dbTool, "user@email.com", True, priority_path) + priority_job = map_tool_to_destination(dbcountJob, theApp, dbTool, "user@email.com", True, priority_path, job_conf_path) self.assertEquals(priority_job, 'Destination4_high') l.check( @@ -280,6 +290,7 @@ class TestDynamicToolDestination(unittest.TestCase): ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Total amount of records: 6'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Running 'test_db' with 'Destination4'."), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No default_priority section found in config. Setting 'med' as default priority."), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Loading file: input1' + script_dir + '/data/test.fasta'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Total amount of records: 6'), @@ -288,7 +299,7 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_no_verbose(self, l): - job = map_tool_to_destination(runJob, theApp, noVBTool, "user@email.com", True, no_verbose_path) + job = map_tool_to_destination(runJob, theApp, noVBTool, "user@email.com", True, no_verbose_path, job_conf_path) self.assertEquals(job, 'Destination1') l.check( @@ -297,7 +308,7 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_authorized_user(self, l): - job = map_tool_to_destination(runJob, theApp, usersTool, "user@email.com", True, users_test_path) + job = map_tool_to_destination(runJob, theApp, usersTool, "user@email.com", True, users_test_path, job_conf_path) self.assertEquals(job, 'special_cluster') l.check( @@ -306,7 +317,7 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_unauthorized_user(self, l): - job = map_tool_to_destination(runJob, theApp, usersTool, "userblah@email.com", True, users_test_path) + job = map_tool_to_destination(runJob, theApp, usersTool, "userblah@email.com", True, users_test_path, job_conf_path) self.assertEquals(job, 'lame_cluster') l.check( @@ -322,7 +333,7 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_bad_nice(self, l): - dt.parse_yaml(path=yt.ivYMLTest11, test=True) + dt.parse_yaml(path=yt.ivYMLTest11, job_conf_path=job_conf_path, test=True) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Running config validation..."), @@ -333,11 +344,11 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_empty_file(self, l): - self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest2, test=True), {}) + self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest2, job_conf_path=job_conf_path, test=True), {}) @log_capture() def test_no_tool_name(self, l): - self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest3, test=True), yt.iv3dict) + self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest3, job_conf_path=job_conf_path, test=True), yt.iv3dict) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Malformed YML; expected job name, but found a list instead!'), @@ -346,7 +357,7 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_no_rule_type(self, l): - self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest4, test=True), yt.ivDict) + self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest4, job_conf_path=job_conf_path, test=True), yt.ivDict) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No rule_type found for rule 1 in 'spades'."), @@ -355,7 +366,7 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_no_rule_lower_bound(self, l): - self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest51, test=True), yt.ivDict) + self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest51, job_conf_path=job_conf_path, test=True), yt.ivDict) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Missing bounds for rule 1 in 'spades'. Ignoring rule."), @@ -364,7 +375,7 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_no_rule_upper_bound(self, l): - self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest52, test=True), yt.ivDict) + self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest52, job_conf_path=job_conf_path, test=True), yt.ivDict) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Missing bounds for rule 1 in 'spades'. Ignoring rule."), @@ -373,7 +384,7 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_no_rule_arg(self, l): - self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest53, test=True), yt.ivDict53) + self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest53, job_conf_path=job_conf_path, test=True), yt.ivDict53) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Found a fail_message for rule 1 in 'spades', but destination is not 'fail'! Setting destination to 'fail'."), @@ -382,7 +393,7 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_bad_rule_type(self, l): - self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest6, test=True), yt.ivDict) + self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest6, job_conf_path=job_conf_path, test=True), yt.ivDict) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Unrecognized rule_type 'iencs' found in 'spades'. Ignoring..."), @@ -391,7 +402,7 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_no_err_msg(self, l): - self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest91, test=True), yt.iv91dict) + self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest91, job_conf_path=job_conf_path, test=True), yt.iv91dict) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No nice_value found for rule 1 in 'spades'. Setting nice_value to 0."), @@ -401,7 +412,7 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_no_default_dest(self, l): - dt.parse_yaml(path=yt.ivYMLTest7, test=True) + dt.parse_yaml(path=yt.ivYMLTest7, job_conf_path=job_conf_path, test=True) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'No global default destination specified in config!'), @@ -410,7 +421,7 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_invalid_category(self, l): - dt.parse_yaml(path=yt.ivYMLTest8, test=True) + dt.parse_yaml(path=yt.ivYMLTest8, job_conf_path=job_conf_path, test=True) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'No global default destination specified in config!'), @@ -420,27 +431,27 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_arguments_no_err_msg(self, l): - self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest12, test=True), yt.iv12dict) + self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest12, job_conf_path=job_conf_path, test=True), yt.iv12dict) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', - "Missing a fail_message for rule 1 in 'spades'. Adding generic fail_message."), + "Missing a fail_message for rule 1 in 'spades'. Adding generic fail_message."), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') ) @log_capture() def test_arguments_no_args(self, l): - self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest131, test=True), yt.iv131dict) + self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest131, job_conf_path=job_conf_path, test=True), yt.iv131dict) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', - "No arguments found for rule 1 in 'spades' despite being of type arguments. Ignoring rule."), + "No arguments found for rule 1 in 'spades' despite being of type arguments. Ignoring rule."), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') ) @log_capture() def test_arguments_no_arg(self, l): - self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest132, test=True), yt.iv132dict) + self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest132, job_conf_path=job_conf_path, test=True), yt.iv132dict) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Found a fail_message for rule 1 in 'spades', but destination is not 'fail'! Setting destination to 'fail'."), @@ -449,14 +460,14 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_return_bool_for_multiple_jobs(self, l): - self.assertFalse(dt.parse_yaml(path=yt.ivYMLTest133, test=True, return_bool=True)) + self.assertFalse(dt.parse_yaml(path=yt.ivYMLTest133, job_conf_path=job_conf_path, test=True, return_bool=True)) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Missing a fail_message for rule 1 in 'smalt'.") ) @log_capture() def test_return_rule_for_multiple_jobs(self, l): - self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest133, test=True), yt.iv133dict) + self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest133, job_conf_path=job_conf_path, test=True), yt.iv133dict) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Missing a fail_message for rule 1 in 'smalt'. Adding generic fail_message."), @@ -465,14 +476,14 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_return_bool_for_no_destination(self, l): - self.assertFalse(dt.parse_yaml(path=yt.ivYMLTest134, test=True, return_bool=True)) + self.assertFalse(dt.parse_yaml(path=yt.ivYMLTest134, job_conf_path=job_conf_path, test=True, return_bool=True)) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No destination specified for rule 1 in 'spades'.") ) @log_capture() def test_return_rule_for_no_destination(self, l): - self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest134, test=True), yt.iv134dict) + self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest134, job_conf_path=job_conf_path, test=True), yt.iv134dict) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No destination specified for rule 1 in 'spades'. Ignoring..."), @@ -481,7 +492,7 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_return_rule_for_reversed_bounds(self, l): - self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest135, test=True), yt.iv135dict) + self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest135, job_conf_path=job_conf_path, test=True), yt.iv135dict) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "lower_bound exceeds upper_bound for rule 1 in 'spades'. Reversing bounds."), @@ -490,14 +501,14 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_return_bool_for_missing_tool_fields(self, l): - self.assertFalse(dt.parse_yaml(path=yt.ivYMLTest136, test=True, return_bool=True)) + self.assertFalse(dt.parse_yaml(path=yt.ivYMLTest136, job_conf_path=job_conf_path, test=True, return_bool=True)) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Tool 'spades' does not have rules nor a default_destination!") ) @log_capture() def test_return_rule_for_missing_tool_fields(self, l): - self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest136, test=True), yt.iv136dict) + self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest136, job_conf_path=job_conf_path, test=True), yt.iv136dict) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Tool 'spades' does not have rules nor a default_destination!"), @@ -506,14 +517,14 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_return_bool_for_blank_tool(self, l): - self.assertFalse(dt.parse_yaml(path=yt.ivYMLTest137, test=True, return_bool=True)) + self.assertFalse(dt.parse_yaml(path=yt.ivYMLTest137, job_conf_path=job_conf_path, test=True, return_bool=True)) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Config section for tool 'spades' is blank!") ) @log_capture() def test_return_rule_for_blank_tool(self, l): - self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest137, test=True), yt.iv137dict) + self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest137, job_conf_path=job_conf_path, test=True), yt.iv137dict) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Config section for tool 'spades' is blank!"), @@ -522,7 +533,7 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_return_bool_for_malformed_users(self, l): - self.assertFalse(dt.parse_yaml(path=yt.ivYMLTest138, test=True, return_bool=True)) + self.assertFalse(dt.parse_yaml(path=yt.ivYMLTest138, job_conf_path=job_conf_path, test=True, return_bool=True)) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Entry '123' in users for rule 1 in tool 'spades' is in an invalid format!"), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Supplied email 'invaliduser.email@com' for rule 1 in tool 'spades' is in an invalid format!") @@ -530,7 +541,7 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_return_rule_for_malformed_users(self, l): - self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest138, test=True), yt.iv138dict) + self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest138, job_conf_path=job_conf_path, test=True), yt.iv138dict) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Entry '123' in users for rule 1 in tool 'spades' is in an invalid format! Ignoring entry."), @@ -540,14 +551,14 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_return_bool_for_no_users(self, l): - self.assertFalse(dt.parse_yaml(path=yt.ivYMLTest139, test=True, return_bool=True)) + self.assertFalse(dt.parse_yaml(path=yt.ivYMLTest139, job_conf_path=job_conf_path, test=True, return_bool=True)) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Couldn't find a list under 'users:'!") ) @log_capture() def test_return_rule_for_no_users(self, l): - self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest139, test=True), yt.iv139dict) + self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest139, job_conf_path=job_conf_path, test=True), yt.iv139dict) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Couldn't find a list under 'users:'! Ignoring rule."), @@ -556,7 +567,7 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_return_bool_for_malformed_user_email(self, l): - self.assertFalse(dt.parse_yaml(path=yt.ivYMLTest140, test=True, return_bool=True)) + self.assertFalse(dt.parse_yaml(path=yt.ivYMLTest140, job_conf_path=job_conf_path, test=True, return_bool=True)) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Supplied email 'invalid.user2@com' for rule 2 in tool 'spades' is in an invalid format!"), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Supplied email 'invalid.user1@com' for rule 2 in tool 'spades' is in an invalid format!"), @@ -565,7 +576,7 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_return_rule_for_malformed_user_email(self, l): - self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest140, test=True), yt.iv140dict) + self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest140, job_conf_path=job_conf_path, test=True), yt.iv140dict) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Supplied email 'invalid.user2@com' for rule 2 in tool 'spades' is in an invalid format! Ignoring email."), @@ -576,7 +587,7 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_return_bool_for_empty_users(self, l): - self.assertFalse(dt.parse_yaml(path=yt.ivYMLTest141, test=True, return_bool=True)) + self.assertFalse(dt.parse_yaml(path=yt.ivYMLTest141, job_conf_path=job_conf_path, test=True, return_bool=True)) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Entry 'None' in users for rule 2 in tool 'spades' is in an invalid format!"), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Entry 'None' in users for rule 2 in tool 'spades' is in an invalid format!"), @@ -585,7 +596,7 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_return_rule_for_empty_users(self, l): - self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest141, test=True), yt.iv141dict) + self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest141, job_conf_path=job_conf_path, test=True), yt.iv141dict) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Entry 'None' in users for rule 2 in tool 'spades' is in an invalid format! Ignoring entry."), @@ -596,7 +607,7 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_return_bool_for_bad_num_input_datasets_bounds(self, l): - self.assertFalse(dt.parse_yaml(path=yt.ivYMLTest142, test=True, return_bool=True)) + self.assertFalse(dt.parse_yaml(path=yt.ivYMLTest142, job_conf_path=job_conf_path, test=True, return_bool=True)) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Error: lower_bound is set to Infinity, but must be lower than upper_bound!"), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "lower_bound exceeds upper_bound for rule 1 in 'smalt'.") @@ -604,7 +615,7 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_return_rule_for_bad_num_input_datasets_bound(self, l): - self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest142, test=True), yt.iv142dict) + self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest142, job_conf_path=job_conf_path, test=True), yt.iv142dict) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Error: lower_bound is set to Infinity, but must be lower than upper_bound! Setting lower_bound to 0!"), @@ -613,14 +624,14 @@ class TestDynamicToolDestination(unittest.TestCase): @log_capture() def test_return_bool_for_worse_num_input_datasets_bounds(self, l): - self.assertFalse(dt.parse_yaml(path=yt.ivYMLTest143, test=True, return_bool=True)) + self.assertFalse(dt.parse_yaml(path=yt.ivYMLTest143, job_conf_path=job_conf_path, test=True, return_bool=True)) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Error: lower_bound is set to Infinity, but must be lower than upper_bound!") ) @log_capture() def test_return_rule_for_worse_num_input_datasets_bound(self, l): - self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest143, test=True), yt.iv143dict) + self.assertEquals(dt.parse_yaml(path=yt.ivYMLTest143, job_conf_path=job_conf_path, test=True), yt.iv143dict) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Error: lower_bound is set to Infinity, but must be lower than upper_bound! Setting lower_bound to 0!"), @@ -628,64 +639,267 @@ class TestDynamicToolDestination(unittest.TestCase): ) @log_capture() - def test_priority_default_destination_without_med_priority_destination(self, l): - dt.parse_yaml(path=yt.ivYMLTest144, test=True) + def test_tool_without_low_default_destination(self, l): + dt.parse_yaml(path=yt.ivYMLTest146, job_conf_path=job_conf_path, test=True) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), - ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No default 'med' priority destination!"), - ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') - ) - - @log_capture() - def test_priority_default_destination_with_invalid_priority_destination(self, l): - dt.parse_yaml(path=yt.ivYMLTest145, test=True) - l.check( - ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), - ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Invalid default priority destination 'mine' found in config!"), - ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') - ) - - @log_capture() - def test_tool_without_med_priority_destination(self, l): - dt.parse_yaml(path=yt.ivYMLTest146, test=True) - l.check( - ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), - ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No 'med' priority destination for rule 1 in 'smalt'. Ignoring..."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No default_priority section found in config. Setting 'med' as default priority."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Invalid priority 'low' for rule 1 in 'smalt'. Ignoring..."), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') ) @log_capture() def test_tool_with_invalid_priority_destination(self, l): - dt.parse_yaml(path=yt.ivYMLTest147, test=True) + dt.parse_yaml(path=yt.ivYMLTest147, job_conf_path=job_conf_path, test=True) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), - ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Invalid priority destination 'mine' for rule 1 in 'smalt'. Ignoring..."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No default_priority section found in config. Setting 'med' as default priority."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Invalid priority 'mine' for rule 1 in 'smalt'. Ignoring..."), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') ) @log_capture() - def test_users_with_invalid_priority(self, l): - dt.parse_yaml(path=yt.ivYMLTest148, test=True) + def test_not_all_priorities_in_tool(self, l): + dt.parse_yaml(path=yt.ivYMLTest149, job_conf_path=job_conf_path, test=True) l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), - ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "User 'user@email.com', priority is not valid! Must be either low, med, or high."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No default_priority section found in config. Setting 'med' as default priority."), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') ) + @log_capture() + def test_rule_destination_not_in_job_conf(self, l): + dt.parse_yaml(path=yt.ivYMLTest150, job_conf_path=job_conf_path, test=True) + l.check( + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No default_priority section found in config. Setting 'med' as default priority."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Destination for 'blegh', rule 1: 'fake_destination' does not exist in job configuration. Ignoring..."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') + ) + + @log_capture() + def test_tool_default_destination_not_in_job_conf_with_no_rules(self, l): + dt.parse_yaml(path=yt.ivYMLTest151, job_conf_path=job_conf_path, test=True) + l.check( + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No default_priority section found in config. Setting 'med' as default priority."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Default destination for 'blah': 'not_true_destination' does not appear in the job configuration."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Tool 'blah' does not have rules nor a default_destination!"), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') + ) + + @log_capture() + def test_default_destination_not_in_job_conf(self, l): + dt.parse_yaml(path=yt.ivYMLTest152, job_conf_path=job_conf_path, test=True) + l.check( + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Default destination 'no_such_dest' does not appear in the job configuration."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No default_priority section found in config. Setting 'med' as default priority."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') + ) + + @log_capture() + def test_rule_destination_without_priority_not_in_job_conf(self, l): + dt.parse_yaml(path=yt.ivYMLTest153, job_conf_path=job_conf_path, test=True) + l.check( + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No default_priority section found in config. Setting 'med' as default priority."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Destination for 'blegh', rule 1: 'fake_destination' does not exist in job configuration. Ignoring..."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') + ) + + @log_capture() + def test_tool_default_destination_without_priority_not_in_job_conf_with_no_rules(self, l): + dt.parse_yaml(path=yt.ivYMLTest154, job_conf_path=job_conf_path, test=True) + l.check( + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No default_priority section found in config. Setting 'med' as default priority."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Default destination for 'blah': 'not_true_destination' does not appear in the job configuration."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Tool 'blah' does not have rules nor a default_destination!"), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') + ) + + @log_capture() + def test_default_destination_without_priority_not_in_job_conf(self, l): + dt.parse_yaml(path=yt.ivYMLTest155, job_conf_path=job_conf_path, test=True) + l.check( + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Default destination 'no_such_dest' does not appear in the job configuration."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') + ) + + @log_capture() + def test_tool_rule_priority_does_not_exist(self, l): + dt.parse_yaml(path=yt.ivYMLTest156, job_conf_path=job_conf_path, test=True) + l.check( + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No default_priority section found in config. Setting 'med' as default priority."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Invalid priority 'notAPriority' for rule 1 in 'aTool'. Ignoring..."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') + ) + + @log_capture() + def test_tool_default_destination_priority_does_not_exist(self, l): + dt.parse_yaml(path=yt.ivYMLTest157, job_conf_path=job_conf_path, test=True) + l.check( + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No default_priority section found in config. Setting 'med' as default priority."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Invalid default destination priority 'notAPriority' for 'aTool'."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') + ) + + @log_capture() + def test_tool_default_destination_not_in_job_conf(self, l): + dt.parse_yaml(path=yt.ivYMLTest158, job_conf_path=job_conf_path, test=True) + l.check( + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No default_priority section found in config. Setting 'med' as default priority."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Default destination for 'blah': 'not_true_destination' does not appear in the job configuration."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') + ) + + @log_capture() + def test_tool_default_destination_without_priority_not_in_job_conf(self, l): + dt.parse_yaml(path=yt.ivYMLTest159, job_conf_path=job_conf_path, test=True) + l.check( + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Default destination for 'blah': 'not_true_destination' does not appear in the job configuration."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') + ) + + @log_capture() + def test_empty_priority_dict(self, l): + dt.parse_yaml(path=yt.ivYMLTest163, job_conf_path=job_conf_path, test=True) + l.check( + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No global default destinations specified in config!"), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') + ) + + @log_capture() + def test_default_dest_is_string_but_priorities_used_in_rule(self, l): + dt.parse_yaml(path=yt.ivYMLTest161, job_conf_path=job_conf_path, test=True) + l.check( + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Invalid priority 'med' for rule 1 in 'blah'. Ignoring..."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') + ) + + @log_capture() + def test_default_dest_is_string_but_priorities_used_in_tool_default_dest(self, l): + dt.parse_yaml(path=yt.ivYMLTest162, job_conf_path=job_conf_path, test=True) + l.check( + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Invalid default destination priority 'med' for 'blah'."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') + ) + + @log_capture() + def test_typo_in_str_default_dest(self, l): + dt.parse_yaml(path=yt.ivYMLTest164, job_conf_path=job_conf_path, test=True) + l.check( + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Default destination 'cluster-kow' does not appear in the job configuration. Did you mean 'cluster_low'?"), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') + ) + + @log_capture() + def test_typo_in_dict_default_dest(self, l): + dt.parse_yaml(path=yt.ivYMLTest165, job_conf_path=job_conf_path, test=True) + l.check( + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Default destination 'cluster_kow' does not appear in the job configuration. Did you mean 'cluster_low'?"), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') + ) + + @log_capture() + def test_typo_in_dict_tool_default_dest(self, l): + dt.parse_yaml(path=yt.ivYMLTest166, job_conf_path=job_conf_path, test=True) + l.check( + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Default destination for 'blah': 'cluster_defaut' does not appear in the job configuration. Did you mean 'cluster_default'?"), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') + ) + + @log_capture() + def test_typo_in_str_tool_default_dest(self, l): + dt.parse_yaml(path=yt.ivYMLTest167, job_conf_path=job_conf_path, test=True) + l.check( + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Default destination for 'blah': 'Destination_3_med' does not appear in the job configuration. Did you mean 'Destination3_med'?"), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') + ) + + @log_capture() + def test_typo_in_str_tool_rule_dest(self, l): + dt.parse_yaml(path=yt.ivYMLTest168, job_conf_path=job_conf_path, test=True) + l.check( + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Destination for 'blah', rule 1: 'thig' does not exist in job configuration. Did you mean 'things'? Ignoring..."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') + ) + + @log_capture() + def test_typo_in_dict_tool_rule_dest(self, l): + dt.parse_yaml(path=yt.ivYMLTest169, job_conf_path=job_conf_path, test=True) + l.check( + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Destination for 'blah', rule 1: 'even_lamerr_cluster' does not exist in job configuration. Did you mean 'even_lamer_cluster'? Ignoring..."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') + ) + + @log_capture() + def test_typo_in_case(self, l): + dt.parse_yaml(path=yt.ivYMLTest170, job_conf_path=job_conf_path, test=True) + l.check( + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Default destination 'destinationf' does not appear in the job configuration. Did you mean 'DestinationF'?"), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') + ) + + @log_capture() + def test_invalid_verbose_value(self, l): + dt.parse_yaml(path=yt.ivYMLTest171, job_conf_path=job_conf_path, test=True) + l.check( + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Verbose value 'notavalue' is not True or False! Falling back to verbose..."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') + ) + + @log_capture() + def test_invalid_default_dest_valid_tool_default_dest_bool(self, l): + self.assertFalse(dt.parse_yaml(path=yt.ivYMLTest172, job_conf_path=job_conf_path, test=True, return_bool=True)) + l.check( + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Default destination 'fake_destination' does not appear in the job configuration."), + ) + + @log_capture() + def test_valid_default_dest_invalid_tool_default_dest_bool(self, l): + self.assertFalse(dt.parse_yaml(path=yt.ivYMLTest173, job_conf_path=job_conf_path, test=True, return_bool=True)) + l.check( + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Default destination for 'blah': 'fake_destination' does not appear in the job configuration."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "Tool 'blah' does not have rules nor a default_destination!"), + ) + # ================================Valid yaml files============================== @log_capture() def test_parse_valid_yml(self, l): - self.assertEqual(dt.parse_yaml(yt.vYMLTest1, test=True), yt.vdictTest1_yml) - self.assertEqual(dt.parse_yaml(yt.vYMLTest2, test=True), yt.vdictTest2_yml) - self.assertEqual(dt.parse_yaml(yt.vYMLTest3, test=True), yt.vdictTest3_yml) - self.assertTrue(dt.parse_yaml(yt.vYMLTest4, test=True, return_bool=True)) - self.assertEqual(dt.parse_yaml(yt.vYMLTest4, test=True), yt.vdictTest4_yml) - self.assertTrue(dt.parse_yaml(yt.vYMLTest5, test=True, return_bool=True)) - self.assertEqual(dt.parse_yaml(yt.vYMLTest5, test=True), yt.vdictTest5_yml) - self.assertTrue(dt.parse_yaml(yt.vYMLTest6, test=True, return_bool=True)) - self.assertEqual(dt.parse_yaml(yt.vYMLTest6, test=True), yt.vdictTest6_yml) - self.assertTrue(dt.parse_yaml(yt.vYMLTest7, test=True, return_bool=True)) - self.assertEqual(dt.parse_yaml(yt.vYMLTest7, test=True), yt.vdictTest7_yml) + self.assertEqual(dt.parse_yaml(yt.vYMLTest1, job_conf_path=job_conf_path, test=True), yt.vdictTest1_yml) + self.assertEqual(dt.parse_yaml(yt.vYMLTest2, job_conf_path=job_conf_path, test=True), yt.vdictTest2_yml) + self.assertEqual(dt.parse_yaml(yt.vYMLTest3, job_conf_path=job_conf_path, test=True), yt.vdictTest3_yml) + self.assertTrue(dt.parse_yaml(yt.vYMLTest4, job_conf_path=job_conf_path, test=True, return_bool=True)) + self.assertEqual(dt.parse_yaml(yt.vYMLTest4, job_conf_path=job_conf_path, test=True), yt.vdictTest4_yml) + self.assertTrue(dt.parse_yaml(yt.vYMLTest5, job_conf_path=job_conf_path, test=True, return_bool=True)) + self.assertEqual(dt.parse_yaml(yt.vYMLTest5, job_conf_path=job_conf_path, test=True), yt.vdictTest5_yml) + self.assertTrue(dt.parse_yaml(yt.vYMLTest6, job_conf_path=job_conf_path, test=True, return_bool=True)) + self.assertEqual(dt.parse_yaml(yt.vYMLTest6, job_conf_path=job_conf_path, test=True), yt.vdictTest6_yml) + self.assertTrue(dt.parse_yaml(yt.vYMLTest7, job_conf_path=job_conf_path, test=True, return_bool=True)) + self.assertEqual(dt.parse_yaml(yt.vYMLTest7, job_conf_path=job_conf_path, test=True), yt.vdictTest7_yml) + self.assertTrue(dt.parse_yaml(yt.vYMLTest160, job_conf_path=job_conf_path, test=True, return_bool=True)) + self.assertEqual(dt.parse_yaml(yt.vYMLTest160, job_conf_path=job_conf_path, test=True), yt.vdictTest160_yml) + self.assertTrue(dt.parse_yaml(yt.vYMLTest164, job_conf_path=job_conf_path, test=True, return_bool=True)) + self.assertEqual(dt.parse_yaml(yt.vYMLTest164, job_conf_path=job_conf_path, test=True), yt.vdictTest164_yml) + l.check( ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.'), @@ -699,8 +913,14 @@ class TestDynamicToolDestination(unittest.TestCase): ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No default_priority section found in config. Setting 'med' as default priority."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', "No default_priority section found in config. Setting 'med' as default priority."), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Running config validation...'), + ('galaxy.jobs.dynamic_tool_destination', 'DEBUG', 'Finished config validation.') ) # ================================Testing str_to_bytes========================== diff --git a/test/unit/jobs/dynamic_tool_destination/ymltests.py b/test/unit/jobs/dynamic_tool_destination/ymltests.py index b5b325c9aa2..0c73b3bb569 100644 --- a/test/unit/jobs/dynamic_tool_destination/ymltests.py +++ b/test/unit/jobs/dynamic_tool_destination/ymltests.py @@ -9,7 +9,7 @@ vYMLTest1 = """ lower_bound: 0 upper_bound: 100000000 destination: things - default_destination: waffles_default + default_destination: cluster_default verbose: True """ @@ -27,14 +27,14 @@ vdictTest1_yml = { ] } }, - 'default_destination': "waffles_default" + 'default_destination': "cluster_default" } # Multiple jobs, multiple rules vYMLTest2 = ''' tools: spades: - default_destination: waffles_default + default_destination: cluster_default smalt: rules: - rule_type: file_size @@ -49,14 +49,14 @@ vYMLTest2 = ''' upper_bound: Infinity fail_message: Too few reads for smalt to work destination: fail - default_destination: waffles_low + default_destination: cluster_low verbose: True ''' vdictTest2_yml = { "tools": { "spades": { - "default_destination": "waffles_default" + "default_destination": "cluster_default" }, "smalt": { "rules": [ @@ -78,7 +78,7 @@ vdictTest2_yml = { ] } }, - 'default_destination': "waffles_low" + 'default_destination': "cluster_low" } # Rule with extra attribute @@ -93,7 +93,7 @@ vYMLTest3 = ''' upper_bound: 100000000 fail_message: Whats hax destination: fail - default_destination: waffles_default + default_destination: cluster_default verbose: True ''' @@ -113,7 +113,7 @@ vdictTest3_yml = { ] } }, - 'default_destination': "waffles_default" + 'default_destination': "cluster_default" } # Arguments type @@ -127,7 +127,7 @@ vYMLTest4 = """ careful: true fail_message: Failure destination: fail - default_destination: waffles_default + default_destination: cluster_default verbose: True """ @@ -147,7 +147,7 @@ vdictTest4_yml = { ] } }, - 'default_destination': "waffles_default" + 'default_destination': "cluster_default" } # Records type @@ -159,8 +159,8 @@ vYMLTest5 = ''' nice_value: 0 lower_bound: 0 upper_bound: 100000000 - destination: waffles_low_4 - default_destination: waffles_default + destination: cluster_low_4 + default_destination: cluster_default verbose: True ''' @@ -173,19 +173,19 @@ vdictTest5_yml = { 'nice_value': 0, "lower_bound": 0, "upper_bound": 100000000, - "destination": "waffles_low_4" + "destination": "cluster_low_4" } ] } }, - 'default_destination': "waffles_default" + 'default_destination': "cluster_default" } # Num_input_datasets type vYMLTest6 = ''' tools: spades: - default_destination: waffles_default + default_destination: cluster_default smalt: rules: - rule_type: num_input_datasets @@ -198,14 +198,14 @@ vYMLTest6 = ''' lower_bound: 200 upper_bound: Infinity destination: cluster_high_32 - default_destination: waffles_low + default_destination: cluster_low verbose: True ''' vdictTest6_yml = { "tools": { "spades": { - "default_destination": "waffles_default" + "default_destination": "cluster_default" }, "smalt": { "rules": [ @@ -225,7 +225,7 @@ vdictTest6_yml = { ] } }, - 'default_destination': "waffles_low" + 'default_destination': "cluster_low" } # One job, one rule, and priority destinations @@ -242,7 +242,7 @@ vYMLTest7 = """ med: things default_destination: priority: - med: waffles_default + med: cluster_default users: user@example.com: priority: med @@ -269,15 +269,108 @@ vdictTest7_yml = { }, 'default_destination': { 'priority': { - 'med': 'waffles_default' + 'med': 'cluster_default' } }, + 'default_priority': 'med', 'users': { 'user@example.com': { 'priority': 'med' } } } + +# No valid priorities but the tool doesn't require one +vYMLTest160 = ''' + default_destination: cluster_low + default_priority: med + tools: + blah: + rules: + - rule_type: num_input_datasets + nice_value: 0 + lower_bound: 0 + upper_bound: Infinity + destination: cluster_default + default_destination: cluster_high + verbose: True +''' + +vdictTest160_yml = { + "tools": { + "blah": { + "rules": [ + { + "rule_type": "num_input_datasets", + "nice_value": 0, + "lower_bound": 0, + "upper_bound": "Infinity", + "destination": "cluster_default" + } + ], + "default_destination": "cluster_high" + } + }, + 'default_destination': 'cluster_low', +} + +# No valid priorities but the tool doesn't require one +vYMLTest164 = ''' + default_destination: + priority: + good: Destination1_med + fast: Destination1_high + default_priority: good + tools: + blah: + rules: + - rule_type: num_input_datasets + nice_value: 0 + lower_bound: 0 + upper_bound: Infinity + destination: + priority: + fast: lame_cluster + default_destination: + priority: + good: cluster_med_4 + fast: cluster_high + verbose: True +''' + +vdictTest164_yml = { + "tools": { + "blah": { + "rules": [ + { + "rule_type": "num_input_datasets", + "nice_value": 0, + "lower_bound": 0, + "upper_bound": "Infinity", + "destination": { + "priority": { + "fast": "lame_cluster" + } + } + } + ], + "default_destination": { + "priority": { + "good": "cluster_med_4", + "fast": "cluster_high" + } + } + } + }, + 'default_destination': { + "priority": { + "good": "Destination1_med", + "fast": "Destination1_high" + } + }, + 'default_priority': 'good' +} + # =====================================================Invalid XML tests========================================================== # Empty file @@ -292,12 +385,12 @@ ivYMLTest3 = ''' upper_bound: 100 lower_bound: 0 destination: fail - default_destination: waffles_default + default_destination: cluster_default verbose: True ''' iv3dict = { - 'default_destination': "waffles_default" + 'default_destination': "cluster_default" } # Rule missing type @@ -310,7 +403,7 @@ ivYMLTest4 = ''' upper_bound: 0 fail_message: No type... destination: fail - default_destination: waffles_default + default_destination: cluster_default verbose: True ''' @@ -324,7 +417,7 @@ ivYMLTest51 = ''' upper_bound: 0 fail_message: No type... destination: fail - default_destination: waffles_default + default_destination: cluster_default verbose: True ''' @@ -338,7 +431,7 @@ ivYMLTest52 = ''' lower_bound: 0 fail_message: No type... destination: fail - default_destination: waffles_default + default_destination: cluster_default verbose: True ''' @@ -352,12 +445,12 @@ ivYMLTest53 = ''' lower_bound: 0 upper_bound: 0 fail_message: No type... - default_destination: waffles_default + default_destination: cluster_default verbose: True ''' ivDict53 = { - 'default_destination': 'waffles_default', + 'default_destination': 'cluster_default', 'tools': { 'spades': { 'rules': [ @@ -387,7 +480,7 @@ ivYMLTest6 = ''' upper_bound: 0 fail_message: No type... destination: fail - default_destination: waffles_default + default_destination: cluster_default verbose: True ''' @@ -398,7 +491,7 @@ ivYMLTest7 = ''' ''' ivDict = { - 'default_destination': "waffles_default" + 'default_destination': "cluster_default" } # Invalid category @@ -416,7 +509,7 @@ ivYMLTest91 = ''' lower_bound: 0 upper_bound: 0 destination: fail - default_destination: waffles_default + default_destination: cluster_default verbose: True ''' @@ -435,7 +528,7 @@ iv91dict = { ] } }, - 'default_destination': "waffles_default" + 'default_destination': "cluster_default" } # Tool default fail no destination @@ -447,9 +540,9 @@ ivYMLTest11 = ''' nice_value: -21 lower_bound: 1 KB upper_bound: Infinity - destination: waffles_low - default_destination: waffles_low - default_destination: waffles_default + destination: cluster_low + default_destination: cluster_low + default_destination: cluster_default verbose: True ''' @@ -463,7 +556,7 @@ ivYMLTest12 = """ arguments: careful: true destination: fail - default_destination: waffles_default + default_destination: cluster_default verbose: True """ @@ -483,7 +576,7 @@ iv12dict = { ] } }, - 'default_destination': "waffles_default" + 'default_destination': "cluster_default" } # Arguments fail no arguments @@ -495,12 +588,12 @@ ivYMLTest131 = """ nice_value: 0 fail_message: Something went wrong destination: fail - default_destination: waffles_default + default_destination: cluster_default verbose: True """ iv131dict = { - 'default_destination': "waffles_default" + 'default_destination': "cluster_default" } # Arguments fail no destination @@ -513,12 +606,12 @@ ivYMLTest132 = """ fail_message: Something went wrong arguments: careful: true - default_destination: waffles_default + default_destination: cluster_default verbose: True """ iv132dict = { - 'default_destination': 'waffles_default', + 'default_destination': 'cluster_default', 'tools': { 'spades': { 'rules': [ @@ -550,8 +643,8 @@ ivYMLTest133 = ''' nice_value: 0 lower_bound: 100000000 upper_bound: Infinity - destination: waffles_low_4 - default_destination: waffles_low + destination: cluster_low_4 + default_destination: cluster_low verbose: True ''' @@ -571,12 +664,12 @@ iv133dict = { 'nice_value': 0, "lower_bound": 100000000, "upper_bound": "Infinity", - "destination": "waffles_low_4" + "destination": "cluster_low_4" } ] } }, - 'default_destination': "waffles_low" + 'default_destination': "cluster_low" } # No destination and no fail_message @@ -588,12 +681,12 @@ ivYMLTest134 = """ upper_bound: 10000 lower_bound: 0 nice_value: 0 - default_destination: waffles_default + default_destination: cluster_default verbose: True """ iv134dict = { - 'default_destination': 'waffles_default', + 'default_destination': 'cluster_default', 'tools': { 'spades': { 'rules': [ @@ -617,13 +710,13 @@ ivYMLTest135 = """ upper_bound: 100 lower_bound: 200 nice_value: 0 - destination: waffles_low_4 - default_destination: waffles_default + destination: cluster_low_4 + default_destination: cluster_default verbose: True """ iv135dict = { - 'default_destination': 'waffles_default', + 'default_destination': 'cluster_default', 'tools': { 'spades': { 'rules': [ @@ -632,7 +725,7 @@ iv135dict = { 'upper_bound': 200, 'lower_bound': 100, 'nice_value': 0, - 'destination': 'waffles_low_4' + 'destination': 'cluster_low_4' } ] } @@ -645,12 +738,12 @@ ivYMLTest136 = """ spades: rules: - default_destination: waffles_default + default_destination: cluster_default verbose: True """ iv136dict = { - 'default_destination': 'waffles_default' + 'default_destination': 'cluster_default' } # Tool is blank; no tool-specific default destination, no rules category @@ -658,12 +751,12 @@ ivYMLTest137 = """ tools: spades: - default_destination: waffles_default + default_destination: cluster_default verbose: True """ iv137dict = { - 'default_destination': 'waffles_default' + 'default_destination': 'cluster_default' } # Tool specifies authorized users with an invalid entry @@ -675,17 +768,17 @@ ivYMLTest138 = """ upper_bound: 200 lower_bound: 100 nice_value: 0 - destination: waffles_low_4 + destination: cluster_low_4 users: - validuser@email.com - invaliduser.email@com - 123 - default_destination: waffles_default + default_destination: cluster_default verbose: True """ iv138dict = { - 'default_destination': 'waffles_default', + 'default_destination': 'cluster_default', 'tools': { 'spades': { 'rules': [ @@ -694,7 +787,7 @@ iv138dict = { 'upper_bound': 200, 'lower_bound': 100, 'nice_value': 0, - 'destination': 'waffles_low_4', + 'destination': 'cluster_low_4', 'users': [ 'validuser@email.com' ] @@ -713,19 +806,19 @@ ivYMLTest139 = """ upper_bound: 600 lower_bound: 200 nice_value: 0 - destination: waffles_high + destination: cluster_high - rule_type: file_size upper_bound: 199 lower_bound: 100 nice_value: 0 - destination: waffles_low_4 + destination: cluster_low_4 users: - default_destination: waffles_default + default_destination: cluster_default verbose: True """ iv139dict = { - 'default_destination': 'waffles_default', + 'default_destination': 'cluster_default', 'tools': { 'spades': { 'rules': [ @@ -734,7 +827,7 @@ iv139dict = { 'upper_bound': 600, 'lower_bound': 200, 'nice_value': 0, - 'destination': 'waffles_high' + 'destination': 'cluster_high' } ] } @@ -750,21 +843,21 @@ ivYMLTest140 = """ upper_bound: 600 lower_bound: 200 nice_value: 0 - destination: waffles_high + destination: cluster_high - rule_type: file_size upper_bound: 199 lower_bound: 100 nice_value: 0 - destination: waffles_low_4 + destination: cluster_low_4 users: - invalid.user1@com - invalid.user2@com - default_destination: waffles_default + default_destination: cluster_default verbose: True """ iv140dict = { - 'default_destination': 'waffles_default', + 'default_destination': 'cluster_default', 'tools': { 'spades': { 'rules': [ @@ -773,7 +866,7 @@ iv140dict = { 'upper_bound': 600, 'lower_bound': 200, 'nice_value': 0, - 'destination': 'waffles_high' + 'destination': 'cluster_high' } ] } @@ -789,21 +882,21 @@ ivYMLTest141 = """ upper_bound: 600 lower_bound: 200 nice_value: 0 - destination: waffles_high + destination: cluster_high - rule_type: file_size upper_bound: 199 lower_bound: 100 nice_value: 0 - destination: waffles_low_4 + destination: cluster_low_4 users: - - - default_destination: waffles_default + default_destination: cluster_default verbose: True """ iv141dict = { - 'default_destination': 'waffles_default', + 'default_destination': 'cluster_default', 'tools': { 'spades': { 'rules': [ @@ -812,7 +905,7 @@ iv141dict = { 'upper_bound': 600, 'lower_bound': 200, 'nice_value': 0, - 'destination': 'waffles_high' + 'destination': 'cluster_high' } ] } @@ -829,12 +922,12 @@ ivYMLTest142 = ''' lower_bound: Infinity upper_bound: 200 destination: cluster_low_4 - default_destination: waffles_low + default_destination: cluster_low verbose: True ''' iv142dict = { - 'default_destination': 'waffles_low', + 'default_destination': 'cluster_low', 'tools': { 'smalt': { 'rules': [ @@ -860,12 +953,12 @@ ivYMLTest143 = ''' lower_bound: Infinity upper_bound: Infinity destination: cluster_low_4 - default_destination: waffles_low + default_destination: cluster_low verbose: True ''' iv143dict = { - 'default_destination': 'waffles_low', + 'default_destination': 'cluster_low', 'tools': { 'smalt': { 'rules': [ @@ -885,7 +978,7 @@ iv143dict = { ivYMLTest144 = ''' default_destination: priority: - low: waffles_low + low: cluster_low verbose: True ''' @@ -893,8 +986,8 @@ ivYMLTest144 = ''' ivYMLTest145 = ''' default_destination: priority: - med: waffles_low - mine: waffles_low + med: cluster_low + mine: cluster_low verbose: True ''' @@ -912,7 +1005,7 @@ ivYMLTest146 = ''' low: cluster_low_4 default_destination: priority: - med: waffles_low + med: cluster_low verbose: True ''' @@ -931,7 +1024,7 @@ ivYMLTest147 = ''' mine: cluster_low_4 default_destination: priority: - med: waffles_low + med: cluster_low verbose: True ''' @@ -939,9 +1032,335 @@ ivYMLTest147 = ''' ivYMLTest148 = ''' default_destination: priority: - med: waffles_low + med: cluster_low users: user@email.com: priority: mine verbose: True ''' + +# not all priorities in tool destinations +ivYMLTest149 = ''' + default_destination: + priority: + med: cluster_low + lowish: cluster_low + high: cluster_default + higher: cluster_high + tools: + yuck: + default_destination: + priority: + med: cluster_default + high: cluster_high + verbose: True +''' + +# rule destination not in job config +ivYMLTest150 = ''' + default_destination: + priority: + med: cluster_low + tools: + blegh: + rules: + - rule_type: num_input_datasets + nice_value: 0 + lower_bound: 0 + upper_bound: Infinity + destination: + priority: + med: fake_destination + verbose: True +''' + +# tool default destination not in job config and no rules +ivYMLTest151 = ''' + default_destination: + priority: + med: cluster_low + tools: + blah: + default_destination: + priority: + med: not_true_destination + verbose: True +''' + +# default destination not in job config +ivYMLTest152 = ''' + default_destination: + priority: + med: no_such_dest + verbose: True +''' + +# rule destination not in job config (without priority dict) +ivYMLTest153 = ''' + default_destination: + priority: + med: cluster_low + tools: + blegh: + rules: + - rule_type: num_input_datasets + nice_value: 0 + lower_bound: 0 + upper_bound: Infinity + destination: fake_destination + verbose: True +''' + +# tool default destination not in job config (without priority dict) and no rules +ivYMLTest154 = ''' + default_destination: + priority: + med: cluster_low + tools: + blah: + default_destination: not_true_destination + verbose: True +''' + +# default destination not in job config (without priority dict) +ivYMLTest155 = ''' + default_destination: no_such_dest + verbose: True +''' + +# tool rule destination priority doesn't exist +ivYMLTest156 = ''' + default_destination: + priority: + med: cluster_default + tools: + aTool: + default_destination: + priority: + med: cluster_low + rules: + - rule_type: num_input_datasets + nice_value: 0 + lower_bound: 0 + upper_bound: Infinity + destination: + priority: + notAPriority: cluster_default + verbose: True +''' + +# tool default destination priority doesn't exist +ivYMLTest157 = ''' + default_destination: + priority: + med: cluster_default + tools: + aTool: + default_destination: + priority: + notAPriority: cluster_low + med: cluster_low + verbose: True +''' + +# tool default destination not in job config +ivYMLTest158 = ''' + default_destination: + priority: + med: cluster_low + tools: + blah: + rules: + - rule_type: num_input_datasets + nice_value: 0 + lower_bound: 0 + upper_bound: Infinity + destination: + priority: + med: cluster_default + default_destination: + priority: + med: not_true_destination + verbose: True +''' + +# tool default destination not in job config (without priority dict) +ivYMLTest159 = ''' + default_destination: cluster_low + tools: + blah: + rules: + - rule_type: num_input_datasets + nice_value: 0 + lower_bound: 0 + upper_bound: Infinity + destination: cluster_default + default_destination: not_true_destination + verbose: True +''' + +# No valid priorities and the tool rule requires them +ivYMLTest161 = ''' + default_destination: cluster_low + tools: + blah: + rules: + - rule_type: num_input_datasets + nice_value: 0 + lower_bound: 0 + upper_bound: Infinity + destination: + priority: + med: cluster_default + default_destination: cluster_high + verbose: True +''' + +# No valid priorities and the tool default_destination requires them +ivYMLTest162 = ''' + default_destination: cluster_low + tools: + blah: + rules: + - rule_type: num_input_datasets + nice_value: 0 + lower_bound: 0 + upper_bound: Infinity + destination: cluster_default + default_destination: + priority: + med: cluster_default + verbose: True +''' +# Nothing in the priority dict +ivYMLTest163 = ''' + default_destination: + priority: + verbose: True +''' + +# Typo in str default destination +ivYMLTest164 = ''' + default_destination: cluster-kow + verbose: True +''' + +# Typo in dict default destination +ivYMLTest165 = ''' + default_destination: + priority: + pr: cluster_kow + default_priority: pr + verbose: True +''' + +# Typo in dict tool default destination +ivYMLTest166 = ''' + default_destination: + priority: + med: cluster_low + default_priority: med + tools: + blah: + default_destination: + priority: + med: cluster_defaut + rules: + - rule_type: num_input_datasets + nice_value: 0 + lower_bound: 0 + upper_bound: Infinity + destination: DestinationF + verbose: True +''' + +# Typo in str tool default destination +ivYMLTest167 = ''' + default_destination: cluster_low + default_priority: cluster_low + tools: + blah: + default_destination: Destination_3_med + rules: + - rule_type: num_input_datasets + nice_value: 0 + lower_bound: 0 + upper_bound: Infinity + destination: DestinationF + verbose: True +''' + +# Typo in dict tool rule destination +ivYMLTest168 = ''' + default_destination: + priority: + med: cluster_default + default_priority: med + tools: + blah: + rules: + - rule_type: num_input_datasets + nice_value: 0 + lower_bound: 0 + upper_bound: Infinity + destination: + priority: + med: thig + verbose: True +''' + +# Typo in str tool rule destination +ivYMLTest169 = ''' + default_destination: + priority: + med: cluster_default + default_priority: med + tools: + blah: + rules: + - rule_type: num_input_datasets + nice_value: 0 + lower_bound: 0 + upper_bound: Infinity + destination: even_lamerr_cluster + default_destination: + priority: + med: cluster_default + verbose: True +''' + +# Typo in str tool rule destination +ivYMLTest170 = ''' + default_destination: + priority: + med: destinationf + default_priority: med + verbose: True +''' + +# Invalid verbose setting +ivYMLTest171 = ''' + default_destination: + priority: + med: DestinationF + default_priority: med + verbose: notavalue +''' + +# invalid default destination and valid tool default destination +ivYMLTest172 = ''' + default_destination: fake_destination + tools: + blah: + default_destination: cluster_default + verbose: True +''' + +# valid default destination and invalid tool default destination +ivYMLTest173 = ''' + default_destination: cluster_default + tools: + blah: + default_destination: fake_destination + verbose: True +''' diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index 4e0cdcdb042..d4028d13a91 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -6,53 +6,38 @@ from __future__ import print_function import errno -import gzip import os import shutil import sys -import tempfile -import zipfile -from json import dumps, loads +from json import dump, load, loads from six.moves.urllib.request import urlopen from galaxy import util from galaxy.datatypes import sniff from galaxy.datatypes.registry import Registry -from galaxy.datatypes.upload_util import ( - handle_sniffable_binary_check, - handle_unsniffable_binary_check, - UploadProblemException, -) +from galaxy.datatypes.upload_util import UploadProblemException from galaxy.util.checkers import ( check_binary, - check_bz2, - check_gzip, - check_html, - check_zip + is_single_file_zip, + is_zip, ) -if sys.version_info < (3, 3): - import bz2file as bz2 -else: - import bz2 - assert sys.version_info[:2] >= (2, 7) -def file_err(msg, dataset, json_file): - json_file.write(dumps(dict(type='dataset', - ext='data', - dataset_id=dataset.dataset_id, - stderr=msg, - failed=True)) + "\n") +def file_err(msg, dataset): # never remove a server-side upload - if dataset.type in ('server_dir', 'path_paste'): - return - try: - os.remove(dataset.path) - except Exception: - pass + if dataset.type not in ('server_dir', 'path_paste'): + try: + os.remove(dataset.path) + except Exception: + pass + return dict(type='dataset', + ext='data', + dataset_id=dataset.dataset_id, + stderr=msg, + failed=True) def safe_dict(d): @@ -76,8 +61,9 @@ def parse_outputs(args): return rval -def add_file(dataset, registry, json_file, output_path): - data_type = None +def add_file(dataset, registry, output_path): + ext = None + compression_type = None line_count = None converted_path = None stdout = None @@ -115,7 +101,7 @@ def add_file(dataset, registry, json_file, output_path): # decompressing archive files before sniffing. auto_decompress = dataset.get('auto_decompress', True) try: - ext = dataset.file_type + dataset.file_type except AttributeError: raise UploadProblemException('Unable to process uploaded file, missing file_type parameter.') @@ -132,184 +118,90 @@ def add_file(dataset, registry, json_file, output_path): if not os.path.getsize(dataset.path) > 0: raise UploadProblemException('The uploaded file is empty') - # Is dataset content supported sniffable binary? + # Does the first 1K contain a null? is_binary = check_binary(dataset.path) - if is_binary: - data_type, ext = handle_sniffable_binary_check(data_type, ext, dataset.path, registry) - if not data_type: - root_datatype = registry.get_datatype_by_extension(dataset.file_type) - if getattr(root_datatype, 'compressed', False): - data_type = 'compressed archive' - ext = dataset.file_type + + # Decompress if needed/desired and determine/validate filetype. If a keep-compressed datatype is explicitly selected + # or if autodetection is selected and the file sniffs as a keep-compressed datatype, it will not be decompressed. + if not link_data_only: + if is_zip(dataset.path) and not is_single_file_zip(dataset.path): + stdout = 'ZIP file contained more than one file, only the first file was added to Galaxy.' + try: + ext, converted_path, compression_type = sniff.handle_uploaded_dataset_file( + dataset.path, + registry, + ext=dataset.file_type, + tmp_prefix='data_id_%s_upload_' % dataset.dataset_id, + tmp_dir=output_adjacent_tmpdir(output_path), + in_place=in_place, + check_content=check_content, + is_binary=is_binary, + auto_decompress=auto_decompress, + uploaded_file_ext=os.path.splitext(dataset.name)[1].lower().lstrip('.'), + convert_to_posix_lines=dataset.to_posix_lines, + convert_spaces_to_tabs=dataset.space_to_tab, + ) + except sniff.InappropriateDatasetContentError as exc: + raise UploadProblemException(str(exc)) + elif dataset.file_type == 'auto': + # Link mode can't decompress anyway, so enable sniffing for keep-compressed datatypes even when auto_decompress + # is enabled + os.environ['GALAXY_SNIFFER_VALIDATE_MODE'] = '1' + ext = sniff.guess_ext(dataset.path, registry.sniff_order, is_binary=is_binary) + os.environ.pop('GALAXY_SNIFFER_VALIDATE_MODE') + + # The converted path will be the same as the input path if no conversion was done (or in-place conversion is used) + converted_path = None if converted_path == dataset.path else converted_path + + # Validate datasets where the filetype was explicitly set using the filetype's sniffer (if any) + if dataset.file_type != 'auto': + datatype = registry.get_datatype_by_extension(dataset.file_type) + # Enable sniffer "validate mode" (prevents certain sniffers from disabling themselves) + os.environ['GALAXY_SNIFFER_VALIDATE_MODE'] = '1' + if hasattr(datatype, 'sniff') and not datatype.sniff(dataset.path): + stdout = ("Warning: The file 'Type' was set to '{ext}' but the file does not appear to be of that" + " type".format(ext=dataset.file_type)) + os.environ.pop('GALAXY_SNIFFER_VALIDATE_MODE') + + # Handle unsniffable binaries + if is_binary and ext == 'binary': + upload_ext = os.path.splitext(dataset.name)[1].lower().lstrip('.') + if registry.is_extension_unsniffable_binary(upload_ext): + stdout = ("Warning: The file's datatype cannot be determined from its contents and was guessed based on" + " its extension, to avoid this warning, manually set the file 'Type' to '{ext}' when uploading" + " this type of file".format(ext=upload_ext)) + ext = upload_ext else: - # See if we have a gzipped file, which, if it passes our restrictions, we'll uncompress - is_gzipped, is_valid = check_gzip(dataset.path, check_content=check_content) - if is_gzipped and not is_valid: - raise UploadProblemException('The gzipped uploaded file contains inappropriate content') - elif is_gzipped and is_valid and auto_decompress: - if not link_data_only: - # We need to uncompress the temp_name file, but BAM files must remain compressed in the BGZF format - CHUNK_SIZE = 2 ** 20 # 1Mb - fd, uncompressed = tempfile.mkstemp(prefix='data_id_%s_upload_gunzip_' % dataset.dataset_id, dir=os.path.dirname(output_path), text=False) - gzipped_file = gzip.GzipFile(dataset.path, 'rb') - while 1: - try: - chunk = gzipped_file.read(CHUNK_SIZE) - except IOError: - os.close(fd) - os.remove(uncompressed) - raise UploadProblemException('Problem decompressing gzipped data') - if not chunk: - break - os.write(fd, chunk) - os.close(fd) - gzipped_file.close() - # Replace the gzipped file with the decompressed file if it's safe to do so - if not in_place: - dataset.path = uncompressed - else: - shutil.move(uncompressed, dataset.path) - os.chmod(dataset.path, 0o644) - dataset.name = dataset.name.rstrip('.gz') - data_type = 'gzip' - if not data_type: - # See if we have a bz2 file, much like gzip - is_bzipped, is_valid = check_bz2(dataset.path, check_content) - if is_bzipped and not is_valid: - raise UploadProblemException('The gzipped uploaded file contains inappropriate content') - elif is_bzipped and is_valid and auto_decompress: - if not link_data_only: - # We need to uncompress the temp_name file - CHUNK_SIZE = 2 ** 20 # 1Mb - fd, uncompressed = tempfile.mkstemp(prefix='data_id_%s_upload_bunzip2_' % dataset.dataset_id, dir=os.path.dirname(output_path), text=False) - bzipped_file = bz2.BZ2File(dataset.path, 'rb') - while 1: - try: - chunk = bzipped_file.read(CHUNK_SIZE) - except IOError: - os.close(fd) - os.remove(uncompressed) - raise UploadProblemException('Problem decompressing bz2 compressed data') - if not chunk: - break - os.write(fd, chunk) - os.close(fd) - bzipped_file.close() - # Replace the bzipped file with the decompressed file if it's safe to do so - if not in_place: - dataset.path = uncompressed - else: - shutil.move(uncompressed, dataset.path) - os.chmod(dataset.path, 0o644) - dataset.name = dataset.name.rstrip('.bz2') - data_type = 'bz2' - if not data_type: - # See if we have a zip archive - is_zipped = check_zip(dataset.path) - if is_zipped and auto_decompress: - if not link_data_only: - CHUNK_SIZE = 2 ** 20 # 1Mb - uncompressed = None - uncompressed_name = None - unzipped = False - z = zipfile.ZipFile(dataset.path) - for name in z.namelist(): - if name.endswith('/'): - continue - if unzipped: - stdout = 'ZIP file contained more than one file, only the first file was added to Galaxy.' - break - fd, uncompressed = tempfile.mkstemp(prefix='data_id_%s_upload_zip_' % dataset.dataset_id, dir=os.path.dirname(output_path), text=False) - if sys.version_info[:2] >= (2, 6): - zipped_file = z.open(name) - while 1: - try: - chunk = zipped_file.read(CHUNK_SIZE) - except IOError: - os.close(fd) - os.remove(uncompressed) - raise UploadProblemException('Problem decompressing zipped data') - if not chunk: - break - os.write(fd, chunk) - os.close(fd) - zipped_file.close() - uncompressed_name = name - unzipped = True - else: - # python < 2.5 doesn't have a way to read members in chunks(!) - try: - with open(uncompressed, 'wb') as outfile: - outfile.write(z.read(name)) - uncompressed_name = name - unzipped = True - except IOError: - os.close(fd) - os.remove(uncompressed) - raise UploadProblemException('Problem decompressing zipped data') - z.close() - # Replace the zipped file with the decompressed file if it's safe to do so - if uncompressed is not None: - if not in_place: - dataset.path = uncompressed - else: - shutil.move(uncompressed, dataset.path) - os.chmod(dataset.path, 0o644) - dataset.name = uncompressed_name - data_type = 'zip' - if not data_type: - data_type, ext = handle_unsniffable_binary_check( - data_type, ext, dataset.path, dataset.name, is_binary, dataset.file_type, check_content, registry - ) - if not data_type: - # We must have a text file - if check_content and check_html(dataset.path): - raise UploadProblemException('The uploaded file contains inappropriate HTML content') - if data_type != 'binary': - if not link_data_only and data_type not in ('gzip', 'bz2', 'zip'): - # Convert universal line endings to Posix line endings if to_posix_lines is True - # and the data is not binary or gzip-, bz2- or zip-compressed. - if dataset.to_posix_lines: - tmpdir = output_adjacent_tmpdir(output_path) - tmp_prefix = 'data_id_%s_convert_' % dataset.dataset_id - if dataset.space_to_tab: - line_count, converted_path = sniff.convert_newlines_sep2tabs(dataset.path, in_place=in_place, tmp_dir=tmpdir, tmp_prefix=tmp_prefix) - else: - line_count, converted_path = sniff.convert_newlines(dataset.path, in_place=in_place, tmp_dir=tmpdir, tmp_prefix=tmp_prefix) - if dataset.file_type == 'auto': - ext = sniff.guess_ext(converted_path or dataset.path, registry.sniff_order) - else: - ext = dataset.file_type - data_type = ext - # Save job info for the framework - if ext == 'auto' and data_type == 'binary': - ext = 'data' - if ext == 'auto' and dataset.ext: - ext = dataset.ext - if ext == 'auto': - ext = 'data' + stdout = ("The uploaded binary file format cannot be determined automatically, please set the file 'Type'" + " manually") + datatype = registry.get_datatype_by_extension(ext) + + # Strip compression extension from name + if compression_type and not getattr(datatype, 'compressed', False) and dataset.name.endswith('.' + compression_type): + dataset.name = dataset.name[:-len('.' + compression_type)] + + # Move dataset if link_data_only: # Never alter a file that will not be copied to Galaxy's local file store. if datatype.dataset_content_needs_grooming(dataset.path): err_msg = 'The uploaded files need grooming, so change your Copy data into Galaxy? selection to be ' + \ 'Copy files into Galaxy instead of Link to files without copying into Galaxy so grooming can be performed.' raise UploadProblemException(err_msg) - if not link_data_only and converted_path: - # Move the dataset to its "real" path - try: - shutil.move(converted_path, output_path) - except OSError as e: - # We may not have permission to remove converted_path - if e.errno != errno.EACCES: - raise - elif not link_data_only: - if purge_source: - shutil.move(dataset.path, output_path) + if not link_data_only: + # Move the dataset to its "real" path. converted_path is a tempfile so we move it even if purge_source is False. + if purge_source or converted_path: + try: + shutil.move(converted_path or dataset.path, output_path) + except OSError as e: + # We may not have permission to remove the input + if e.errno != errno.EACCES: + raise else: shutil.copy(dataset.path, output_path) + # Write the job info - stdout = stdout or 'uploaded %s file' % data_type + stdout = stdout or 'uploaded %s file' % ext info = dict(type='dataset', dataset_id=dataset.dataset_id, ext=ext, @@ -318,13 +210,14 @@ def add_file(dataset, registry, json_file, output_path): line_count=line_count) if dataset.get('uuid', None) is not None: info['uuid'] = dataset.get('uuid') - json_file.write(dumps(info) + "\n") + # FIXME: does this belong here? also not output-adjacent-tmpdir aware =/ if not link_data_only and datatype and datatype.dataset_content_needs_grooming(output_path): # Groom the dataset content if necessary datatype.groom_dataset_content(output_path) + return info -def add_composite_file(dataset, json_file, output_path, files_path): +def add_composite_file(dataset, output_path, files_path): if dataset.composite_files: os.mkdir(files_path) for name, value in dataset.composite_files.items(): @@ -352,10 +245,33 @@ def add_composite_file(dataset, json_file, output_path, files_path): # Move the dataset to its "real" path shutil.move(dataset.primary_file, output_path) # Write the job info - info = dict(type='dataset', + return dict(type='dataset', dataset_id=dataset.dataset_id, stdout='uploaded %s file' % dataset.file_type) - json_file.write(dumps(info) + "\n") + + +def __read_paramfile(path): + with open(path) as fh: + obj = load(fh) + # If there's a single dataset in an old-style paramfile it'll still parse, but it'll be a dict + assert type(obj) == list + return obj + + +def __read_old_paramfile(path): + datasets = [] + with open(path) as fh: + for line in fh: + datasets.append(loads(line)) + return datasets + + +def __write_job_metadata(metadata): + # TODO: make upload/set_metadata compatible with https://github.com/galaxyproject/galaxy/pull/4437 + with open('galaxy.json', 'w') as fh: + for meta in metadata: + dump(meta, fh) + fh.write('\n') def output_adjacent_tmpdir(output_path): @@ -373,13 +289,17 @@ def __main__(): sys.exit(1) output_paths = parse_outputs(sys.argv[4:]) - json_file = open('galaxy.json', 'w') registry = Registry() registry.load_datatypes(root_dir=sys.argv[1], config=sys.argv[2]) - for line in open(sys.argv[3], 'r'): - dataset = loads(line) + try: + datasets = __read_paramfile(sys.argv[3]) + except (ValueError, AssertionError): + datasets = __read_old_paramfile(sys.argv[3]) + + metadata = [] + for dataset in datasets: dataset = util.bunch.Bunch(**safe_dict(dataset)) try: output_path = output_paths[int(dataset.dataset_id)][0] @@ -389,18 +309,12 @@ def __main__(): try: if dataset.type == 'composite': files_path = output_paths[int(dataset.dataset_id)][1] - add_composite_file(dataset, json_file, output_path, files_path) + metadata.append(add_composite_file(dataset, output_path, files_path)) else: - add_file(dataset, registry, json_file, output_path) + metadata.append(add_file(dataset, registry, output_path)) except UploadProblemException as e: - file_err(e.message, dataset, json_file) - # clean up paramfile - # TODO: this will not work when running as the actual user unless the - # parent directory is writable by the user. - try: - os.remove(sys.argv[3]) - except Exception: - pass + metadata.append(file_err(e.message, dataset)) + __write_job_metadata(metadata) if __name__ == '__main__': diff --git a/tools/data_source/upload.xml b/tools/data_source/upload.xml index 664e27eddc4..0c0fea4ce93 100644 --- a/tools/data_source/upload.xml +++ b/tools/data_source/upload.xml @@ -1,12 +1,12 @@ - + from your computer - - upload.py $GALAXY_ROOT_DIR $GALAXY_DATATYPES_CONF_FILE $paramfile + + python '$__tool_directory__/upload.py' $GALAXY_ROOT_DIR $GALAXY_DATATYPES_CONF_FILE $paramfile #set $outnum = 0 #while $varExists('output%i' % $outnum): #set $output = $getVar('output%i' % $outnum)