@@ -170,7 +170,8 @@ export default {
return this.item.published ? "accessible via link and published" : "accessible via link";
},
item_url() {
- return `${window.location.protocol}//${window.location.hostname}:${window.location.port}${getAppRoot()}${
+ const port = window.location.port ? `:${window.location.port}` : "";
+ return `${window.location.protocol}//${window.location.hostname}${port}${getAppRoot()}${
this.item.username_and_slug
}`;
},
@@ -263,7 +264,7 @@ export default {
.catch(error => (this.err_msg = error.response.data.err_msg));
},
createSlugHandler: function() {
- var on_start = function(text_elt) {
+ const on_start = function(text_elt) {
// Replace URL with URL text.
$("#item-url").hide();
$("#item-url-text").show();
@@ -279,14 +280,14 @@ export default {
);
});
};
- var on_finish = function(text_elt) {
+ const on_finish = function(text_elt) {
// Replace URL text with URL.
$("#item-url-text").hide();
$("#item-url").show();
// Set URL to new value.
- var new_url = $("#item-url-text").text();
- var item_url_obj = $("#item-url");
+ const new_url = $("#item-url-text").text();
+ const item_url_obj = $("#item-url");
item_url_obj.attr("href", new_url);
item_url_obj.text(new_url);
};
diff --git a/client/galaxy/scripts/components/Tags/mounts.js b/client/galaxy/scripts/components/Tags/mounts.js
index 832bacd414a..2ac287cdcc4 100644
--- a/client/galaxy/scripts/components/Tags/mounts.js
+++ b/client/galaxy/scripts/components/Tags/mounts.js
@@ -43,8 +43,6 @@ const makoClickHandler = (options, vm) =>
const { tagClickFn = "none", clickUrl } = options;
switch (tagClickFn) {
- // I made this match the existing behavior, but I am not clear on
- // the reason why this link redirects to a raw json page
case "community_tag_click":
if (undefined !== clickUrl) {
const suffix = tag.value ? `:${tag.value}` : "";
diff --git a/client/galaxy/scripts/components/admin/Jobs.vue b/client/galaxy/scripts/components/admin/Jobs.vue
new file mode 100644
index 00000000000..f3500e08676
--- /dev/null
+++ b/client/galaxy/scripts/components/admin/Jobs.vue
@@ -0,0 +1,370 @@
+
+
+
+ Jobs
+
+
+ {{ message }}
+
+
+
+ Unfinished and recently finished jobs are displayed on this page. The 'cutoff' input box will do two
+ things -- it will limit the display of unfinished jobs to only those jobs that have not had their job
+ state updated recently, and it will limit the recently finished jobs list to only displaying jobs that
+ have finished since the cutoff.
+
+ If any jobs are displayed, you may choose to stop them. Your stop message will be displayed to the user as:
+ "This job was stopped by an administrator: <YOUR MESSAGE> For more information or
+ help, report this error".
+
+
+ Waiting for data
+
+
+
+
+
+
+
+
+
+
+
+ Refresh
+
+
+
+
+
+
+
+
+ Job dispatching is currently
+ {{ jobLockDisplay ? "locked" : "unlocked" }}
+
+
+
+
+
+
+
+
+
+
+ Clear (esc)
+
+
+
+
+ {{ showCommandLine ? "Hide" : "Show" }} Command Line
+
+
+
+
+
+
+
+
+
+
+ Submit
+
+
+
+
+
+
+
+
+
+ There are no unfinished jobs to show with current cutoff time.
+
+
+
+ Unfinished Jobs: These jobs are unfinished and have had their state updated in the previous
+ {{ cutoffDisplay }} seconds.
+
+
+
+
+
+
+
+
+
+
+
+ {{ data.value["id"] }}
+
+
+
+
+
Command Line
+
{{ row.item.command_line }}
+
+
+
+
+ There are no recently finished jobs to show with current cutoff time.
+
+
+
+ Recent Jobs: These jobs have completed in the previous {{ cutoffDisplay }} seconds.
+
+
+
+
+
+ {{ data.value["id"] }}
+
+
+
+
+
Command Line
+
{{ row.item.command_line }}
+
+
+
+
+
+
+
+
+
+
diff --git a/client/galaxy/scripts/entry/admin/AdminRouter.js b/client/galaxy/scripts/entry/admin/AdminRouter.js
index 3379920a6aa..03b5e41743f 100644
--- a/client/galaxy/scripts/entry/admin/AdminRouter.js
+++ b/client/galaxy/scripts/entry/admin/AdminRouter.js
@@ -7,6 +7,7 @@ import QueryStringParsing from "utils/query-string-parsing";
import Router from "layout/router";
import DataTables from "components/admin/DataTables.vue";
import DataTypes from "components/admin/DataTypes.vue";
+import Jobs from "components/admin/Jobs.vue";
import DataManagerView from "components/admin/DataManager/DataManagerView.vue";
import DataManagerRouter from "components/admin/DataManager/DataManagerRouter.vue";
import Register from "components/login/Register.vue";
@@ -33,6 +34,7 @@ export const getAdminRouter = (Galaxy, options) => {
"(/)admin(/)form(/)(:form_id)": "show_form",
"(/)admin/data_tables": "show_data_tables",
"(/)admin/data_types": "show_data_types",
+ "(/)admin/jobs": "show_jobs",
"(/)admin/data_manager*path": "show_data_manager",
"*notFound": "not_found"
},
@@ -115,6 +117,10 @@ export const getAdminRouter = (Galaxy, options) => {
this._display_vue_helper(DataTypes);
},
+ show_jobs: function() {
+ this._display_vue_helper(Jobs);
+ },
+
show_error_stack: function() {
this._display_vue_helper(ErrorStack);
},
diff --git a/client/galaxy/scripts/entry/panels/admin-panel.js b/client/galaxy/scripts/entry/panels/admin-panel.js
index a94035e382e..44e60acc517 100644
--- a/client/galaxy/scripts/entry/panels/admin-panel.js
+++ b/client/galaxy/scripts/entry/panels/admin-panel.js
@@ -39,6 +39,7 @@ var AdminPanel = Backbone.View.extend({
{
title: _l("Manage jobs"),
url: "admin/jobs",
+ target: "__use_router__",
id: "admin-link-jobs"
},
{
diff --git a/client/galaxy/scripts/galaxy.interactive_environments.js b/client/galaxy/scripts/galaxy.interactive_environments.js
index 6ae025c5bb7..3f371508b09 100644
--- a/client/galaxy/scripts/galaxy.interactive_environments.js
+++ b/client/galaxy/scripts/galaxy.interactive_environments.js
@@ -1,6 +1,6 @@
-/* global $ */
-/* global toastr */
-// TODO: this file is transpiled and used directly without bundling; fix imports when that is no longer the case.
+import $ from "jquery";
+import { Toast } from "ui/toast";
+
/**
* Internal function to remove content from the main area and add the notebook.
* Not idempotent
@@ -35,7 +35,7 @@ export function make_spin_state(
sleep_step,
log_attempts
) {
- var s = {
+ return {
type: typeof type !== "undefined" ? type : "GIE spin",
ajax_timeout: typeof ajax_timeout_init !== "undefined" ? ajax_timeout_init : 2000,
ajax_timeout_max: typeof ajax_timeout_max !== "undefined" ? ajax_timeout_max : 16000,
@@ -46,7 +46,6 @@ export function make_spin_state(
log_attempts: typeof log_attempts !== "undefined" ? log_attempts : true,
count: 0
};
- return s;
}
/* Log/display an error when spinning fails. */
@@ -54,8 +53,8 @@ export function spin_error(console_msg, user_msg, clear) {
console.log(console_msg);
if (clear) clear_main_area();
if (typeof user_msg == "string") {
- toastr.clear();
- toastr.error(user_msg, "Error", {
+ Toast.clear();
+ Toast.error(user_msg, "Error", {
closeButton: true,
timeOut: 0,
extendedTimeOut: 0,
@@ -112,7 +111,7 @@ export function spin(url, bool_response, success_callback, timeout_callback, err
}
}
};
- if (bool_response) ajax_params["dataType"] = "json";
+ if (bool_response) ajax_params.dataType = "json";
$.ajax(ajax_params);
};
console.log(`Setting up new spinner for ${spin_state.type} on ${url}`);
@@ -131,7 +130,7 @@ function spin_until(url, bool_response, messages, success_callback, spin_state)
var message_once = (message, spin_state) => {
if (spin_state.count == 1) {
display_spinner();
- toastr.info(message, null, {
+ Toast.info(message, null, {
closeButton: true,
timeOut: 0,
extendedTimeOut: 0,
@@ -141,22 +140,22 @@ function spin_until(url, bool_response, messages, success_callback, spin_state)
};
var wrapped_success = data => {
if (!bool_response || (bool_response && data == true)) {
- console.log(messages["success"]);
+ console.log(messages.success);
clear_main_area();
- toastr.clear();
+ Toast.clear();
success_callback();
} else if (bool_response && data == false) {
- message_once(messages["not_ready"], spin_state);
+ message_once(messages.not_ready, spin_state);
return false; // keep spinning
} else {
- spin_error(`Invalid response to ${spin_state.type} request`, messages["invalid_response"], true);
+ spin_error(`Invalid response to ${spin_state.type} request`, messages.invalid_response, true);
}
return true; // stop spinning
};
var timeout_error = (jqxhr, status, error) => {
- message_once(messages["waiting"], spin_state);
+ message_once(messages.waiting, spin_state);
if (spin_state.count == warn_at) {
- toastr.warning(messages["wait_warn"], "Warning", {
+ Toast.warning(messages.wait_warn, "Warning", {
closeButton: true,
timeOut: 0,
extendedTimeOut: 0,
@@ -235,7 +234,7 @@ export function keepAlive(notebookAccessURL) {
if (request_count > 30) {
window.clearInterval(interval);
clear_main_area();
- toastr.error("Could not connect to IE, contact your administrator", "Error", {
+ Toast.error("Could not connect to IE, contact your administrator", "Error", {
closeButton: true,
timeOut: 20000,
tapToDismiss: false
@@ -245,3 +244,15 @@ export function keepAlive(notebookAccessURL) {
});
}, 10000);
}
+
+export default {
+ append_notebook,
+ clear_main_area,
+ display_spinner,
+ make_spin_state,
+ spin_error,
+ spin,
+ test_ie_availability,
+ load_when_ready,
+ keepAlive
+};
diff --git a/client/galaxy/scripts/mvc/history/history-contents.js b/client/galaxy/scripts/mvc/history/history-contents.js
index a0049811ff6..54eb7757e80 100644
--- a/client/galaxy/scripts/mvc/history/history-contents.js
+++ b/client/galaxy/scripts/mvc/history/history-contents.js
@@ -442,7 +442,8 @@ export var HistoryContents = _super.extend(BASE_MVC.LoggableMixin).extend({
data: JSON.stringify({
content: id,
source: contentType,
- type: type
+ type: type,
+ copy_elements: true
})
})
.done(response => {
diff --git a/client/galaxy/scripts/mvc/library/library-dataset-view.js b/client/galaxy/scripts/mvc/library/library-dataset-view.js
index ec35272dac6..bd16d475e7f 100644
--- a/client/galaxy/scripts/mvc/library/library-dataset-view.js
+++ b/client/galaxy/scripts/mvc/library/library-dataset-view.js
@@ -611,516 +611,588 @@ var LibraryDatasetView = Backbone.View.extend({
templateDataset: function() {
return _.template(
- [
- // CONTAINER START
- '
',
- "<% if (is_admin) { %>",
- "You are logged in as an administrator therefore you can manage any dataset on this Galaxy instance. Please make sure you understand the consequences.",
- "<% } else { %>",
- "You can assign any number of roles to any of the following permission types. However please read carefully the implications of such actions.",
- "<% } %>",
- "
",
- '
',
- "
Library-related permissions
",
- "
Roles that can modify the library item
",
- '',
- '
User with any of these roles can modify name, metadata, and other information about this library item.
',
- "",
- "
Dataset-related permissions
",
- '
Changes made below will affect every library item that was created from this dataset and also every history this dataset is part of.
',
- '<% if (!item.get("is_unrestricted")) { %>',
- '
',
- "User has to have all these roles in order to access this dataset.",
- " Users without access permission cannot have other permissions on this dataset.",
- " If there are no access roles set on the dataset it is considered unrestricted.",
- "
",
- "
Roles that can manage permissions on the dataset
",
- '',
- '
',
- "User with any of these roles can manage permissions of this dataset. If you remove yourself you will lose the ability manage this dataset unless you are an admin.",
- "
",
- '",
- "
",
- // CONTAINER END
- "
"
- ].join("")
+
Dataset: <%= _.escape(item.get("name")) %>
+
+ <% if (is_admin) { %>
+ You are logged in as an administrator therefore you can manage any dataset
+ on this Galaxy instance. Please make sure you understand the consequences.
+ <% } else { %>
+ You can assign any number of roles to any of the following permission types. However please
+ read carefully the implications of such actions.
+ <% } %>
+
+
+
Library-related permissions
+
Roles that can modify the library item
+
+
+ User with any of these roles can modify name, metadata,
+ and other information about this library item.
+
+
+
Dataset-related permissions
+
+ Changes made below will affect every library item that was created from
+ this dataset and also every history this dataset is part of.
+
+ User has to have all these roles in order to access this dataset.
+ Users without access permission cannot have other permissions on this dataset.
+ If there are no access roles set on the dataset it is considered unrestricted.
+
+
Roles that can manage permissions on the dataset
+
+
+ User with any of these roles can manage permissions of this dataset.
+ If you remove yourself you will lose the ability manage this dataset unless you are an admin.
+
',
- "<% if (is_admin) { %>",
- "You are logged in as an administrator therefore you can manage any folder on this Galaxy instance. Please make sure you understand the consequences.",
- "<% } else { %>",
- "You can assign any number of roles to any of the following permission types. However please read carefully the implications of such actions.",
- "<% }%>",
- "
",
- '
',
- "
Folder permissions
",
- "
",
- "Roles that can manage permissions on this folder",
- "
",
- '',
- '
',
- "User with any of these roles can manage permissions on this folder.",
- "
",
- "
",
- "Roles that can add items to this folder",
- "
",
- '',
- '
',
- "User with any of these roles can add items to this folder (folders and datasets).",
- "
",
- "
",
- "Roles that can modify this folder",
- "
",
- '',
- '
',
- "User with any of these roles can modify this folder (name, etc.).",
- "
+ <% if (is_admin) { %>
+ You are logged in as an administrator therefore you can manage any
+ folder on this Galaxy instance. Please make sure you understand the consequences.
+ <% } else { %>
+ You can assign any number of roles to any of the following permission types.
+ However please read carefully the implications of such actions.
+ <% }%>
+
+
+
Folder permissions
+
+ Roles that can manage permissions on this folder
+
+
+
+ User with any of these roles can manage permissions on this folder.
+
+
+ Roles that can add items to this folder
+
+
+
+ User with any of these roles can add items to this folder (folders and datasets).
+
+
+ Roles that can modify this folder
+
+
+
+ User with any of these roles can modify this folder (name, etc.).
+
',
- "This folder is either empty or you do not have proper access permissions to see the contents. If you expected something to show up",
- ' please consult the library security wikipage.',
- "
+ This folder is either empty or you do not have proper access permissions to see the contents.
+ If you expected something to show up please consult the
+
+ library security wikipage
+ .
+
",
- "List: Generic collection which groups any number of datasets into a set; similar to file system folder.",
- "
",
- "
",
- "Paired: Simple collection containing exactly two sequence datasets; one reverse and the other forward.",
- "
",
- "
",
- "List of Pairs: Advanced collection containing any number of Pairs; imagine as Pair-type collections inside of a List-type collection.",
- "
",
- "
",
- "From Rules: Use Galaxy's rule builder to describe collections. This is more of an advanced feature that allows building any number of collections or any type.",
- "
",
- "
",
- "
",
- // history selection/creation
- '
',
- "
Select history
",
- '',
- " or create new: ",
- '',
- "
",
- "
"
- ].join("")
+ `
+
+
Which datasets?
+
+
+
+
+
Collection type
+
+
+
+
Which type to choose?
+
+
List
+
Generic collection which groups any number of datasets into a set; similar to file system folder.
+
+
Paired
+
Simple collection containing exactly two sequence datasets; one reverse and the other forward.
+
+
List of Pairs
+
Advanced collection containing any number of Pairs; imagine as Pair-type collections inside of a List-type collection.
+
+
From Rules
+
Use Galaxy's rule builder to describe collections. This is more of an advanced feature that allows building any number of collections or any type.
',
- "<% if (is_admin) { %>",
- "You are logged in as an administrator therefore you can manage any library on this Galaxy instance. Please make sure you understand the consequences.",
- "<% } else { %>",
- "You can assign any number of roles to any of the following permission types. However please read carefully the implications of such actions.",
- "<% }%>",
- "
",
- '
',
- "
Library permissions
",
- "
Roles that can access the library
",
- '',
- '
',
- "User with any of these roles can access this library. If there are no access roles set on the library it is considered unrestricted.",
- "
",
- "
Roles that can manage permissions on this library
",
- '',
- '
',
- "User with any of these roles can manage permissions on this library (includes giving access).",
- "
",
- "
Roles that can add items to this library
",
- '',
- '
',
- "User with any of these roles can add items to this library (folders and datasets).",
- "
",
- "
Roles that can modify this library
",
- '',
- '
',
- "User with any of these roles can modify this library (name, synopsis, etc.).",
- "
+ <% if (is_admin) { %>
+ You are logged in as an administrator therefore you can manage any library
+ on this Galaxy instance. Please make sure you understand the consequences.
+ <% } else { %>
+ You can assign any number of roles to any of the following permission types.
+ However please read carefully the implications of such actions.
+ <% }%>
+
+
+
Library permissions
+
Roles that can access the library
+
+
+ User with any of these roles can access this library.
+ If there are no access roles set on the library it is considered unrestricted.
+
+
Roles that can manage permissions on this library
+
+
+ User with any of these roles can manage permissions on this library
+ (includes giving access).
+
+
Roles that can add items to this library
+
+
+ User with any of these roles can add items to this library (folders and datasets).
+
+
Roles that can modify this library
+
+
+ User with any of these roles can modify this library (name, synopsis, etc.).
+
"
- ].join("")
- );
+ templatePaginatorText: function() {
+ return _.template(`per page, <%- total_libraries_count %> total`);
}
});
diff --git a/client/galaxy/scripts/mvc/tool/tool-form-base.js b/client/galaxy/scripts/mvc/tool/tool-form-base.js
index 2d4c5d9e2aa..ceb1a0dd109 100644
--- a/client/galaxy/scripts/mvc/tool/tool-form-base.js
+++ b/client/galaxy/scripts/mvc/tool/tool-form-base.js
@@ -75,15 +75,9 @@ export default FormBase.extend({
`${options.name} ${options.description} (Galaxy Version ${options.version})`,
operations: !options.hide_operations && this._operations(),
onchange: function() {
- const Galaxy = getGalaxyInstance();
self.deferred.reset();
self.deferred.execute(process => {
self.model.get("postchange")(process, self);
- if (self.model.get("listen_to_history")) {
- process.then(() => {
- self.stopListening(Galaxy.currHistoryPanel.collection);
- });
- }
});
}
});
diff --git a/client/galaxy/scripts/mvc/tool/tool-form.js b/client/galaxy/scripts/mvc/tool/tool-form.js
index a47e97b40df..eb720d453b3 100644
--- a/client/galaxy/scripts/mvc/tool/tool-form.js
+++ b/client/galaxy/scripts/mvc/tool/tool-form.js
@@ -242,7 +242,10 @@ var View = Backbone.View.extend({
toolId: job_def.tool_id
});
}
- Galaxy.currHistoryPanel && Galaxy.currHistoryPanel.refreshContents();
+ if (Galaxy.currHistoryPanel) {
+ self.form.stopListening(Galaxy.currHistoryPanel.collection);
+ Galaxy.currHistoryPanel.refreshContents();
+ }
},
error: function(response) {
callback && callback();
diff --git a/client/galaxy/scripts/mvc/ui/popup-menu.js b/client/galaxy/scripts/mvc/ui/popup-menu.js
index 410166d607c..6166a6e078e 100644
--- a/client/galaxy/scripts/mvc/ui/popup-menu.js
+++ b/client/galaxy/scripts/mvc/ui/popup-menu.js
@@ -5,7 +5,7 @@
import $ from "jquery";
import _ from "underscore";
import Backbone from "backbone";
-var PopupMenu = Backbone.View.extend({
+const PopupMenu = Backbone.View.extend({
//TODO: maybe better as singleton off the Galaxy obj
/** Cache the desired button element and options, set up the button click handler
* NOTE: attaches this view as HTML/jQ data on the button for later use.
@@ -20,11 +20,10 @@ var PopupMenu = Backbone.View.extend({
this.$button.data("popupmenu", this);
// set up button click -> open menu behavior
- var menu = this;
this.$button.click(event => {
// if there's already a menu open, remove it
$(".popmenu-wrapper").remove();
- menu._renderAndShow(event);
+ this._renderAndShow(event);
return false;
});
},
@@ -51,13 +50,12 @@ var PopupMenu = Backbone.View.extend({
// set up behavior on each link/anchor elem
if (this.options.length) {
- var menu = this;
- this.$(".popupmenu-option").each(function(i) {
- var option = menu.options[i];
+ this.$(".popupmenu-option").each((i, el) => {
+ const option = this.options[i];
// if the option has 'func', call that function when the anchor is clicked
if (option.func) {
- $(this).click(event => {
- option.func.call(menu, event, option);
+ $(el).click(event => {
+ option.func.call(this, event, option);
// We must preventDefault otherwise clicking "cancel"
// on a purge or something still navigates and causes
// the action.
@@ -86,9 +84,9 @@ var PopupMenu = Backbone.View.extend({
} else if (option.header) {
return `
${option.html}
`;
}
- var href = option.href || "javascript:void(0);";
- var target = option.target ? `target="${option.target}"` : "";
- var check = option.checked ? '' : "";
+ const href = option.href || "javascript:void(0);";
+ const target = option.target ? `target="${option.target}"` : "";
+ const check = option.checked ? '' : "";
return `${check}${option.html}`;
}).join("");
},
@@ -96,8 +94,8 @@ var PopupMenu = Backbone.View.extend({
// get the absolute position/offset for the menu
_getShownPosition: function(clickEvent) {
// display menu horiz. centered on click...
- var menuWidth = this.$el.width();
- var x = clickEvent.pageX - menuWidth / 2;
+ const menuWidth = this.$el.width();
+ let x = clickEvent.pageX - menuWidth / 2;
// adjust to handle horiz. scroll and window dimensions ( draw entirely on visible screen area )
x = Math.min(x, $(document).scrollLeft() + $(window).width() - menuWidth - 5);
@@ -111,11 +109,14 @@ var PopupMenu = Backbone.View.extend({
// bind an event handler to all available frames so that when anything is clicked
// the menu is removed from the DOM and the event handler unbinds itself
_setUpCloseBehavior: function() {
- var menu = this;
//TODO: alternately: focus hack, blocking overlay, jquery.blockui
// function to close popup and unbind itself
- function closePopup(event) {
+ const closePopup = event => {
+ //do nothing if header item is clicked
+ if (event.target.classList.contains("dropdown-header")) {
+ return;
+ }
$(document).off("click.close_popup");
try {
if (window && window.parent !== window) {
@@ -134,19 +135,19 @@ var PopupMenu = Backbone.View.extend({
console.debug(err);
}
}
- menu.remove();
- }
+ this.remove();
+ };
- $("html").one("click.close_popup", closePopup);
+ $("html").on("click.close_popup", closePopup);
try {
if (window && window.parent !== window) {
$(window.parent.document)
.find("html")
- .one("click.close_popup", closePopup);
+ .on("click.close_popup", closePopup);
} else {
$("iframe#galaxy_main")
.contents()
- .one("click.close_popup", closePopup);
+ .on("click.close_popup", closePopup);
}
} catch (err) {
if (err instanceof DOMException) {
@@ -177,7 +178,7 @@ var PopupMenu = Backbone.View.extend({
// search for a menu option by its html
findIndexByHtml: function(html) {
- for (var i = 0; i < this.options.length; i++) {
+ for (let i = 0; i < this.options.length; i++) {
if (_.has(this.options[i], "html") && this.options[i].html === html) {
return i;
}
@@ -200,134 +201,5 @@ PopupMenu.create = function _create($button, options) {
return new PopupMenu($button, options);
};
-// -----------------------------------------------------------------------------
-// the following class functions are bridges from the original make_popupmenu and make_popup_menus
-// to the newer backbone.js PopupMenu
-
-/** Create a PopupMenu from simple map initial_options activated by clicking button_element.
- * Converts initial_options to object array used by PopupMenu.
- * @param {jQuery|DOMElement} button_element element which, when clicked, activates menu
- * @param {Object} initial_options map of key -> values, where
- * key is option text, value is fn to call when option is clicked
- * @returns {PopupMenu} the PopupMenu created
- */
-PopupMenu.make_popupmenu = (button_element, initial_options) => {
- var convertedOptions = [];
- _.each(initial_options, (optionVal, optionKey) => {
- var newOption = { html: optionKey };
-
- // keys with null values indicate: header
- if (optionVal === null) {
- // !optionVal? (null only?)
- newOption.header = true;
-
- // keys with function values indicate: a menu option
- } else if ($.type(optionVal) === "function") {
- newOption.func = optionVal;
- }
- //TODO:?? any other special optionVals?
- // there was no divider option originally
- convertedOptions.push(newOption);
- });
- return new PopupMenu($(button_element), convertedOptions);
-};
-
-/** Find all anchors in $parent (using selector) and covert anchors into a PopupMenu options map.
- * @param {jQuery} $parent the element that contains the links to convert to options
- * @param {String} selector jq selector string to find links
- * @returns {Object[]} the options array to initialize a PopupMenu
- */
-//TODO: lose parent and selector, pass in array of links, use map to return options
-PopupMenu.convertLinksToOptions = ($parent, selector) => {
- $parent = $($parent);
- selector = selector || "a";
- var options = [];
- $parent.find(selector).each((elem, i) => {
- var option = {};
- var $link = $(elem);
-
- // convert link text to the option text (html) and the href into the option func
- option.html = $link.text();
- if ($link.attr("href")) {
- var linkHref = $link.attr("href");
- var linkTarget = $link.attr("target");
- var confirmText = $link.attr("confirm");
-
- option.func = () => {
- // if there's a "confirm" attribute, throw up a confirmation dialog, and
- // if the user cancels - do nothing
- if (confirmText && !confirm(confirmText)) {
- return;
- }
-
- // if there's no confirm attribute, or the user accepted the confirm dialog:
- switch (linkTarget) {
- // relocate the center panel
- case "_parent":
- window.parent.location = linkHref;
- break;
-
- // relocate the entire window
- case "_top":
- window.top.location = linkHref;
- break;
-
- // relocate this panel
- default:
- window.location = linkHref;
- }
- };
- }
- options.push(option);
- });
- return options;
-};
-
-/** Create a single popupmenu from existing DOM button and anchor elements
- * @param {jQuery} $buttonElement the element that when clicked will open the menu
- * @param {jQuery} $menuElement the element that contains the anchors to convert into a menu
- * @param {String} menuElementLinkSelector jq selector string used to find anchors to be made into menu options
- * @returns {PopupMenu} the PopupMenu (Backbone View) that can render, control the menu
- */
-PopupMenu.fromExistingDom = ($buttonElement, $menuElement, menuElementLinkSelector) => {
- $buttonElement = $($buttonElement);
- $menuElement = $($menuElement);
- var options = PopupMenu.convertLinksToOptions($menuElement, menuElementLinkSelector);
- // we're done with the menu (having converted it to an options map)
- $menuElement.remove();
- return new PopupMenu($buttonElement, options);
-};
-
-/** Create all popupmenus within a document or a more specific element
- * @param {DOMElement} parent the DOM element in which to search for popupmenus to build (defaults to document)
- * @param {String} menuSelector jq selector string to find popupmenu menu elements (defaults to "div[popupmenu]")
- * @param {Function} buttonSelectorBuildFn the function to build the jq button selector.
- * Will be passed $menuElement, parent.
- * (Defaults to return '#' + $menuElement.attr( 'popupmenu' ); )
- * @returns {PopupMenu[]} array of popupmenus created
- */
-PopupMenu.make_popup_menus = (parent, menuSelector, buttonSelectorBuildFn) => {
- parent = parent || document;
- // orig. Glx popupmenu menus have a (non-std) attribute 'popupmenu'
- // which contains the id of the button that activates the menu
- menuSelector = menuSelector || "div[popupmenu]";
- // default to (orig. Glx) matching button to menu by using the popupmenu attr of the menu as the id of the button
- buttonSelectorBuildFn = buttonSelectorBuildFn || (($menuElement, parent) => `#${$menuElement.attr("popupmenu")}`);
-
- // aggregate and return all PopupMenus
- var popupMenusCreated = [];
- $(parent)
- .find(menuSelector)
- .each(function() {
- var $menuElement = $(this);
-
- var $buttonElement = $(parent).find(buttonSelectorBuildFn($menuElement, parent));
-
- popupMenusCreated.push(PopupMenu.fromDom($buttonElement, $menuElement));
- $buttonElement.addClass("popup");
- });
- return popupMenusCreated;
-};
-
// =============================================================================
export default PopupMenu;
diff --git a/client/galaxy/scripts/mvc/ui/ui-select-content.js b/client/galaxy/scripts/mvc/ui/ui-select-content.js
index 9c3658f2178..d9baa250c59 100644
--- a/client/galaxy/scripts/mvc/ui/ui-select-content.js
+++ b/client/galaxy/scripts/mvc/ui/ui-select-content.js
@@ -356,7 +356,7 @@ const View = Backbone.View.extend({
},
{
multiple: cnf.multiple,
- library: cnf.library,
+ library: !!cnf.library,
format: null
}
);
diff --git a/client/galaxy/scripts/mvc/ui/ui-slider.js b/client/galaxy/scripts/mvc/ui/ui-slider.js
index 6ce0d64ba94..8f3c39c915a 100644
--- a/client/galaxy/scripts/mvc/ui/ui-slider.js
+++ b/client/galaxy/scripts/mvc/ui/ui-slider.js
@@ -1,11 +1,9 @@
-import $ from "jquery";
import Backbone from "backbone";
import Utils from "utils/utils";
-var View = Backbone.View.extend({
+const View = Backbone.View.extend({
initialize: function(options) {
- var self = this;
this.model =
(options && options.model) ||
new Backbone.Model({
@@ -26,51 +24,25 @@ var View = Backbone.View.extend({
this.$slider = this.$(".ui-form-slider-element");
// add text field event
- var pressed = [];
this.$text
- .on("change", function() {
- self.value($(this).val());
+ .on("change", e => {
+ this.value(e.currentTarget.value);
})
- .on("keyup", e => {
- pressed[e.which] = false;
- })
- .on("keydown", function(e) {
- var v = e.which;
- pressed[v] = true;
- if (self.model.get("is_workflow") && pressed[16] && v == 52) {
- self.value("$");
- event.preventDefault();
- } else if (
- !(
- v == 8 ||
- v == 9 ||
- v == 13 ||
- v == 37 ||
- v == 39 ||
- (v >= 48 && v <= 57) ||
- (v >= 96 && v <= 105) ||
- ((v == 190 || v == 110) &&
- $(this)
- .val()
- .indexOf(".") == -1 &&
- self.model.get("precise")) ||
- ((v == 189 || v == 109) &&
- $(this)
- .val()
- .indexOf("-") == -1) ||
- self._isParameter($(this).val()) ||
- pressed[91] ||
- pressed[17]
- )
- ) {
- event.preventDefault();
+ .on("input", e => {
+ const input = e.currentTarget;
+ if (this._isParameter(input.value)) {
+ return;
}
+ else if (!this.model.get("precise")) {
+ input.value = input.value.split(".")[0];
+ }
+ input.value = input.value.replace(/[^0-9eE.-]/g, '');
});
// build slider, cannot be rebuild in render
- var opts = this.model.attributes;
+ const opts = this.model.attributes;
this.has_slider = opts.max !== null && opts.min !== null && opts.max > opts.min;
- var step = opts.step;
+ let step = opts.step;
if (!step) {
if (opts.precise && this.has_slider) {
step = (opts.max - opts.min) / opts.split;
@@ -80,7 +52,7 @@ var View = Backbone.View.extend({
}
if (this.has_slider) {
this.$slider.slider({ min: opts.min, max: opts.max, step: step }).on("slide", (event, ui) => {
- self.value(ui.value);
+ this.value(ui.value);
});
}
@@ -90,7 +62,7 @@ var View = Backbone.View.extend({
},
render: function() {
- var value = this.model.get("value");
+ const value = this.model.get("value");
if (this.has_slider) {
this.$slider.slider("value", value);
this.$slider.show();
diff --git a/client/galaxy/scripts/mvc/upload/upload-ftp.js b/client/galaxy/scripts/mvc/upload/upload-ftp.js
index 19417c3cf5d..98f59d512d6 100644
--- a/client/galaxy/scripts/mvc/upload/upload-ftp.js
+++ b/client/galaxy/scripts/mvc/upload/upload-ftp.js
@@ -2,7 +2,6 @@
import _ from "underscore";
import $ from "jquery";
import Backbone from "backbone";
-import { getAppRoot } from "onload/loadConfig";
import { getGalaxyInstance } from "app";
import Utils from "utils/utils";
import UploadUtils from "mvc/upload/upload-utils";
@@ -15,7 +14,7 @@ export default Backbone.View.extend({
class_remove: "upload-icon-button fa fa-check-square-o",
class_partial: "upload-icon-button fa fa-minus-square-o",
help_enabled: true,
- oidc_text: ` If you are signed-in to Galaxy using a third-party identity and you don't have a Galaxy password please go to this page and request a password for your Galaxy account.`,
+ oidc_text: ` If you are signed-in to Galaxy using a third-party identity and you do not have a Galaxy password please use the reset password option in the login form with your email to create a password for your account.`,
help_text: `This Galaxy server allows you to upload files via FTP. To upload some files, log in to the FTP server at ${
options.ftp_upload_site
} using your Galaxy credentials.
diff --git a/client/galaxy/scripts/mvc/workflow/workflow-terminals.js b/client/galaxy/scripts/mvc/workflow/workflow-terminals.js
index a3db7d9eaf0..0f1015c9cf0 100644
--- a/client/galaxy/scripts/mvc/workflow/workflow-terminals.js
+++ b/client/galaxy/scripts/mvc/workflow/workflow-terminals.js
@@ -435,6 +435,10 @@ var InputTerminal = BaseInputTerminal.extend({
}
if (thisMapOver.isCollection && thisMapOver.canMatch(otherCollectionType)) {
return this._producesAcceptableDatatype(other);
+ } else if (this.multiple && new CollectionTypeDescription('list').canMatch(otherCollectionType)) {
+ // This handles the special case of a list input being connected to a multiple="true" data input.
+ // Nested lists would be correctly mapped over by the above condition.
+ return this._producesAcceptableDatatype(other);
} else {
// Need to check if this would break constraints...
var mappingConstraints = this._mappingConstraints();
diff --git a/client/galaxy/scripts/mvc/workflow/workflow-view.js b/client/galaxy/scripts/mvc/workflow/workflow-view.js
index 23875f61489..b6bde09ec82 100644
--- a/client/galaxy/scripts/mvc/workflow/workflow-view.js
+++ b/client/galaxy/scripts/mvc/workflow/workflow-view.js
@@ -258,7 +258,7 @@ export default Backbone.View.extend({
for (let i = 0; i < workflow_versions.length; i++) {
const current_wf = workflow_versions[i];
- let version_text = `Version ${current_wf["version"]}, ${current_wf["steps"]} steps`;
+ let version_text = `Version ${current_wf.version}, ${current_wf.steps} steps`;
let selected = false;
if (i == self.workflow.workflow_version) {
version_text = `${version_text} (active)`;
@@ -287,7 +287,9 @@ export default Backbone.View.extend({
$("#workflow-version-switch").unbind("change");
if (this.value != self.workflow.workflow_version) {
if (self.workflow && self.workflow.has_changes) {
- const r = confirm("There are unsaved changes to your workflow which will be lost. Continue ?");
+ const r = window.confirm(
+ "There are unsaved changes to your workflow which will be lost. Continue ?"
+ );
if (r == false) {
// We rebuild the version select list, to reset the selected version
self.build_version_select();
@@ -774,6 +776,7 @@ export default Backbone.View.extend({
// Fix width to computed width
// Now add floats
var buttons = $("");
+ buttons.append($(`
diff --git a/config/plugins/interactive_environments/hicbrowser/static/js/hicbrowser.js b/config/plugins/interactive_environments/hicbrowser/static/js/main.js
similarity index 84%
rename from config/plugins/interactive_environments/hicbrowser/static/js/hicbrowser.js
rename to config/plugins/interactive_environments/hicbrowser/static/js/main.js
index a7803b7d16a..f0164119b9a 100644
--- a/config/plugins/interactive_environments/hicbrowser/static/js/hicbrowser.js
+++ b/config/plugins/interactive_environments/hicbrowser/static/js/main.js
@@ -3,12 +3,12 @@ var IES = window.IES;
// Load an interactive environment (IE) from a remote URL
// @param {String} hicexplorer_access_url: the URL embeded in the page and loaded
-function load_hicexplorer(hicexplorer_access_url){
+function load_hicexplorer(hicexplorer_access_url) {
// When the page has completely loaded...
- $( document ).ready(function() {
+ $(document).ready(function() {
// Test if we can access the GIE, and if so, execute the function
// to load the GIE for the user.
- IES.test_ie_availability(hicexplorer_access_url, function(){
+ IES.test_ie_availability(hicexplorer_access_url, function() {
IES.append_notebook(hicexplorer_access_url);
});
});
diff --git a/config/plugins/interactive_environments/hicbrowser/templates/hicbrowser.mako b/config/plugins/interactive_environments/hicbrowser/templates/hicbrowser.mako
index cf69cc8f633..b4ad8329dab 100644
--- a/config/plugins/interactive_environments/hicbrowser/templates/hicbrowser.mako
+++ b/config/plugins/interactive_environments/hicbrowser/templates/hicbrowser.mako
@@ -29,19 +29,14 @@
${ ie.load_default_js() }
+ ${ ie.load_default_app() }
diff --git a/config/plugins/interactive_environments/jupyter/static/js/jupyter.js b/config/plugins/interactive_environments/jupyter/static/js/main.js
similarity index 60%
rename from config/plugins/interactive_environments/jupyter/static/js/jupyter.js
rename to config/plugins/interactive_environments/jupyter/static/js/main.js
index 3d386889dba..3d0ac5b7118 100644
--- a/config/plugins/interactive_environments/jupyter/static/js/jupyter.js
+++ b/config/plugins/interactive_environments/jupyter/static/js/main.js
@@ -2,23 +2,25 @@
var IES = window.IES;
var toastr = window.toastr;
-function message_failed_auth(password){
+function message_failed_auth(password) {
toastr.info(
- "Automatic authorization failed. You can manually login with: " + password + " More details ...",
+ "Automatic authorization failed. You can manually login with: " +
+ password +
+ " More details ...",
"Please login manually",
- {'closeButton': true, 'timeOut': 100000, 'tapToDismiss': false}
+ { closeButton: true, timeOut: 100000, tapToDismiss: false }
);
}
-function message_failed_connection(){
+function message_failed_connection() {
toastr.error(
"Could not connect to Jupyter Notebook. Please contact your administrator. More details ...",
- "Security warning",
- {'closeButton': true, 'timeOut': 20000, 'tapToDismiss': true}
+ "Security warning",
+ { closeButton: true, timeOut: 20000, tapToDismiss: true }
);
}
-function message_no_auth(){
+function message_no_auth() {
// No longer a security issue, proxy validates Galaxy session token.
/*
toastr.warning(
@@ -29,7 +31,6 @@ function message_no_auth(){
*/
}
-
/**
* Load an interactive environment (IE) from a remote URL
* @param {String} password: password used to authenticate to the remote resource
@@ -37,68 +38,82 @@ function message_no_auth(){
* @param {String} notebook_access_url: the URL embeded in the page and loaded
*
*/
-function load_notebook(password, notebook_login_url, notebook_access_url){
+function load_notebook(password, notebook_login_url, notebook_access_url) {
// Test notebook_login_url for accessibility, executing the login+load function whenever
// we've successfully connected to the IE.
- IES.test_ie_availability(notebook_login_url, function(){
+ IES.test_ie_availability(notebook_login_url, function() {
_handle_notebook_loading(password, notebook_login_url, notebook_access_url);
keep_alive();
});
}
-
-function keep_alive(){
+function keep_alive() {
/**
- * This is needed to keep the container alive. If the user leaves this site
- * this function is not constantly pinging the container, the container will
- * terminate itself.
- */
+ * This is needed to keep the container alive. If the user leaves this site
+ * this function is not constantly pinging the container, the container will
+ * terminate itself.
+ */
var warn_at = 4;
var count_max = 60;
// we sleep 15 seconds between requests and the default timeout for the Jupyter container is 120 seconds, so start
// with a pretty high ajax timeout. sleep starts low because we want to get the warning up pretty quickly
var spin_state = IES.make_spin_state("IE keepalive", 8000, 16000, 2000, 5000, 15000, 5000, false);
- var success = function(){
+ var success = function() {
console.log("IE keepalive request succeeded");
toastr.clear();
- if(spin_state.count >= warn_at){
+ if (spin_state.count >= warn_at) {
toastr.clear();
- toastr.success(
- "Interactive environment connection restored",
- {'closeButton': true, 'timeOut': 5000, 'extendedTimeOut': 2000, 'tapToDismiss': true}
- );
+ toastr.success("Interactive environment connection restored", {
+ closeButton: true,
+ timeOut: 5000,
+ extendedTimeOut: 2000,
+ tapToDismiss: true
+ });
}
spin_state.count = 0;
spin_state.timeout_count = 0;
spin_state.error_count = 0;
- return false; // keep spinning
+ return false; // keep spinning
};
- var timeout_error = function(jqxhr, status, error){
- console.log("IE keepalive request failed " + spin_state.count + " time(s) of " + count_max + " max: " + status + ": " + error);
- if(spin_state.count == warn_at){
+ var timeout_error = function(jqxhr, status, error) {
+ console.log(
+ "IE keepalive request failed " +
+ spin_state.count +
+ " time(s) of " +
+ count_max +
+ " max: " +
+ status +
+ ": " +
+ error
+ );
+ if (spin_state.count == warn_at) {
toastr.warning(
- "Your browser has been unable to contact the interactive environment for "
- + spin_state.count + " consecutive attempts, if you do not reestablish "
- + "a connection, your IE container may be terminated.",
+ "Your browser has been unable to contact the interactive environment for " +
+ spin_state.count +
+ " consecutive attempts, if you do not reestablish " +
+ "a connection, your IE container may be terminated.",
"Warning",
- {'closeButton': true, 'timeOut': 0, 'extendedTimeOut': 0, 'tapToDismiss': false}
+ { closeButton: true, timeOut: 0, extendedTimeOut: 0, tapToDismiss: false }
);
- return false; // keep spinning
- }else if(spin_state.count >= count_max){
- IES.spin_error("IE keepalive failure limit reached", "Lost connection to interactive environment, contact your administrator", false);
- return true; // stop spinning
+ return false; // keep spinning
+ } else if (spin_state.count >= count_max) {
+ IES.spin_error(
+ "IE keepalive failure limit reached",
+ "Lost connection to interactive environment, contact your administrator",
+ false
+ );
+ return true; // stop spinning
}
};
console.log("IE keepalive worker starting");
IES.spin(notebook_keepalive_url, false, success, timeout_error, timeout_error, spin_state);
}
-
/**
* Must be implemented by IEs
*/
-function _handle_notebook_loading(password, notebook_login_url, notebook_access_url){
- if ( ie_password_auth ) {
+function _handle_notebook_loading(password, notebook_login_url, notebook_access_url) {
+ if (ie_password_auth) {
// Make an AJAX POST
$.ajax({
type: "POST",
@@ -106,29 +121,28 @@ function _handle_notebook_loading(password, notebook_login_url, notebook_access_
url: notebook_login_url,
// With our password
data: {
- 'password': password
+ password: password
},
xhrFields: {
withCredentials: true
},
// If that is successful, load the notebook
- success: function(){
+ success: function() {
IES.append_notebook(notebook_access_url);
},
- error: function(jqxhr, status, error){
- if(ie_password_auth){
+ error: function(jqxhr, status, error) {
+ if (ie_password_auth) {
// Failure happens due to CORS
message_failed_auth(password);
IES.append_notebook(notebook_access_url);
- }else{
+ } else {
message_failed_connection();
// Do we want to try and load the notebook anyway? Just in case?
IES.append_notebook(notebook_access_url);
}
}
});
- }
- else {
+ } else {
// Not using password auth, just embed it to avoid content-origin issues.
message_no_auth();
IES.append_notebook(notebook_access_url);
diff --git a/config/plugins/interactive_environments/jupyter/templates/jupyter.mako b/config/plugins/interactive_environments/jupyter/templates/jupyter.mako
index fc8db1f3e5d..eb26eb34fc7 100644
--- a/config/plugins/interactive_environments/jupyter/templates/jupyter.mako
+++ b/config/plugins/interactive_environments/jupyter/templates/jupyter.mako
@@ -50,6 +50,7 @@ notebook_keepalive_url = ie_request.url_template('${PROXY_URL}/ipython/tree')
${ ie.load_default_js() }
+${ ie.load_default_app() }
@@ -58,16 +59,10 @@ ${ ie.default_javascript_variables() }
var notebook_login_url = '${ notebook_login_url }';
var notebook_access_url = '${ notebook_access_url }';
var notebook_keepalive_url = '${ notebook_keepalive_url }';
-${ ie.plugin_require_config() }
// Load notebook
-
-requirejs(['galaxy.interactive_environments', 'plugin/jupyter'], function(IES){
- // This global is not awesome, get rid of it when possible (when IES are a part of the build process)
- window.IES = IES;
- IES.load_when_ready(ie_readiness_url, function(){
- load_notebook(ie_password, notebook_login_url, notebook_access_url);
- });
+IES.load_when_ready(ie_readiness_url, function(){
+ load_notebook(ie_password, notebook_login_url, notebook_access_url);
});
diff --git a/config/plugins/interactive_environments/neo/static/js/neo.js b/config/plugins/interactive_environments/neo/static/js/main.js
similarity index 76%
rename from config/plugins/interactive_environments/neo/static/js/neo.js
rename to config/plugins/interactive_environments/neo/static/js/main.js
index 37244a7dd6e..06571d8328a 100644
--- a/config/plugins/interactive_environments/neo/static/js/neo.js
+++ b/config/plugins/interactive_environments/neo/static/js/main.js
@@ -3,10 +3,10 @@ var IES = window.IES;
// Load an interactive environment (IE) from a remote URL
// @param {String} notebook_access_url: the URL embeded in the page and loaded
-function load_notebook(notebook_access_url){
+function load_notebook(notebook_access_url) {
// Test if we can access the GIE, and if so, execute the function
// to load the GIE for the user.
- IES.test_ie_availability(notebook_access_url, function(){
+ IES.test_ie_availability(notebook_access_url, function() {
IES.append_notebook(notebook_access_url);
});
}
diff --git a/config/plugins/interactive_environments/neo/templates/neo.mako b/config/plugins/interactive_environments/neo/templates/neo.mako
index c1df7807375..24f3e968cd8 100644
--- a/config/plugins/interactive_environments/neo/templates/neo.mako
+++ b/config/plugins/interactive_environments/neo/templates/neo.mako
@@ -23,20 +23,14 @@
${ ie.load_default_js() }
+ ${ ie.load_default_app() }
diff --git a/config/plugins/interactive_environments/phinch/static/js/main.js b/config/plugins/interactive_environments/phinch/static/js/main.js
new file mode 100644
index 00000000000..be12aeecca2
--- /dev/null
+++ b/config/plugins/interactive_environments/phinch/static/js/main.js
@@ -0,0 +1,54 @@
+// Globals to be rid of
+var IES = window.IES;
+var toastr = window.toastr;
+
+function load_notebook(url) {
+ IES.test_ie_availability(url, function() {
+ append_notebook(url);
+ });
+}
+
+function append_notebook(url) {
+ IES.clear_main_area();
+ $("#main").append(
+ ''
+ );
+}
+
+function keep_alive(notebook_access_url) {
+ /**
+ * This is needed to keep the container alive. If the user leaves this site
+ * this function is not constantly pinging the container, the container will
+ * terminate itself.
+ */
+
+ var request_count = 0;
+ var interval = window.setInterval(function() {
+ $.ajax({
+ url: notebook_access_url,
+ xhrFields: {
+ withCredentials: true
+ },
+ type: "GET",
+ timeout: 500,
+ success: function() {
+ console.log("Connected to IE, returning");
+ },
+ error: function(jqxhr, status, error) {
+ request_count++;
+ console.log("Request " + request_count);
+ if (request_count > 30) {
+ window.clearInterval(interval);
+ IES.clear_main_area();
+ toastr.error("Could not connect to IE, contact your administrator", "Error", {
+ closeButton: true,
+ timeOut: 20000,
+ tapToDismiss: false
+ });
+ }
+ }
+ });
+ }, 10000);
+}
diff --git a/config/plugins/interactive_environments/phinch/static/js/phinch.js b/config/plugins/interactive_environments/phinch/static/js/phinch.js
deleted file mode 100644
index 1fea1fc08fc..00000000000
--- a/config/plugins/interactive_environments/phinch/static/js/phinch.js
+++ /dev/null
@@ -1,54 +0,0 @@
-// Globals to be rid of
-var IES = window.IES;
-var toastr = window.toastr;
-
-function load_notebook(url){
- IES.test_ie_availability(url, function(){
- append_notebook(url);
- });
-}
-
-function append_notebook(url){
- IES.clear_main_area();
- $('#main').append(
- ''
- );
-}
-
-
-function keep_alive(notebook_access_url){
- /**
- * This is needed to keep the container alive. If the user leaves this site
- * this function is not constantly pinging the container, the container will
- * terminate itself.
- */
-
- var request_count = 0;
- var interval = window.setInterval(function(){
- $.ajax({
- url: notebook_access_url,
- xhrFields: {
- withCredentials: true
- },
- type: "GET",
- timeout: 500,
- success: function(){
- console.log("Connected to IE, returning");
- },
- error: function(jqxhr, status, error){
- request_count++;
- console.log("Request " + request_count);
- if(request_count > 30){
- window.clearInterval(interval);
- IES.clear_main_area();
- toastr.error(
- "Could not connect to IE, contact your administrator",
- "Error",
- {'closeButton': true, 'timeOut': 20000, 'tapToDismiss': false}
- );
- }
- }
- });
- }, 10000);
-}
-
diff --git a/config/plugins/interactive_environments/phinch/templates/phinch.mako b/config/plugins/interactive_environments/phinch/templates/phinch.mako
index a98e1309cb3..fc84a905ae9 100644
--- a/config/plugins/interactive_environments/phinch/templates/phinch.mako
+++ b/config/plugins/interactive_environments/phinch/templates/phinch.mako
@@ -21,21 +21,18 @@ url = ie_request.url_template('${PROXY_URL}/phinch/')
%>
-${ ie.load_default_js() }
+ ${ ie.load_default_js() }
+ ${ ie.load_default_app() }
diff --git a/config/plugins/interactive_environments/rstudio/config/allowed_images.yml.sample b/config/plugins/interactive_environments/rstudio/config/allowed_images.yml.sample
index a07739283d7..d9b061d93ce 100644
--- a/config/plugins/interactive_environments/rstudio/config/allowed_images.yml.sample
+++ b/config/plugins/interactive_environments/rstudio/config/allowed_images.yml.sample
@@ -6,7 +6,7 @@
# appropriate `apt-get/pip install` statements.
---
-
- image: erasche/docker-rstudio-notebook:19.05
+ image: quay.io/erasche/docker-rstudio-notebook:19.05
description: |
This familiar R analysis software suite will let you explore your
datasets in depth. Comes with ggplot2, RODBC, maps, shinyapps, knitr,
diff --git a/config/plugins/interactive_environments/rstudio/static/js/main.js b/config/plugins/interactive_environments/rstudio/static/js/main.js
new file mode 100755
index 00000000000..bf4bcac3dfc
--- /dev/null
+++ b/config/plugins/interactive_environments/rstudio/static/js/main.js
@@ -0,0 +1,24 @@
+// Assumed globals we need to get rid of
+var IES = window.IES;
+var toastr = window.toastr;
+
+function message_failed_connection() {
+ toastr.error("Could not connect to RStudio. Please contact your administrator.", "Security warning", {
+ closeButton: true,
+ timeOut: 20000,
+ tapToDismiss: true
+ });
+}
+
+/**
+ * Load an interactive environment (IE) from a remote URL
+ * @param {String} notebook_access_url: the URL embeded in the page and loaded
+ *
+ */
+function load_notebook(notebook_access_url) {
+ // Test notebook_login_url for accessibility, executing the login+load function whenever
+ // we've successfully connected to the IE.
+ IES.test_ie_availability(notebook_access_url, function() {
+ IES.append_notebook(notebook_access_url);
+ });
+}
diff --git a/config/plugins/interactive_environments/rstudio/static/js/rstudio.js b/config/plugins/interactive_environments/rstudio/static/js/rstudio.js
deleted file mode 100755
index c8be493fd5c..00000000000
--- a/config/plugins/interactive_environments/rstudio/static/js/rstudio.js
+++ /dev/null
@@ -1,32 +0,0 @@
-// Assumed globals we need to get rid of
-var IES = window.IES;
-var toastr = window.toastr;
-
-function message_failed_connection(){
- toastr.error(
- "Could not connect to RStudio. Please contact your administrator.",
- "Security warning",
- {'closeButton': true, 'timeOut': 20000, 'tapToDismiss': true}
- );
-}
-
-
-/**
- * Load an interactive environment (IE) from a remote URL
- * @param {String} notebook_access_url: the URL embeded in the page and loaded
- *
- */
-function load_notebook(notebook_access_url){
- // Test notebook_login_url for accessibility, executing the login+load function whenever
- // we've successfully connected to the IE.
- IES.test_ie_availability(notebook_access_url, function(){
- $.ajax({
- type: 'GET',
- url: notebook_access_url,
- success: function(response_text){
- IES.append_notebook(notebook_access_url);
- }
- });
-
- });
-}
diff --git a/config/plugins/interactive_environments/rstudio/templates/rstudio.mako b/config/plugins/interactive_environments/rstudio/templates/rstudio.mako
index a9c0bf776db..fd181ac41cc 100755
--- a/config/plugins/interactive_environments/rstudio/templates/rstudio.mako
+++ b/config/plugins/interactive_environments/rstudio/templates/rstudio.mako
@@ -27,18 +27,15 @@ notebook_access_url = ie_request.url_template('${PROXY_URL}/rstudio/')
${ ie.load_default_js() }
+${ ie.load_default_app() }
diff --git a/doc/source/admin/conda_faq.rst b/doc/source/admin/conda_faq.rst
index a4fc5a0221c..96aeadea7c1 100644
--- a/doc/source/admin/conda_faq.rst
+++ b/doc/source/admin/conda_faq.rst
@@ -393,15 +393,31 @@ Gitter or IRC channel.
17. How can I upgrade Conda?
****************************
-Many potential issues with Conda have been resolved with fixes in Conda itself. If
-you let Galaxy install Conda prior to the release of 17.01 you probably have version
-3.19.3. This can be updated to 4.2.13 with the following command:
+Many potential issues with Conda have been resolved with fixes in Conda itself.
+The Conda installed by Galaxy can be updated to e.g. version 4.6.14 with the
+following command:
.. code-block:: bash
- $ python >=3.6,<3.7.0a0
+ - python 3.5*
+ Use "conda info " to see the dependencies for each package.
+
+Then you need to also update the ``python`` package installed in the base
+environment by appending to the ``conda install`` command above an appropriate
+specification, which for the example error above would be ``python==3.6``:
+
+.. code-block:: bash
+
+ $ `__.
Setting up the Proxy
^^^^^^^^^^^^^^^^^^^^
-Currently the Galaxy proxy is a NodeJS+Sqlite3 proxy.
+The Galaxy IE Proxy is a NodeJS+Sqlite3 application. The NodeJS that is
+installed by default into the Galaxy Virtualenv is suitable for an execution
+environment for the Galaxy IE Proxy.
-- Node has recently upgraded, and our proxy is pinned to an old version of
- sqlite3. As such you'll currently need to have an older version of Node
- available (0.10.X - 0.11.X vintage).
-- One can use `NVM `__ to install and manage multiple versions of Node.
-- We're working on solutions in this space to provide a better deployment
- mechanism here and fewer dependencies.
-- Please note that if you have NodeJS installed under Ubuntu, it often
- installs to ``/usr/bin/nodejs``, whereas ``npm`` expects it to be
- ``/usr/bin/node``. You will need to create that symlink yourself.
+- Note that if you have NodeJS installed under Ubuntu, it often installs to
+ ``/usr/bin/nodejs``, whereas ``npm`` expects it to be ``/usr/bin/node``. You
+ may need to create that symlink yourself.
Once Node and npm are ready to go, you'll need to install the dependencies
diff --git a/doc/source/dev/interactive_environments.rst b/doc/source/dev/interactive_environments.rst
index a9372fdc228..7f457228c7e 100644
--- a/doc/source/dev/interactive_environments.rst
+++ b/doc/source/dev/interactive_environments.rst
@@ -205,9 +205,10 @@ We'll continue appending to our ``helloworld.mako`` the HTML code that's actuall
-
+
${ ie.load_default_js() }
+
+ ${ ie.load_default_app() }
@@ -217,18 +218,11 @@ We'll continue appending to our ``helloworld.mako`` the HTML code that's actuall
var notebook_login_url = 'unused';
var notebook_access_url = '${ notebook_access_url }';
- // Load code with require.js
- ${ ie.plugin_require_config() }
// Load notebook
- // This will load code from static/helloworld.js, often used to handle
- // things like Login. The load_notebook function will eventually append
- // an IFrame to the below.
- requirejs(['galaxy.interactive_environments', 'plugin/helloworld'], function(IES){
- window.IES = IES
- IES.load_when_ready(ie_readiness_url, function(){
- load_notebook(notebook_access_url);
- });
+ // The load_notebook function will eventually append an IFrame to the below.
+ IES.load_when_ready(ie_readiness_url, function(){
+ load_notebook(notebook_access_url);
});
@@ -262,7 +256,7 @@ authentication of users, so you don't have to worry about it, and can just
assume that only the correct user will have access to a given notebook.
In the ``static/`` directory, we generally create a ``js/`` directory below that,
-and create a ``{gie}.js`` (so, ``static/js/helloworld.js``) file in there.
+and create a ``main.js`` file in there.
That file will have a function, ``load_notebook`` which will check if the GIE is available,
and when it is, display it to the user.
diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py
index 6ada00bde4a..a447c6ba5a5 100644
--- a/lib/galaxy/datatypes/binary.py
+++ b/lib/galaxy/datatypes/binary.py
@@ -1107,6 +1107,66 @@ class Cool(H5):
return "Cool (HDF5) file (%s)." % (nice_size(dataset.get_size()))
+class MCool(H5):
+ """
+ Class describing the multi-resolution cool format (https://github.com/mirnylab/cooler)
+ """
+
+ file_ext = "mcool"
+
+ def sniff(self, filename):
+ """
+ >>> from galaxy.datatypes.sniff import get_test_fname
+ >>> fname = get_test_fname('matrix.mcool')
+ >>> MCool().sniff(fname)
+ True
+ >>> fname = get_test_fname('matrix.cool')
+ >>> MCool().sniff(fname)
+ False
+ >>> fname = get_test_fname('test.mz5')
+ >>> MCool().sniff(fname)
+ False
+ >>> fname = get_test_fname('wiggle.wig')
+ >>> MCool().sniff(fname)
+ False
+ >>> fname = get_test_fname('biom2_sparse_otu_table_hdf5.biom2')
+ >>> MCool().sniff(fname)
+ False
+ """
+
+ MAGIC = "HDF5::Cooler"
+ URL = "https://github.com/mirnylab/cooler"
+
+ if super(MCool, self).sniff(filename):
+ keys0 = ['resolutions']
+ with h5py.File(filename, 'r') as handle:
+ if not all(name in handle.keys() for name in keys0):
+ return False
+ res0 = list(handle['resolutions'].keys())[0]
+ keys = ['chroms', 'bins', 'pixels', 'indexes']
+ fmt = handle['resolutions'][res0].attrs.get('format', None)
+ url = handle['resolutions'][res0].attrs.get('format-url', None)
+ if fmt == MAGIC or url == URL:
+ if not all(name in handle['resolutions'][res0].keys() for name in keys):
+ return False
+ return True
+ return False
+
+ def set_peek(self, dataset, is_multi_byte=False):
+ if not dataset.dataset.purged:
+ dataset.peek = "Multi-resolution Cool (HDF5) file for storing genomic interaction data."
+ dataset.blurb = nice_size(dataset.get_size())
+ else:
+ dataset.peek = 'file does not exist'
+ dataset.blurb = 'file purged from disk'
+
+ def display_peek(self, dataset):
+ try:
+ return dataset.peek
+ except Exception:
+ return "MCool (HDF5) file (%s)." % (nice_size(dataset.get_size()))
+
+
class Scf(Binary):
"""Class describing an scf binary sequence file"""
edam_format = "format_1632"
diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py
index c0feebd7d52..c0ba39e006b 100644
--- a/lib/galaxy/datatypes/data.py
+++ b/lib/galaxy/datatypes/data.py
@@ -1030,8 +1030,14 @@ def get_file_peek(file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skipc
:param is_multi_byte: deprecated
:type is_multi_byte: bool
- >>> fname = get_test_fname('4.bed')
- >>> assert get_file_peek(fname, LINE_COUNT=1) == u'chr22\\t30128507\\t31828507\\tuc003bnx.1_cds_2_0_chr22_29227_f\\t0\\t+\\n'
+ >>> def assert_peek_is(file_name, expected, *args, **kwd):
+ ... path = get_test_fname(file_name)
+ ... peek = get_file_peek(path, *args, **kwd)
+ ... assert peek == expected, "%s != %s" % (peek, expected)
+ >>> assert_peek_is('0_nonewline', u'0')
+ >>> assert_peek_is('0.txt', u'0\\n')
+ >>> assert_peek_is('4.bed', u'chr22\\t30128507\\t31828507\\tuc003bnx.1_cds_2_0_chr22_29227_f\\t0\\t+\\n', LINE_COUNT=1)
+ >>> assert_peek_is('1.bed', u'chr1\\t147962192\\t147962580\\tCCDS989.1_cds_0_0_chr1_147962193_r\\t0\\t-\\nchr1\\t147984545\\t147984630\\tCCDS990.1_cds_0_0_chr1_147984546_f\\t0\\t+\\n', LINE_COUNT=2)
"""
# Set size for file.readline() to a negative number to force it to
# read until either a newline or EOF. Needed for datasets with very
@@ -1042,20 +1048,27 @@ def get_file_peek(file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skipc
skipchars = []
lines = []
count = 0
+
+ last_line_break = False
with compression_utils.get_fileobj(file_name, "U") as temp:
while count < LINE_COUNT:
try:
line = temp.readline(WIDTH)
except UnicodeDecodeError:
return "binary file"
- if not line_wrap:
- if line.endswith('\n'):
- line = line[:-1]
- else:
- while True:
- i = temp.read(1)
- if not i or i == '\n':
- break
+ if line == "":
+ break
+ last_line_break = False
+ if line.endswith('\n'):
+ line = line[:-1]
+ last_line_break = True
+ elif not line_wrap:
+ while True:
+ i = temp.read(1)
+ if i == '\n':
+ last_line_break = True
+ if not i or i == '\n':
+ break
skip_line = False
for skipchar in skipchars:
if line.startswith(skipchar):
@@ -1064,4 +1077,4 @@ def get_file_peek(file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skipc
if not skip_line:
lines.append(line)
count += 1
- return '\n'.join(lines)
+ return '\n'.join(lines) + ('\n' if last_line_break else '')
diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py
index 404f9c53a11..d35c83ffd4c 100644
--- a/lib/galaxy/datatypes/sniff.py
+++ b/lib/galaxy/datatypes/sniff.py
@@ -14,7 +14,11 @@ import sys
import tempfile
import zipfile
-from six import StringIO, text_type
+from six import (
+ PY3,
+ StringIO,
+ text_type,
+)
from six.moves import filter
from six.moves.urllib.request import urlopen
@@ -103,28 +107,40 @@ def stream_to_file(stream, suffix='', prefix='', dir=None, text=False, **kwd):
return stream_to_open_named_file(stream, fd, temp_name, **kwd)
-def convert_newlines(fname, in_place=True, tmp_dir=None, tmp_prefix="gxupload"):
+def convert_newlines(fname, in_place=True, tmp_dir=None, tmp_prefix="gxupload", block_size=128 * 1024, regexp=None):
"""
Converts in place a file from universal line endings
to Posix line endings.
-
- >>> fname = get_test_fname('temp.txt')
- >>> with open(fname, 'wt') as fh:
- ... _ = fh.write("1 2\\r3 4")
- >>> convert_newlines(fname, tmp_prefix="gxtest", tmp_dir=tempfile.gettempdir())
- (2, None)
- >>> open(fname).read()
- '1 2\\n3 4\\n'
"""
fd, temp_name = tempfile.mkstemp(prefix=tmp_prefix, dir=tmp_dir)
- with io.open(fd, mode="wt", encoding='utf-8') as fp:
- i = None
- for i, line in enumerate(io.open(fname, encoding='utf-8')):
- fp.write("%s\n" % line.rstrip("\r\n"))
- if i is None:
- i = 0
+ i = 0
+ if PY3:
+ NEWLINE_BYTE = 10
+ CR_BYTE = 13
else:
- i += 1
+ NEWLINE_BYTE = "\n"
+ CR_BYTE = "\r"
+ with io.open(fd, mode="wb") as fp, io.open(fname, mode="rb") as fi:
+ last_char = None
+ block = fi.read(block_size)
+ last_block = b""
+ while block:
+ if last_char == CR_BYTE and block.startswith(b"\n"):
+ # Last block ended with CR, new block startswith newline.
+ # Since we replace CR with newline in the previous iteration we skip the first byte
+ block = block[1:]
+ if block:
+ last_char = block[-1]
+ block = block.replace(b"\r\n", b"\n").replace(b"\r", b"\n")
+ if regexp:
+ block = b"\t".join(regexp.split(block))
+ fp.write(block)
+ i += block.count(b"\n")
+ last_block = block
+ block = fi.read(block_size)
+ if last_block and last_block[-1] != NEWLINE_BYTE:
+ i += 1
+ fp.write(b"\n")
if in_place:
shutil.move(temp_name, fname)
# Return number of lines in file.
@@ -133,47 +149,9 @@ def convert_newlines(fname, in_place=True, tmp_dir=None, tmp_prefix="gxupload"):
return (i, temp_name)
-def sep2tabs(fname, in_place=True, patt=r"\s+", tmp_dir=None, tmp_prefix="gxupload"):
+def convert_newlines_sep2tabs(fname, in_place=True, patt=br"[^\S\n]+", tmp_dir=None, tmp_prefix="gxupload"):
"""
- Transforms in place a 'sep' separated file to a tab separated one
-
- >>> fname = get_test_fname('temp.txt')
- >>> with open(fname, 'wt') as fh:
- ... _ = fh.write(u"1 2\\n3 4\\n")
- >>> sep2tabs(fname)
- (2, None)
- >>> open(fname).read()
- '1\\t2\\n3\\t4\\n'
- """
- regexp = re.compile(patt)
- fd, temp_name = tempfile.mkstemp(prefix=tmp_prefix, dir=tmp_dir)
- with io.open(fd, mode="wt", encoding='utf-8') as fp:
- i = None
- for i, line in enumerate(io.open(fname, encoding='utf-8')):
- if line.endswith("\r"):
- line = line.rstrip('\r')
- elems = regexp.split(line)
- fp.write(u"%s\r" % '\t'.join(elems))
- else:
- line = line.rstrip('\n')
- elems = regexp.split(line)
- fp.write(u"%s\n" % '\t'.join(elems))
- if i is None:
- i = 0
- else:
- i += 1
- if in_place:
- shutil.move(temp_name, fname)
- # Return number of lines in file.
- return (i, None)
- else:
- return (i, temp_name)
-
-
-def convert_newlines_sep2tabs(fname, in_place=True, patt=r"\s+", tmp_dir=None, tmp_prefix="gxupload"):
- """
- Combines above methods: convert_newlines() and sep2tabs()
- so that files do not need to be read twice
+ Converts newlines in a file to posix newlines and replaces spaces with tabs.
>>> fname = get_test_fname('temp.txt')
>>> with open(fname, 'wt') as fh:
@@ -184,18 +162,7 @@ def convert_newlines_sep2tabs(fname, in_place=True, patt=r"\s+", tmp_dir=None, t
'1\\t2\\n3\\t4\\n'
"""
regexp = re.compile(patt)
- fd, temp_name = tempfile.mkstemp(prefix=tmp_prefix, dir=tmp_dir)
- with io.open(fd, mode="wt", encoding='utf-8') as fp:
- for i, line in enumerate(io.open(fname, encoding='utf-8')):
- line = line.rstrip('\r\n')
- elems = regexp.split(line)
- fp.write(u"%s\n" % '\t'.join(elems))
- if in_place:
- shutil.move(temp_name, fname)
- # Return number of lines in file.
- return (i + 1, None)
- else:
- return (i + 1, temp_name)
+ return convert_newlines(fname, in_place, tmp_dir, tmp_prefix, regexp=regexp)
def iter_headers(fname_or_file_prefix, sep, count=60, comment_designator=None):
@@ -474,6 +441,9 @@ def guess_ext(fname, sniff_order, is_binary=False):
>>> fname = get_test_fname('1.mtx')
>>> guess_ext(fname, sniff_order)
'mtx'
+ >>> fname = get_test_fname('1imzml')
+ >>> guess_ext(fname, sniff_order) # This test case is ensuring doesn't throw exception, actual value could change if non-utf encoding handling improves.
+ 'data'
"""
file_prefix = FilePrefix(fname)
file_ext = run_sniffers_raw(file_prefix, sniff_order, is_binary)
@@ -555,8 +525,9 @@ def zip_single_fileobj(path):
class FilePrefix(object):
def __init__(self, filename):
- binary = False
+ non_utf8_error = None
compressed_format = None
+ contents_header_bytes = None
contents_header = None # First MAX_BYTES of the file.
truncated = False
# A future direction to optimize sniffing even more for sniffers at the top of the list
@@ -565,20 +536,23 @@ class FilePrefix(object):
# populates contents_header while providing a StringIO-like interface until the file is read
# but then would fallback to native string_io()
try:
- compressed_format, f = compression_utils.get_fileobj_raw(filename)
+ compressed_format, f = compression_utils.get_fileobj_raw(filename, "rb")
try:
- contents_header = f.read(SNIFF_PREFIX_BYTES)
- truncated = len(contents_header) == SNIFF_PREFIX_BYTES
+ contents_header_bytes = f.read(SNIFF_PREFIX_BYTES)
+ truncated = len(contents_header_bytes) == SNIFF_PREFIX_BYTES
+ contents_header = contents_header_bytes.decode("utf-8")
finally:
f.close()
- except UnicodeDecodeError:
- binary = True
+ except UnicodeDecodeError as e:
+ non_utf8_error = e
self.truncated = truncated
self.filename = filename
- self.binary = binary
+ self.non_utf8_error = non_utf8_error
+ self.binary = non_utf8_error is not None # obviously wrong
self.compressed_format = compressed_format
self.contents_header = contents_header
+ self.contents_header_bytes = contents_header_bytes
self._file_size = None
@property
@@ -588,8 +562,8 @@ class FilePrefix(object):
return self._file_size
def string_io(self):
- if self.binary:
- raise Exception("Attempting to create a StringIO object for binary data.")
+ if self.non_utf8_error is not None:
+ raise self.non_utf8_error
rval = StringIO(self.contents_header)
return rval
diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py
index da06227416a..2eabeb7e732 100644
--- a/lib/galaxy/datatypes/tabular.py
+++ b/lib/galaxy/datatypes/tabular.py
@@ -4,6 +4,7 @@ Tabular datatype
from __future__ import absolute_import
import abc
+import binascii
import csv
import logging
import os
@@ -691,11 +692,11 @@ class BaseVcf(Tabular):
MetadataElement(name="viz_filter_cols", desc="Score column for visualization", default=[5], param=metadata.ColumnParameter, optional=True, multiple=True, visible=False)
MetadataElement(name="sample_names", default=[], desc="Sample names", readonly=True, visible=False, optional=True, no_value=[])
- def sniff_prefix(self, file_prefix):
+ def _sniff(self, fname_or_file_prefix):
# Because this sniffer is run on compressed files that might be BGZF (due to the VcfGz subclass), we should
# handle unicode decode errors. This should ultimately be done in get_headers(), but guess_ext() currently
# relies on get_headers() raising this exception.
- headers = get_headers(file_prefix, '\n', count=1)
+ headers = get_headers(fname_or_file_prefix, '\n', count=1)
return headers[0][0].startswith("##fileformat=VCF")
def display_peek(self, dataset):
@@ -745,14 +746,29 @@ class BaseVcf(Tabular):
class Vcf(BaseVcf):
file_ext = 'vcf'
+ def sniff_prefix(self, file_prefix):
+ return self._sniff(file_prefix)
+
class VcfGz(BaseVcf, binary.Binary):
+ # This class name is a misnomer, should be VcfBgzip
file_ext = 'vcf_bgzip'
compressed = True
compressed_format = "gzip"
MetadataElement(name="tabix_index", desc="Vcf Index File", param=metadata.FileParameter, file_ext="tbi", readonly=True, no_value=None, visible=False, optional=True)
+ def sniff(self, filename):
+ if not self._sniff(filename):
+ return False
+ # Check that the file is compressed with bgzip (not gzip), i.e. the
+ # compressed format is BGZF, as explained in
+ # http://samtools.github.io/hts-specs/SAMv1.pdf
+ with open(filename, 'rb') as fh:
+ fh.seek(-28, 2)
+ last28 = fh.read()
+ return binascii.hexlify(last28) == b'1f8b08040000000000ff0600424302001b0003000000000000000000'
+
def set_meta(self, dataset, **kwd):
super(BaseVcf, self).set_meta(dataset, **kwd)
""" Creates the index for the VCF file. """
diff --git a/lib/galaxy/datatypes/test/0_nonewline b/lib/galaxy/datatypes/test/0_nonewline
new file mode 100644
index 00000000000..c227083464f
--- /dev/null
+++ b/lib/galaxy/datatypes/test/0_nonewline
@@ -0,0 +1 @@
+0
\ No newline at end of file
diff --git a/lib/galaxy/datatypes/test/1imzml b/lib/galaxy/datatypes/test/1imzml
new file mode 100644
index 00000000000..6b45d2a3667
--- /dev/null
+++ b/lib/galaxy/datatypes/test/1imzml
@@ -0,0 +1,380 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/test/dosimzml b/lib/galaxy/datatypes/test/dosimzml
new file mode 100644
index 00000000000..6ff4f5c078d
--- /dev/null
+++ b/lib/galaxy/datatypes/test/dosimzml
@@ -0,0 +1,380 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/test/matrix.mcool b/lib/galaxy/datatypes/test/matrix.mcool
new file mode 100644
index 00000000000..bf42ef345fe
Binary files /dev/null and b/lib/galaxy/datatypes/test/matrix.mcool differ
diff --git a/lib/galaxy/datatypes/test/vcf_gzipped.vcf.gz b/lib/galaxy/datatypes/test/vcf_gzipped.vcf.gz
new file mode 100644
index 00000000000..67e5ea92790
Binary files /dev/null and b/lib/galaxy/datatypes/test/vcf_gzipped.vcf.gz differ
diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py
index 06162cf7b96..274e1c69a80 100644
--- a/lib/galaxy/dependencies/__init__.py
+++ b/lib/galaxy/dependencies/__init__.py
@@ -34,20 +34,46 @@ class ConditionalDependencies(object):
def parse_configs(self):
self.config = load_app_properties(config_file=self.config_file)
- job_conf_xml = self.config.get(
- "job_config_file",
- join(dirname(self.config_file), 'job_conf.xml'))
- try:
- for plugin in ElementTree.parse(job_conf_xml).find('plugins').findall('plugin'):
- if 'load' in plugin.attrib:
- self.job_runners.append(plugin.attrib['load'])
- except (OSError, IOError):
- pass
- try:
- for plugin in ElementTree.parse(job_conf_xml).findall('.//destination/param[@id="rules_module"]'):
- self.job_rule_modules.append(plugin.text)
- except (OSError, IOError):
- pass
+
+ def load_job_config_dict(job_conf_dict):
+ for runner in job_conf_dict.get("runners"):
+ if "load" in runner:
+ self.job_runners.append(runner.get("load"))
+ if "rules_module" in runner:
+ self.job_rule_modules.append(plugin.text)
+ if "params" in runner:
+ runner_params = runner["params"]
+ if "rules_module" in runner_params:
+ self.job_rule_modules.append(plugin.text)
+
+ if "job_config" in self.config:
+ load_job_config_dict(self.config.get("job_config"))
+ else:
+ job_conf_path = self.config.get(
+ "job_config_file",
+ join(dirname(self.config_file), 'job_conf.xml'))
+ if '.xml' in job_conf_path:
+ try:
+ try:
+ for plugin in ElementTree.parse(job_conf_path).find('plugins').findall('plugin'):
+ if 'load' in plugin.attrib:
+ self.job_runners.append(plugin.attrib['load'])
+ except (OSError, IOError):
+ pass
+ try:
+ for plugin in ElementTree.parse(job_conf_path).findall('.//destination/param[@id="rules_module"]'):
+ self.job_rule_modules.append(plugin.text)
+ except (OSError, IOError):
+ pass
+ except ElementTree.ParseError:
+ pass
+ else:
+ try:
+ job_conf_dict = yaml.safe_load(job_conf_path)
+ load_job_config_dict(job_conf_dict)
+ except (OSError, IOError):
+ pass
+
object_store_conf_xml = self.config.get(
"object_store_config_file",
join(dirname(self.config_file), 'object_store_conf.xml'))
diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
index 5d05ea3e7a9..ea41b6dc6ff 100644
--- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
+++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
@@ -36,7 +36,7 @@ cachetools==3.1.0
certifi==2019.3.9
cffi==1.12.3
chardet==3.0.4
-cheetah3==3.2.1
+cheetah3==3.2.2
cliff==2.14.1
cloudauthz==0.2.0
cloudbridge==2.0.0
diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py
index ca1c5b3b4ce..33bcb0b4728 100644
--- a/lib/galaxy/jobs/__init__.py
+++ b/lib/galaxy/jobs/__init__.py
@@ -20,6 +20,7 @@ from tempfile import NamedTemporaryFile
from xml.etree import ElementTree
import six
+import yaml
from pulsar.client.staging import COMMAND_VERSION_FILENAME
import galaxy
@@ -46,7 +47,7 @@ log = logging.getLogger(__name__)
# that import Galaxy internals - but it shouldn't be used in Galaxy's code
# itself.
TOOL_PROVIDED_JOB_METADATA_FILE = 'galaxy.json'
-TOOL_PROVIDED_JOB_METADATA_KEYS = ['name', 'info', 'dbkey']
+TOOL_PROVIDED_JOB_METADATA_KEYS = ['name', 'info', 'dbkey', 'created_from_basename']
# Override with config.default_job_shell.
DEFAULT_JOB_SHELL = '/bin/bash'
@@ -80,10 +81,6 @@ class JobDestination(Bunch):
super(JobDestination, self).__init__(**kwds)
- # Store tags as a list
- if self.tags is not None:
- self['tags'] = [x.strip() for x in self.tags.split(',')]
-
class JobToolConfiguration(Bunch):
"""
@@ -112,6 +109,139 @@ def config_exception(e, file):
return Exception(message)
+def job_config_xml_to_dict(config, root):
+ config_dict = {}
+
+ runners = {}
+ config_dict["runners"] = runners
+
+ # Parser plugins section populate 'runners' and 'dynamic' in config_dict.
+ plugins = root.find('plugins')
+ if plugins is not None:
+ for plugin in ConfiguresHandlers._findall_with_required(plugins, 'plugin', ('id', 'type', 'load')):
+ if plugin.get('type') == 'runner':
+ workers = plugin.get('workers', plugins.get('workers', JobConfiguration.DEFAULT_NWORKERS))
+ runner_kwds = JobConfiguration.get_params(config, plugin)
+ plugin_id = plugin.get('id')
+ runner_info = dict(id=plugin_id,
+ load=plugin.get('load'),
+ workers=int(workers),
+ kwds=runner_kwds)
+ runners[plugin_id] = runner_info
+ else:
+ log.error('Unknown plugin type: %s' % plugin.get('type'))
+
+ for plugin in ConfiguresHandlers._findall_with_required(plugins, 'plugin', ('id', 'type')):
+ if plugin.get('id') == 'dynamic' and plugin.get('type') == 'runner':
+ config_dict["dynamic"] = JobConfiguration.get_params(config, plugin)
+
+ handling_config_dict = ConfiguresHandlers.xml_to_dict(config, root.find("handlers"))
+ config_dict["handling"] = handling_config_dict
+
+ # Parse destinations
+ environments = []
+
+ destinations = root.find('destinations')
+ for destination in ConfiguresHandlers._findall_with_required(destinations, 'destination', ('id', 'runner')):
+ destination_id = destination.get('id')
+ destination_metrics = destination.get("metrics", None)
+
+ environment = {"id": destination_id}
+
+ metrics_to_dict = {"src": "default"}
+ if destination_metrics:
+ if not util.asbool(destination_metrics):
+ metrics_to_dict = {"src": "disabled"}
+ else:
+ metrics_to_dict = {"src": "path", "path": destination_metrics}
+ else:
+ metrics_elements = ConfiguresHandlers._findall_with_required(destination, 'job_metrics', ())
+ if metrics_elements:
+ metrics_to_dict = {"src": "xml_element", 'xml_element': metrics_elements[0]}
+
+ environment["metrics"] = metrics_to_dict
+
+ params = JobConfiguration.get_params(config, destination)
+ # TODO: handle enabled/disabled in configure_from
+ environment['params'] = params
+ environment['env'] = JobConfiguration.get_envs(destination)
+ destination_resubmits = JobConfiguration.get_resubmits(destination)
+ if destination_resubmits:
+ environment['resubmit'] = destination_resubmits
+ # TODO: handle empty resubmits defaults in configure_from
+
+ runner = destination.get('runner')
+ if runner:
+ environment['runner'] = runner
+
+ tags = destination.get('tags')
+ # Store tags as a list
+ if tags is not None:
+ tags = [x.strip() for x in tags.split(',')]
+ environment['tags'] = tags
+
+ environments.append(environment)
+
+ config_dict['execution'] = {
+ 'environments': environments,
+ }
+ default_destination = ConfiguresHandlers.get_xml_default(config, destinations)
+ if default_destination:
+ config_dict['execution']['default'] = default_destination
+
+ resources_config_dict = {}
+ resource_groups = {}
+
+ # Parse resources...
+ resources = root.find('resources')
+ if resources is not None:
+ default_resource_group = resources.get("default", None)
+ if default_resource_group:
+ resources_config_dict["default"] = default_resource_group
+
+ for group in ConfiguresHandlers._findall_with_required(resources, 'group'):
+ group_id = group.get('id')
+ fields_str = group.get('fields', None) or group.text or ''
+ fields = [f for f in fields_str.split(",") if f]
+ resource_groups[group_id] = fields
+
+ resources_config_dict["groups"] = resource_groups
+ config_dict["resources"] = resources_config_dict
+
+ # Parse tool mappings
+ tools = root.find('tools')
+ config_dict['tools'] = []
+ if tools is not None:
+ for tool in ConfiguresHandlers._findall_with_required(tools, 'tool'):
+ # There can be multiple definitions with identical ids, but different params
+ tool_mapping_conf = {}
+ for key in ['handler', 'destination', 'id', 'resources']:
+ value = tool.get(key)
+ if value:
+ if key == "destination":
+ key = "environment"
+ tool_mapping_conf[key] = value
+ tool_mapping_conf["params"] = JobConfiguration.get_params(config, tool)
+ config_dict['tools'].append(tool_mapping_conf)
+
+ limits_config = []
+ limits = root.find('limits')
+ if limits is not None:
+ for limit in JobConfiguration._findall_with_required(limits, 'limit', ('type',)):
+ limit_dict = {}
+ for key in ['type', 'tag', 'id', 'window']:
+ if key == 'type' and key.startswith('destination_'):
+ key = 'environment_%s' % key[len("destination_"):]
+ value = limit.get(key)
+ if value:
+ limit_dict[key] = value
+ limit_dict['value'] = limit.text
+ limits_config.append(limit_dict)
+
+ config_dict['limits'] = limits_config
+ return config_dict
+
+
class JobConfiguration(ConfiguresHandlers):
"""A parser and interface to advanced job management features.
@@ -162,7 +292,7 @@ class JobConfiguration(ConfiguresHandlers):
default_resubmit_condition = self.app.config.default_job_resubmission_condition
if default_resubmit_condition:
default_resubmits.append(dict(
- destination=None,
+ environment=None,
condition=default_resubmit_condition,
handler=None,
delay=None,
@@ -171,10 +301,26 @@ class JobConfiguration(ConfiguresHandlers):
self.__parse_resource_parameters()
# Initialize the config
- job_config_file = self.app.config.job_config_file
try:
- tree = load(job_config_file)
- self.__parse_job_conf_xml(tree)
+ if 'job_config' in self.app.config.config_dict:
+ job_config_dict = self.app.config.config_dict["job_config"]
+ else:
+ job_config_file = self.app.config.job_config_file
+ if '.xml' in job_config_file:
+ tree = load(job_config_file)
+ job_config_dict = self.__parse_job_conf_xml(tree)
+ else:
+ with open(job_config_file, "r") as f:
+ job_config_dict = yaml.safe_load(f)
+
+ # Load tasks if configured
+ if self.app.config.use_tasked_jobs:
+ job_config_dict["runners"]["tasks"] = dict(id='tasks', load='tasks', workers=self.app.config.local_task_queue_workers, kwds={})
+
+ self._configure_from_dict(job_config_dict)
+
+ log.debug('Done loading job configuration')
+
except IOError:
log.warning('Job configuration "%s" does not exist, using default job configuration',
self.app.config.job_config_file)
@@ -182,43 +328,33 @@ class JobConfiguration(ConfiguresHandlers):
except Exception as e:
raise config_exception(e, job_config_file)
- def __parse_job_conf_xml(self, tree):
- """Loads the new-style job configuration from options in the job config file (by default, job_conf.xml).
-
- :param tree: Object representing the root ```` object in the job config file.
- :type tree: ``xml.etree.ElementTree.Element``
- """
- root = tree.getroot()
- log.debug('Loading job configuration from %s' % self.app.config.job_config_file)
-
- # Parse job plugins
- plugins = root.find('plugins')
- if plugins is not None:
- for plugin in self._findall_with_required(plugins, 'plugin', ('id', 'type', 'load')):
- if plugin.get('type') == 'runner':
- workers = plugin.get('workers', plugins.get('workers', JobConfiguration.DEFAULT_NWORKERS))
- runner_kwds = self.__get_params(plugin)
- if not self.__is_enabled(runner_kwds):
+ def _configure_from_dict(self, job_config_dict):
+ for runner_id, runner_info in job_config_dict["runners"].items():
+ if "kwds" not in runner_info:
+ # convert all 'extra' parameters into kwds, allows defining a runner
+ # with a flat dictionary.
+ kwds = {}
+ for key, value in runner_info.items():
+ if key in ['id', 'load', 'workers']:
continue
- runner_info = dict(id=plugin.get('id'),
- load=plugin.get('load'),
- workers=int(workers),
- kwds=runner_kwds)
- self.runner_plugins.append(runner_info)
- else:
- log.error('Unknown plugin type: %s' % plugin.get('type'))
- for plugin in self._findall_with_required(plugins, 'plugin', ('id', 'type')):
- if plugin.get('id') == 'dynamic' and plugin.get('type') == 'runner':
- self.dynamic_params = self.__get_params(plugin)
+ kwds[key] = value
+ runner_info["kwds"] = kwds
- # Load tasks if configured
- if self.app.config.use_tasked_jobs:
- self.runner_plugins.append(dict(id='tasks', load='tasks', workers=self.app.config.local_task_queue_workers))
+ if not self.__is_enabled(runner_info.get("kwds")):
+ continue
+ runner_info["id"] = runner_id
+ if runner_id == "dynamic":
+ log.warning('Deprecated treatment of dynamic running configuration as an actual job runner.')
+ self.dynamic_params = runner_info["kwds"]
+ continue
+ self.runner_plugins.append(runner_info)
+ if "dynamic" in job_config_dict:
+ self.dynamic_params = job_config_dict.get("dynamic", None)
# Parse handlers
- handlers_conf = root.find('handlers')
- self._init_handler_assignment_methods(handlers_conf)
- self._init_handlers(handlers_conf)
+ handling_config_dict = job_config_dict.get("handling", {})
+ self._init_handler_assignment_methods(handling_config_dict)
+ self._init_handlers(handling_config_dict)
if not self.handler_assignment_methods_configured:
self._set_default_handler_assignment_methods()
else:
@@ -227,38 +363,51 @@ class JobConfiguration(ConfiguresHandlers):
for tag, handlers in [(t, h) for t, h in self.handlers.items() if isinstance(h, list)]:
log.info("Tag [%s] handlers: %s", tag, ', '.join(handlers))
- # Parse destinations
- destinations = root.find('destinations')
+ # Parse environments
job_metrics = self.app.job_metrics
- for destination in self._findall_with_required(destinations, 'destination', ('id', 'runner')):
- id = destination.get('id')
- destination_metrics = destination.get("metrics", None)
- if destination_metrics:
- if not util.asbool(destination_metrics):
- # disable
- job_metrics.set_destination_instrumenter(id, None)
- else:
- metrics_conf_path = self.app.config.resolve_path(destination_metrics)
- job_metrics.set_destination_conf_file(id, metrics_conf_path)
- else:
- metrics_elements = self._findall_with_required(destination, 'job_metrics', ())
- if metrics_elements:
- job_metrics.set_destination_conf_element(id, metrics_elements[0])
- job_destination = JobDestination(**dict(destination.items()))
- params = self.__get_params(destination)
- if not self.__is_enabled(params):
+ execution_dict = job_config_dict.get('execution', {})
+ environments = execution_dict.get("environments", [])
+ enviroment_iter = map(lambda e: (e["id"], e), environments) if isinstance(environments, list) else environments.items()
+ for environment_id, environment_dict in enviroment_iter:
+ metrics = environment_dict.get("metrics") or {"src": "default"}
+ metrics_src = metrics.get("src") or "default"
+ if metrics_src != "default":
+ # customized metrics for this environment.
+ if metrics_src == "disabled":
+ job_metrics.set_destination_instrumenter(environment_id, None)
+ elif metrics_src == "xml_element":
+ metrics_element = metrics.get("xml_element")
+ job_metrics.set_destination_conf_element(environment_id, metrics_element)
+ elif metrics_src == "path":
+ metrics_conf_path = self.app.config.resolve_path(metrics.get("path"))
+ job_metrics.set_destination_conf_file(environment_id, metrics_conf_path)
+
+ destination_kwds = {}
+
+ params = environment_dict.get("params")
+ if params is None:
+ # Treat the excess keys in the environment as the destination parameters
+ # allowing a flat configuration of these things.
+ params = {}
+ for key, value in environment_dict.items():
+ if key in ['id', 'tags', 'runner', 'shell', 'env', 'resubmit']:
+ continue
+ params[key] = value
+ environment_dict["params"] = params
+
+ for key in ['tags', 'runner', 'shell', 'env', 'resubmit', 'params']:
+ if key in environment_dict:
+ destination_kwds[key] = environment_dict[key]
+ destination_kwds["id"] = environment_id
+ job_destination = JobDestination(**destination_kwds)
+ if not self.__is_enabled(job_destination.params):
continue
- job_destination['params'] = params
- job_destination['env'] = self.__get_envs(destination)
- destination_resubmits = self.__get_resubmits(destination)
- if destination_resubmits:
- resubmits = destination_resubmits
- else:
+ if not job_destination.resubmit:
resubmits = self.default_resubmits
- job_destination["resubmit"] = resubmits
+ job_destination.resubmit = resubmits
- self.destinations[id] = (job_destination,)
+ self.destinations[environment_id] = (job_destination,)
if job_destination.tags is not None:
for tag in job_destination.tags:
if tag not in self.destinations:
@@ -266,29 +415,30 @@ class JobConfiguration(ConfiguresHandlers):
self.destinations[tag].append(job_destination)
# Determine the default destination
- self.default_destination_id = self._get_default(
- self.app.config, destinations, list(self.destinations.keys()), auto=True)
+ self.default_destination_id = self._ensure_default_set(execution_dict.get("default"), list(self.destinations.keys()), auto=True)
- # Parse resources...
- resources = root.find('resources')
- if resources is not None:
- self.default_resource_group = resources.get("default", None)
- for group in self._findall_with_required(resources, 'group'):
- id = group.get('id')
- fields_str = group.get('fields', None) or group.text or ''
- fields = [f for f in fields_str.split(",") if f]
- self.resource_groups[id] = fields
+ # Read in resources
+ resources = job_config_dict.get("resources", {})
+ self.default_resource_group = resources.get("default", None)
+ for group_id, fields in resources.get("groups", {}).items():
+ self.resource_groups[group_id] = fields
- # Parse tool mappings
- tools = root.find('tools')
- if tools is not None:
- for tool in self._findall_with_required(tools, 'tool'):
- # There can be multiple definitions with identical ids, but different params
- id = tool.get('id').lower().rstrip('/')
- if id not in self.tools:
- self.tools[id] = list()
- self.tools[id].append(JobToolConfiguration(**dict(tool.items())))
- self.tools[id][-1]['params'] = self.__get_params(tool)
+ tools = job_config_dict.get('tools', [])
+ for tool in tools:
+ tool_id = tool.get('id').lower().rstrip('/')
+ if tool_id not in self.tools:
+ self.tools[tool_id] = list()
+ params = tool.get("params")
+ if params is None:
+ params = {}
+ for key, value in tool.items():
+ if key in ["environment", "handler", "id"]:
+ continue
+ params[key] = value
+ tool["params"] = params
+ if "environment" in tool:
+ tool["destination"] = tool.pop("environment")
+ self.tools[tool_id].append(JobToolConfiguration(**dict(tool.items())))
types = dict(registered_user_concurrent_jobs=int,
anonymous_user_concurrent_jobs=int,
@@ -297,26 +447,28 @@ class JobConfiguration(ConfiguresHandlers):
output_size=util.size_to_bytes)
# Parse job limits
- limits = root.find('limits')
- if limits is not None:
- for limit in self._findall_with_required(limits, 'limit', ('type',)):
- type = limit.get('type')
- # concurrent_jobs renamed to destination_user_concurrent_jobs in job_conf.xml
- if type in ('destination_user_concurrent_jobs', 'concurrent_jobs', 'destination_total_concurrent_jobs'):
- id = limit.get('tag', None) or limit.get('id')
- if type == 'destination_total_concurrent_jobs':
- self.limits.destination_total_concurrent_jobs[id] = int(limit.text)
- else:
- self.limits.destination_user_concurrent_jobs[id] = int(limit.text)
- elif type == 'total_walltime':
- self.limits.total_walltime["window"] = (
- int(limit.get('window')) or 30
- )
- self.limits.total_walltime["raw"] = (
- types.get(type, str)(limit.text)
- )
- elif limit.text:
- self.limits.__dict__[type] = types.get(type, str)(limit.text)
+ for limit_dict in job_config_dict.get("limits", []):
+ limit_type = limit_dict.get('type')
+ if limit_type.startswith("environment_"):
+ limit_type = 'destination_%s' % limit_type[len("environment_"):]
+
+ limit_value = limit_dict.get("value")
+ # concurrent_jobs renamed to destination_user_concurrent_jobs in job_conf.xml
+ if limit_type in ('destination_user_concurrent_jobs', 'concurrent_jobs', 'destination_total_concurrent_jobs'):
+ id = limit_dict.get('tag', None) or limit_dict.get('id')
+ if limit_type == 'destination_total_concurrent_jobs':
+ self.limits.destination_total_concurrent_jobs[id] = int(limit_value)
+ else:
+ self.limits.destination_user_concurrent_jobs[id] = int(limit_value)
+ elif limit_type == 'total_walltime':
+ self.limits.total_walltime["window"] = (
+ int(limit_dict.get('window')) or 30
+ )
+ self.limits.total_walltime["raw"] = (
+ types.get(limit_type, str)(limit_value)
+ )
+ elif limit_value:
+ self.limits.__dict__[limit_type] = types.get(limit_type, str)(limit_value)
if self.limits.walltime is not None:
h, m, s = [int(v) for v in self.limits.walltime.split(':')]
@@ -329,13 +481,23 @@ class JobConfiguration(ConfiguresHandlers):
0, s, 0, 0, m, h
)
- log.debug('Done loading job configuration')
+ def __parse_job_conf_xml(self, tree):
+ """Loads the new-style job configuration from options in the job config file (by default, job_conf.xml).
- def _parse_handler(self, handler_id, handler_element):
- for plugin in handler_element.findall('plugin'):
+ :param tree: Object representing the root ```` object in the job config file.
+ :type tree: ``xml.etree.ElementTree.Element``
+ """
+ root = tree.getroot()
+ log.debug('Loading job configuration from %s' % self.app.config.job_config_file)
+
+ job_config_dict = job_config_xml_to_dict(self.app.config, root)
+ return job_config_dict
+
+ def _parse_handler(self, handler_id, process_dict):
+ for plugin_id in process_dict.get("plugins") or []:
if handler_id not in self.handler_runner_plugins:
self.handler_runner_plugins[handler_id] = []
- self.handler_runner_plugins[handler_id].append(plugin.get('id'))
+ self.handler_runner_plugins[handler_id].append(plugin_id)
def __set_default_job_conf(self):
# Run jobs locally
@@ -383,14 +545,8 @@ class JobConfiguration(ConfiguresHandlers):
def __parse_resource_parameters(self):
self.resource_parameters = util.parse_resource_parameters(self.app.config.job_resource_params_file)
- def __get_params(self, parent):
- """Parses any child tags in to a dictionary suitable for persistence.
-
- :param parent: Parent element in which to find child tags.
- :type parent: ``xml.etree.ElementTree.Element``
-
- :returns: dict
- """
+ @staticmethod
+ def get_params(config, parent):
rval = {}
for param in parent.findall('param'):
key = param.get('id')
@@ -406,12 +562,23 @@ class JobConfiguration(ConfiguresHandlers):
param_value = os.environ.get(environ_var, param_value)
elif 'from_config' in param.attrib:
config_val = param.attrib['from_config']
- param_value = self.app.config.config_dict.get(config_val, param_value)
+ param_value = config.config_dict.get(config_val, param_value)
rval[key] = param_value
return rval
- def __get_envs(self, parent):
+ def __get_params(self, parent):
+ """Parses any child tags in to a dictionary suitable for persistence.
+
+ :param parent: Parent element in which to find child tags.
+ :type parent: ``xml.etree.ElementTree.Element``
+
+ :returns: dict
+ """
+ return JobConfiguration.get_params(self.app.config, parent)
+
+ @staticmethod
+ def get_envs(parent):
"""Parses any child tags in to a dictionary suitable for persistence.
:param parent: Parent element in which to find child tags.
@@ -430,7 +597,8 @@ class JobConfiguration(ConfiguresHandlers):
))
return rval
- def __get_resubmits(self, parent):
+ @staticmethod
+ def get_resubmits(parent):
"""Parses any child tags in to a dictionary suitable for persistence.
:param parent: Parent element in which to find child tags.
@@ -442,7 +610,7 @@ class JobConfiguration(ConfiguresHandlers):
for resubmit in parent.findall('resubmit'):
rval.append(dict(
condition=resubmit.get('condition'),
- destination=resubmit.get('destination'),
+ environment=resubmit.get('destination'),
handler=resubmit.get('handler'),
delay=resubmit.get('delay'),
))
@@ -451,7 +619,7 @@ class JobConfiguration(ConfiguresHandlers):
def __is_enabled(self, params):
"""Check for an enabled parameter - pop it out - and return as boolean."""
enabled = True
- if "enabled" in params:
+ if "enabled" in (params or {}):
raw_enabled = params.pop("enabled")
enabled = util.asbool(raw_enabled)
@@ -596,7 +764,7 @@ class JobConfiguration(ConfiguresHandlers):
log.warning("Job runner classes must be subclassed from BaseJobRunner, %s has bases: %s" % (id, runner_class.__bases__))
continue
try:
- rval[id] = runner_class(self.app, runner['workers'], **runner.get('kwds', {}))
+ rval[id] = runner_class(self.app, runner.get('workers', JobConfiguration.DEFAULT_NWORKERS), **runner.get('kwds', {}))
except TypeError:
log.exception("Job runner '%s:%s' has not been converted to a new-style runner or encountered TypeError on load",
module_name, class_name)
diff --git a/lib/galaxy/jobs/output_checker.py b/lib/galaxy/jobs/output_checker.py
index a5fb26ba0e5..d644389b13a 100644
--- a/lib/galaxy/jobs/output_checker.py
+++ b/lib/galaxy/jobs/output_checker.py
@@ -146,10 +146,9 @@ def check_output(stdio_regexes, stdio_exit_codes, stdout, stderr, tool_exit_code
if stderr:
state = DETECTED_JOB_STATE.GENERIC_ERROR
- if DETECTED_JOB_STATE != DETECTED_JOB_STATE.OK and stderr:
- if stderr:
- peak = stderr[0:ERROR_PEAK]
- log.debug("job failed, standard error is - [%s]" % peak)
+ if state != DETECTED_JOB_STATE.OK:
+ peak = stderr[0:ERROR_PEAK] if stderr else ""
+ log.debug("job failed, detected state %s, standard error is - [%s]" % (state, peak))
except Exception:
log.exception("Job state check encountered unexpected exception; assuming execution successful")
diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py
index 0e0f5c89b80..38c4634b4c5 100644
--- a/lib/galaxy/jobs/runners/pulsar.py
+++ b/lib/galaxy/jobs/runners/pulsar.py
@@ -12,6 +12,7 @@ from time import sleep
import packaging.version
import pulsar.core
+import six
import yaml
from pulsar.client import (
build_client_manager,
@@ -449,7 +450,7 @@ class PulsarJobRunner(AsynchronousJobRunner):
user = job_wrapper.get_job().user
if user:
for key, value in params.items():
- if value:
+ if value and isinstance(value, six.string_types):
params[key] = model.User.expand_user_properties(user, value)
env = getattr(job_wrapper.job_destination, "env", [])
diff --git a/lib/galaxy/jobs/runners/slurm.py b/lib/galaxy/jobs/runners/slurm.py
index 333fd55d563..ec2ce131d45 100644
--- a/lib/galaxy/jobs/runners/slurm.py
+++ b/lib/galaxy/jobs/runners/slurm.py
@@ -28,13 +28,9 @@ SLURM_MEMORY_LIMIT_EXCEEDED_MSG = 'slurmstepd: error: Exceeded job memory limit'
SLURM_MEMORY_LIMIT_EXCEEDED_PARTIAL_WARNINGS = [': Exceeded job memory limit at some point.',
': Exceeded step memory limit at some point.']
SLURM_MEMORY_LIMIT_SCAN_SIZE = 16 * 1024 * 1024 # 16MB
-SLURM_UNABLE_TO_ADD_TASK_TO_MEMORY_CG_MSG_RE = re.compile(r"""slurmstepd: error: task/cgroup: unable to add task\[pid=\d+\] to memory cg '\(null\)'$""")
-SLURM_UNABLE_TO_CREATE_CGROUP_MSG_RE = re.compile(r"""slurmstepd: error: xcgroup_instantiate: unable to create cgroup '[^']+' : No space left on device$""")
-SLURM_UNABLE_TO_INSTANCIATE_CGROUP_MSG_RE = re.compile(r"""slurmstepd: error: jobacct_gather/cgroup: unable to instanciate (job|user) \d+ memory cgroup$""")
+SLURM_CGROUP_RE = re.compile(r"""slurmstepd: .*cgroup.*$""")
SLURM_TOP_WARNING_RES = (
- SLURM_UNABLE_TO_ADD_TASK_TO_MEMORY_CG_MSG_RE,
- SLURM_UNABLE_TO_CREATE_CGROUP_MSG_RE,
- SLURM_UNABLE_TO_INSTANCIATE_CGROUP_MSG_RE
+ SLURM_CGROUP_RE,
)
# These messages are returned to the user
diff --git a/lib/galaxy/jobs/runners/state_handlers/resubmit.py b/lib/galaxy/jobs/runners/state_handlers/resubmit.py
index f9cbd744f7c..69f9761e4d6 100644
--- a/lib/galaxy/jobs/runners/state_handlers/resubmit.py
+++ b/lib/galaxy/jobs/runners/state_handlers/resubmit.py
@@ -97,7 +97,8 @@ def _handle_resubmit_definitions(resubmit_definitions, app, job_runner, job_stat
else:
job_log_prefix = "(%s)" % (job_state.job_wrapper.job_id)
- destination = resubmit['destination']
+ # Is destination needed here, might these be serialized to the database?
+ destination = resubmit.get('environment') or resubmit.get('destination')
log.info("%s Job will be resubmitted to '%s' because %s at "
"the '%s' destination",
job_log_prefix,
diff --git a/lib/galaxy/managers/citations.py b/lib/galaxy/managers/citations.py
index 8eba445c341..bdae2df866e 100644
--- a/lib/galaxy/managers/citations.py
+++ b/lib/galaxy/managers/citations.py
@@ -1,6 +1,5 @@
import functools
import logging
-import os
import requests
from beaker.cache import CacheManager
@@ -26,8 +25,8 @@ class CitationsManager(object):
citation_collection.add(citation)
return citation_collection.citations
- def parse_citation(self, citation_elem, tool_directory):
- return parse_citation(citation_elem, tool_directory, self)
+ def parse_citation(self, citation_elem):
+ return parse_citation(citation_elem, self)
def _get_tool(self, tool_id):
tool = self.app.toolbox.get_tool(tool_id)
@@ -55,10 +54,9 @@ class DoiCache(object):
return self._cache.get(key=doi, createfunc=createfunc)
-def parse_citation(elem, directory, citation_manager):
- """ Parse an abstract citation entry from the specified XML element.
- The directory parameter should be used to find external files for this
- citation.
+def parse_citation(elem, citation_manager):
+ """
+ Parse an abstract citation entry from the specified XML element.
"""
citation_type = elem.attrib.get('type', None)
citation_class = CITATION_CLASSES.get(citation_type, None)
@@ -66,7 +64,7 @@ def parse_citation(elem, directory, citation_manager):
log.warning("Unknown or unspecified citation type: %s" % citation_type)
return None
try:
- citation = citation_class(elem, directory, citation_manager)
+ citation = citation_class(elem, citation_manager)
except Exception as e:
raise Exception("Invalid citation of type '%s' with content '%s': %s" % (citation_type, elem.text, e))
return citation
@@ -118,16 +116,8 @@ class BaseCitation(object):
class BibtexCitation(BaseCitation):
- def __init__(self, elem, directory, citation_manager):
- bibtex_file = elem.attrib.get("file", None)
- if bibtex_file:
- raw_bibtex = open(os.path.join(directory, bibtex_file), "r").read()
- else:
- raw_bibtex = elem.text.strip()
- self._set_raw_bibtex(raw_bibtex)
-
- def _set_raw_bibtex(self, raw_bibtex):
- self.raw_bibtex = raw_bibtex
+ def __init__(self, elem, citation_manager):
+ self.raw_bibtex = elem.text.strip()
def to_bibtex(self):
return self.raw_bibtex
@@ -136,7 +126,7 @@ class BibtexCitation(BaseCitation):
class DoiCitation(BaseCitation):
BIBTEX_UNSET = object()
- def __init__(self, elem, directory, citation_manager):
+ def __init__(self, elem, citation_manager):
self.__doi = elem.text.strip()
self.doi_cache = citation_manager.doi_cache
self.raw_bibtex = DoiCitation.BIBTEX_UNSET
diff --git a/lib/galaxy/managers/hdas.py b/lib/galaxy/managers/hdas.py
index 1842665ea78..08bd08e505f 100644
--- a/lib/galaxy/managers/hdas.py
+++ b/lib/galaxy/managers/hdas.py
@@ -300,7 +300,8 @@ class HDASerializer( # datasets._UnflattenedMetadataDatasetAssociationSerialize
'annotation',
- 'api_type'
+ 'api_type',
+ 'created_from_basename',
], include_keys_from='summary')
self.add_view('extended', [
@@ -354,7 +355,8 @@ class HDASerializer( # datasets._UnflattenedMetadataDatasetAssociationSerialize
# TODO: to DatasetAssociationSerializer
'accessible' : lambda i, k, user=None, **c: self.manager.is_accessible(i, user, **c),
'api_type' : lambda *a, **c: 'file',
- 'type' : lambda *a, **c: 'file'
+ 'type' : lambda *a, **c: 'file',
+ 'created_from_basename' : lambda i, k, **c: i.created_from_basename,
})
def serialize(self, hda, keys, user=None, **context):
diff --git a/lib/galaxy/metadata/__init__.py b/lib/galaxy/metadata/__init__.py
index 5314c6dc3dd..e7351771522 100644
--- a/lib/galaxy/metadata/__init__.py
+++ b/lib/galaxy/metadata/__init__.py
@@ -143,7 +143,7 @@ class PortableDirectoryMetadataGenerator(MetadataCollectionStrategy):
script_path = os.path.join(metadata_dir, "set.py")
with open(script_path, "w") as f:
f.write(SET_METADATA_SCRIPT)
- return 'pwd; python "metadata/set.py"'
+ return 'python "metadata/set.py"'
else:
# return args to galaxy_ext.metadata.set_metadata required to build
return ''
diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index faebe5498d2..bb8db8a0292 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -42,6 +42,7 @@ from sqlalchemy.orm import (
)
from sqlalchemy.schema import UniqueConstraint
+import galaxy.exceptions
import galaxy.model.metadata
import galaxy.model.orm.now
import galaxy.model.tags
@@ -2156,8 +2157,10 @@ class Dataset(StorableObject, RepresentById):
def get_file_name(self):
if not self.external_filename:
assert self.object_store is not None, "Object Store has not been initialized for dataset %s" % self.id
- filename = self.object_store.get_filename(self)
- return filename
+ if self.object_store.exists(self):
+ return self.object_store.get_filename(self)
+ else:
+ return ''
else:
filename = self.external_filename
# Make filename absolute
@@ -2175,10 +2178,16 @@ class Dataset(StorableObject, RepresentById):
# actual database column so if SA instantiates this object - the
# attribute won't exist yet.
if not getattr(self, "external_extra_files_path", None):
- return self.object_store.get_filename(self, dir_only=True, extra_dir=self._extra_files_rel_path)
+ if self.object_store.exists(self, dir_only=True, extra_dir=self._extra_files_rel_path):
+ return self.object_store.get_filename(self, dir_only=True, extra_dir=self._extra_files_rel_path)
+ return ''
else:
return os.path.abspath(self.external_extra_files_path)
+ def create_extra_files_path(self):
+ if not self.extra_files_path_exists():
+ self.object_store.create(self, dir_only=True, extra_dir=self._extra_files_rel_path)
+
def set_extra_files_path(self, extra_files_path):
if not extra_files_path:
self.external_extra_files_path = None
@@ -2266,11 +2275,12 @@ class Dataset(StorableObject, RepresentById):
def full_delete(self):
"""Remove the file and extra files, marks deleted and purged"""
# os.unlink( self.file_name )
- self.object_store.delete(self)
+ try:
+ self.object_store.delete(self)
+ except galaxy.exceptions.ObjectNotFound:
+ pass
if self.object_store.exists(self, extra_dir=self._extra_files_rel_path, dir_only=True):
self.object_store.delete(self, entire_dir=True, extra_dir=self._extra_files_rel_path, dir_only=True)
- # if os.path.exists( self.extra_files_path ):
- # shutil.rmtree( self.extra_files_path )
# TODO: purge metadata files
self.deleted = True
self.purged = True
@@ -2510,6 +2520,16 @@ class DatasetInstance(object):
"""Detects whether there is any data"""
return self.dataset.has_data()
+ def get_created_from_basename(self):
+ return self.dataset.created_from_basename
+
+ def set_created_from_basename(self, created_from_basename):
+ if self.dataset.created_from_basename is not None:
+ raise Exception("Underlying dataset already has a created_from_basename set.")
+ self.dataset.created_from_basename = created_from_basename
+
+ created_from_basename = property(get_created_from_basename, set_created_from_basename)
+
def get_raw_data(self):
"""Returns the full data. To stream it open the file_name and read/write as needed"""
return self.datatype.get_raw_data(self)
@@ -3393,6 +3413,7 @@ class LibraryDataset(RepresentById):
state=ldda.state,
name=ldda.name,
file_name=ldda.file_name,
+ created_from_basename=ldda.created_from_basename,
uploaded_by=ldda.user.email,
message=ldda.message,
date_uploaded=ldda.create_time.isoformat(),
@@ -3545,7 +3566,8 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, RepresentById):
data_type=ldda.datatype.__class__.__module__ + '.' + ldda.datatype.__class__.__name__,
genome_build=ldda.dbkey,
misc_info=ldda.info,
- misc_blurb=ldda.blurb)
+ misc_blurb=ldda.blurb,
+ created_from_basename=ldda.created_from_basename)
if ldda.dataset.uuid is None:
rval['uuid'] = None
else:
diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py
index f7f563c54c8..25f91256140 100644
--- a/lib/galaxy/model/mapping.py
+++ b/lib/galaxy/model/mapping.py
@@ -263,6 +263,7 @@ model.Dataset.table = Table(
Column("object_store_id", TrimmedString(255), index=True),
Column("external_filename", TEXT),
Column("_extra_files_path", TEXT),
+ Column("created_from_basename", TEXT),
Column('file_size', Numeric(15, 0)),
Column('total_size', Numeric(15, 0)),
Column('uuid', UUIDType()))
diff --git a/lib/galaxy/model/migrate/versions/0152_add_metadata_file_uuid.py b/lib/galaxy/model/migrate/versions/0152_add_metadata_file_uuid.py
index f9d26724a37..0f25287393c 100644
--- a/lib/galaxy/model/migrate/versions/0152_add_metadata_file_uuid.py
+++ b/lib/galaxy/model/migrate/versions/0152_add_metadata_file_uuid.py
@@ -6,9 +6,10 @@ from __future__ import print_function
import logging
-from sqlalchemy import Column, MetaData, Table
+from sqlalchemy import Column, MetaData
from galaxy.model.custom_types import UUIDType
+from galaxy.model.migrate.versions.util import add_column, drop_column
log = logging.getLogger(__name__)
@@ -19,14 +20,8 @@ def upgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
- metadata_file_table = Table("metadata_file", metadata, autoload=True)
-
- try:
- uuid_column = Column('uuid', UUIDType())
- uuid_column.create(metadata_file_table)
- assert uuid_column is metadata_file_table.c.uuid
- except Exception:
- log.exception("Adding column 'uuid' to `MetadataFile` table failed.")
+ uuid_column = Column('uuid', UUIDType())
+ add_column(uuid_column, 'metadata_file', metadata)
def downgrade(migrate_engine):
@@ -34,10 +29,4 @@ def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
- metadata_file_table = Table("metadata_file", metadata, autoload=True)
-
- try:
- column = metadata_file_table.c.uuid
- column.drop()
- except Exception:
- log.exception("Dropping 'uuid' column from `metadata_file` table failed.")
+ drop_column('uuid', 'metadata_file', metadata)
diff --git a/lib/galaxy/model/migrate/versions/0154_created_from_basename.py b/lib/galaxy/model/migrate/versions/0154_created_from_basename.py
new file mode 100644
index 00000000000..c4664b48e97
--- /dev/null
+++ b/lib/galaxy/model/migrate/versions/0154_created_from_basename.py
@@ -0,0 +1,29 @@
+"""
+Adds created_from_basename to dataset.
+"""
+import datetime
+import logging
+
+from sqlalchemy import Column, MetaData, TEXT
+
+from galaxy.model.migrate.versions.util import add_column, drop_column
+
+now = datetime.datetime.utcnow
+log = logging.getLogger(__name__)
+metadata = MetaData()
+
+
+def upgrade(migrate_engine):
+ metadata.bind = migrate_engine
+ print(__doc__)
+ metadata.reflect()
+
+ created_from_basename_column = Column("created_from_basename", TEXT, default=None)
+ add_column(created_from_basename_column, 'dataset', metadata)
+
+
+def downgrade(migrate_engine):
+ metadata.bind = migrate_engine
+ metadata.reflect()
+
+ drop_column('created_from_basename', 'dataset', metadata)
diff --git a/lib/galaxy/model/store/discover.py b/lib/galaxy/model/store/discover.py
index 697aefc15b0..2f5964d4ed9 100644
--- a/lib/galaxy/model/store/discover.py
+++ b/lib/galaxy/model/store/discover.py
@@ -55,6 +55,7 @@ class ModelPersistenceContext(object):
tag_list=[],
sources=[],
hashes=[],
+ created_from_basename=None,
):
sa_session = self.sa_session
@@ -116,6 +117,9 @@ class ModelPersistenceContext(object):
hash_object.hash_value = hash_dict["hash_value"]
primary_data.dataset.hashes.append(hash_object)
+ if created_from_basename is not None:
+ primary_data.created_from_basename = created_from_basename
+
self.flush()
if tag_list:
@@ -204,6 +208,7 @@ class ModelPersistenceContext(object):
sources = discovered_file.match.sources
hashes = discovered_file.match.hashes
+ created_from_basename = discovered_file.match.created_from_basename
dataset = self.create_dataset(
ext=ext,
@@ -217,6 +222,7 @@ class ModelPersistenceContext(object):
tag_list=tag_list,
sources=sources,
hashes=hashes,
+ created_from_basename=created_from_basename,
)
log.debug(
"(%s) Created dynamic collection dataset for path [%s] with element identifier [%s] for output [%s] %s",
@@ -475,6 +481,8 @@ def persist_elements_to_folder(model_persistence_context, elements, library_fold
sources = fields_match.sources
hashes = fields_match.hashes
+ created_from_basename = fields_match.created_from_basename
+
model_persistence_context.create_dataset(
ext=ext,
designation=designation,
@@ -487,6 +495,7 @@ def persist_elements_to_folder(model_persistence_context, elements, library_fold
link_data=link_data,
sources=sources,
hashes=hashes,
+ created_from_basename=created_from_basename,
)
@@ -517,6 +526,8 @@ def persist_hdas(elements, model_persistence_context):
sources = fields_match.sources
hashes = fields_match.hashes
+ created_from_basename = fields_match.created_from_basename
+
dataset = model_persistence_context.create_dataset(
ext=ext,
designation=designation,
@@ -529,6 +540,7 @@ def persist_hdas(elements, model_persistence_context):
primary_data=primary_dataset,
sources=sources,
hashes=hashes,
+ created_from_basename=created_from_basename,
)
dataset.raw_set_dataset_state('ok')
if not hda_id:
@@ -701,6 +713,10 @@ class JsonCollectedDatasetMatch(object):
def hashes(self):
return self.as_dict.get("hashes", [])
+ @property
+ def created_from_basename(self):
+ return self.as_dict.get("created_from_basename")
+
class RegexCollectedDatasetMatch(JsonCollectedDatasetMatch):
diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py
index b73a4a8ad56..6294ff77832 100644
--- a/lib/galaxy/objectstore/__init__.py
+++ b/lib/galaxy/objectstore/__init__.py
@@ -467,7 +467,10 @@ class DiskObjectStore(ObjectStore):
# construct and return hashed path
if os.path.exists(path):
return path
- return self._construct_path(obj, **kwargs)
+ path = self._construct_path(obj, **kwargs)
+ if not os.path.exists(path):
+ raise ObjectNotFound
+ return path
def update_from_file(self, obj, file_name=None, create=False, **kwargs):
"""`create` parameter is not used in this implementation."""
diff --git a/lib/galaxy/objectstore/s3_multipart_upload.py b/lib/galaxy/objectstore/s3_multipart_upload.py
index f564c1f8524..64b15ce76f9 100644
--- a/lib/galaxy/objectstore/s3_multipart_upload.py
+++ b/lib/galaxy/objectstore/s3_multipart_upload.py
@@ -5,13 +5,10 @@ This parallelizes the task over available cores using multiprocessing.
Code mostly taken form CloudBioLinux.
"""
-import contextlib
-import functools
import glob
-import multiprocessing
import os
import subprocess
-from multiprocessing.pool import IMapIterator
+import threading
try:
import boto
@@ -20,13 +17,6 @@ except ImportError:
boto = None
-def map_wrap(f):
- @functools.wraps(f)
- def wrapper(args):
- return f(*args)
- return wrapper
-
-
def mp_from_ids(s3server, mp_id, mp_keyname, mp_bucketname):
"""Get the multipart upload from the bucket and multipart IDs.
@@ -51,7 +41,6 @@ def mp_from_ids(s3server, mp_id, mp_keyname, mp_bucketname):
return mp
-@map_wrap
def transfer_part(s3server, mp_id, mp_keyname, mp_bucketname, i, part):
"""Transfer a part of a multipart upload. Designed to be run in parallel.
"""
@@ -64,8 +53,6 @@ def transfer_part(s3server, mp_id, mp_keyname, mp_bucketname, i, part):
def multipart_upload(s3server, bucket, s3_key_name, tarball, mb_size):
"""Upload large files using Amazon's multipart upload functionality.
"""
- cores = multiprocessing.cpu_count()
-
def split_file(in_file, mb_size, split_num=5):
prefix = os.path.join(os.path.dirname(in_file),
"%sS3PART" % (os.path.basename(s3_key_name)))
@@ -80,29 +67,11 @@ def multipart_upload(s3server, bucket, s3_key_name, tarball, mb_size):
mp = bucket.initiate_multipart_upload(s3_key_name,
reduced_redundancy=s3server['use_rr'])
- with multimap(cores) as pmap:
- for _ in pmap(transfer_part, ((s3server, mp.id, mp.key_name, mp.bucket_name, i, part)
- for (i, part) in
- enumerate(split_file(tarball, mb_size, cores)))):
- pass
+ for (i, part) in enumerate(split_file(tarball, mb_size)):
+ t = threading.Thread(
+ target=transfer_part,
+ args=(s3server, mp.id, mp.key_name, mp.bucket_name, i, part))
+ t.start()
+ t.join()
+
mp.complete_upload()
-
-
-@contextlib.contextmanager
-def multimap(cores=None):
- """Provide multiprocessing imap like function.
-
- The context manager handles setting up the pool, worked around interrupt issues
- and terminating the pool on completion.
- """
- if cores is None:
- cores = max(multiprocessing.cpu_count() - 1, 1)
-
- def wrapper(func):
- def wrap(self, timeout=None):
- return func(self, timeout=timeout if timeout is not None else 1e100)
- return wrap
- IMapIterator.next = wrapper(IMapIterator.next)
- pool = multiprocessing.Pool(cores)
- yield pool.imap
- pool.terminate()
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index c1f7c980723..f6954e0efcf 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -464,6 +464,7 @@ class Tool(Dictifiable):
self.guid = guid
self.old_id = None
self.version = None
+ self.python_template_version = None
self._lineage = None
self.dependencies = []
# populate toolshed repository info, if available
@@ -643,6 +644,14 @@ class Tool(Dictifiable):
message = template % (self.id, self.profile)
raise Exception(message)
+ self.python_template_version = tool_source.parse_python_template_version()
+ if self.python_template_version is None:
+ # If python_template_version not specified we assume tools with profile versions >= 19.05 are python 3 ready
+ if self.profile >= 19.05:
+ self.python_template_version = '3.5'
+ else:
+ self.python_template_version = '2.7'
+
# Get the (user visible) name of the tool
self.name = tool_source.parse_name()
if not self.name and dynamic:
@@ -1065,7 +1074,7 @@ class Tool(Dictifiable):
if citation_elem.tag != "citation":
pass
if hasattr(self.app, 'citations_manager'):
- citation = self.app.citations_manager.parse_citation(citation_elem, self.tool_dir)
+ citation = self.app.citations_manager.parse_citation(citation_elem)
if citation:
citations.append(citation)
return citations
diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py
index c5f6ce5f31d..8ff721b6d28 100644
--- a/lib/galaxy/tools/actions/__init__.py
+++ b/lib/galaxy/tools/actions/__init__.py
@@ -1,5 +1,6 @@
import json
import logging
+import os
import re
from json import dumps
@@ -377,7 +378,8 @@ class DefaultToolAction(object):
wrapped_params.params,
inp_data,
inp_dataset_collections,
- input_ext
+ input_ext,
+ python_template_version=tool.python_template_version,
)
create_datasets = True
dataset = None
@@ -391,6 +393,10 @@ class DefaultToolAction(object):
break
data = app.model.HistoryDatasetAssociation(extension=ext, dataset=dataset, create_dataset=create_datasets, flush=False)
+ if create_datasets:
+ from_work_dir = output.from_work_dir
+ if from_work_dir is not None:
+ data.dataset.created_from_basename = os.path.basename(from_work_dir)
if hidden is None:
hidden = output.hidden
if not hidden and dataset_collection_elements is not None: # Mapping over a collection - hide datasets
@@ -744,7 +750,7 @@ class DefaultToolAction(object):
if output.label:
params['tool'] = tool
params['on_string'] = on_text
- return fill_template(output.label, context=params)
+ return fill_template(output.label, context=params, python_template_version=tool.python_template_version)
else:
return self._get_default_data_name(dataset, tool, on_text=on_text, trans=trans, incoming=incoming, history=history, params=params, job_params=job_params)
@@ -761,7 +767,7 @@ class DefaultToolAction(object):
if output.actions:
for action in output.actions.actions:
if action.tag == "metadata" and action.default:
- metadata_new_value = fill_template(action.default, context=params).split(",")
+ metadata_new_value = fill_template(action.default, context=params, python_template_version=tool.python_template_version).split(",")
dataset.metadata.__setattr__(str(action.name), metadata_new_value)
def _get_default_data_name(self, dataset, tool, on_text=None, trans=None, incoming=None, history=None, params=None, job_params=None, **kwd):
@@ -924,7 +930,7 @@ def get_ext_or_implicit_ext(hda):
return hda.ext
-def determine_output_format(output, parameter_context, input_datasets, input_dataset_collections, random_input_ext):
+def determine_output_format(output, parameter_context, input_datasets, input_dataset_collections, random_input_ext, python_template_version='3'):
""" Determines the output format for a dataset based on an abstract
description of the output (galaxy.tools.parser.ToolOutput), the parameter
wrappers, a map of the input datasets (name => HDA), and the last input
@@ -986,7 +992,7 @@ def determine_output_format(output, parameter_context, input_datasets, input_dat
if '$' not in check:
# allow a simple name or more complex specifications
check = '${%s}' % check
- if str(fill_template(check, context=parameter_context)) == when_elem.get('value', None):
+ if fill_template(check, context=parameter_context, python_template_version=python_template_version) == when_elem.get('value', None):
ext = when_elem.get('format', ext)
except Exception: # bad tag input value; possibly referencing a param within a different conditional when block or other nonexistent grouping construct
continue
diff --git a/lib/galaxy/tools/data_fetch.py b/lib/galaxy/tools/data_fetch.py
index 75eaa204d48..90b35f64c69 100644
--- a/lib/galaxy/tools/data_fetch.py
+++ b/lib/galaxy/tools/data_fetch.py
@@ -108,6 +108,7 @@ def _fetch_target(upload_config, target):
dbkey = item.get("dbkey", "?")
requested_ext = item.get("ext", "auto")
info = item.get("info", None)
+ created_from_basename = item.get("created_from_basename", None)
tags = item.get("tags", [])
object_id = item.get("object_id", None)
link_data_only = upload_config.link_data_only
@@ -162,6 +163,8 @@ def _fetch_target(upload_config, target):
rval["object_id"] = object_id
if tags:
rval["tags"] = tags
+ if created_from_basename:
+ rval["created_from_basename"] = created_from_basename
return rval
elements = elements_tree_map(_resolve_src, items)
diff --git a/lib/galaxy/tools/deps/containers.py b/lib/galaxy/tools/deps/containers.py
index 5a8234f84db..35229b4e631 100644
--- a/lib/galaxy/tools/deps/containers.py
+++ b/lib/galaxy/tools/deps/containers.py
@@ -206,7 +206,9 @@ class ContainerRegistry(object):
if self.enable_beta_mulled_containers:
default_resolvers.extend([
CachedMulledDockerContainerResolver(self.app_info, namespace="biocontainers"),
+ CachedMulledDockerContainerResolver(self.app_info, namespace="local"),
CachedMulledSingularityContainerResolver(self.app_info, namespace="biocontainers"),
+ CachedMulledSingularityContainerResolver(self.app_info, namespace="local"),
MulledDockerContainerResolver(self.app_info, namespace="biocontainers"),
MulledSingularityContainerResolver(self.app_info, namespace="biocontainers"),
BuildMulledDockerContainerResolver(self.app_info),
diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py
index 1b741b7dc97..162f0c906d1 100644
--- a/lib/galaxy/tools/evaluation.py
+++ b/lib/galaxy/tools/evaluation.py
@@ -469,7 +469,7 @@ class ToolEvaluator(object):
return
try:
# Substituting parameters into the command
- command_line = fill_template(command, context=param_dict)
+ command_line = fill_template(command, context=param_dict, python_template_version=self.tool.python_template_version)
cleaned_command_line = []
# Remove leading and trailing whitespace from each line for readability.
for line in command_line.split('\n'):
@@ -575,7 +575,7 @@ class ToolEvaluator(object):
def __write_workdir_file(self, config_filename, content, context, is_template=True):
if is_template:
- value = fill_template(content, context=context)
+ value = fill_template(content, context=context, python_template_version=self.tool.python_template_version)
else:
value = unicodify(content)
with io.open(config_filename, "w", encoding='utf-8') as f:
diff --git a/lib/galaxy/tools/parameters/output_collect.py b/lib/galaxy/tools/parameters/output_collect.py
index caca15031cd..81c10a8d22d 100644
--- a/lib/galaxy/tools/parameters/output_collect.py
+++ b/lib/galaxy/tools/parameters/output_collect.py
@@ -316,6 +316,7 @@ def collect_primary_datasets(job_context, output, input_ext):
extra_files_path = new_primary_datasets_attributes.get('extra_files', None)
if extra_files_path:
extra_files_path_joined = os.path.join(job_working_directory, extra_files_path)
+ primary_data.dataset.create_extra_files_path()
for root, dirs, files in os.walk(extra_files_path_joined):
extra_dir = os.path.join(primary_data.extra_files_path, root.replace(extra_files_path_joined, '', 1).lstrip(os.path.sep))
extra_dir = os.path.normpath(extra_dir)
diff --git a/lib/galaxy/tools/parser/cwl.py b/lib/galaxy/tools/parser/cwl.py
index 576023848b5..edf744acbb1 100644
--- a/lib/galaxy/tools/parser/cwl.py
+++ b/lib/galaxy/tools/parser/cwl.py
@@ -150,6 +150,9 @@ class CwlToolSource(ToolSource):
def parse_profile(self):
return "16.04"
+ def parse_python_template_version(self):
+ return '3.5'
+
class CwlPageSource(PageSource):
diff --git a/lib/galaxy/tools/parser/interface.py b/lib/galaxy/tools/parser/interface.py
index aa6ade42b69..f63e51ce8e5 100644
--- a/lib/galaxy/tools/parser/interface.py
+++ b/lib/galaxy/tools/parser/interface.py
@@ -218,6 +218,12 @@ class ToolSource(object):
""" Return tool profile version as Galaxy major e.g. 16.01 or 16.04.
"""
+ @abstractmethod
+ def parse_python_template_version(self):
+ """
+ Return minimum python version that the tool template has been developed against.
+ """
+
def macro_paths(self):
return []
diff --git a/lib/galaxy/tools/parser/xml.py b/lib/galaxy/tools/parser/xml.py
index 1f46c406dc9..6c7f32af29c 100644
--- a/lib/galaxy/tools/parser/xml.py
+++ b/lib/galaxy/tools/parser/xml.py
@@ -5,6 +5,8 @@ import traceback
import uuid
from math import isinf
+import packaging.version
+
from galaxy.tools.deps import requirements
from galaxy.util import string_as_bool, xml_text, xml_to_string
from galaxy.util.odict import odict
@@ -483,6 +485,12 @@ class XmlToolSource(ToolSource):
# - Enable buggy interpreter attribute.
return self.root.get("profile", "16.01")
+ def parse_python_template_version(self):
+ python_template_version = self.root.get("python_template_version", None)
+ if python_template_version is not None:
+ python_template_version = packaging.version.parse(python_template_version)
+ return python_template_version
+
def _test_elem_to_dict(test_elem, i):
rval = dict(
diff --git a/lib/galaxy/tools/parser/yaml.py b/lib/galaxy/tools/parser/yaml.py
index 65ce7747644..88500de805c 100644
--- a/lib/galaxy/tools/parser/yaml.py
+++ b/lib/galaxy/tools/parser/yaml.py
@@ -1,3 +1,5 @@
+import packaging.version
+
from galaxy.tools.deps import requirements
from galaxy.util.odict import odict
from .interface import InputSource
@@ -162,6 +164,12 @@ class YamlToolSource(ToolSource):
def parse_profile(self):
return self.root_dict.get("profile", "16.04")
+ def parse_python_template_version(self):
+ python_template_version = self.root_dict.get("python_template_version", None)
+ if python_template_version is not None:
+ python_template_version = packaging.version.parse(python_template_version)
+ return python_template_version
+
def _parse_test(i, test_dict):
inputs = test_dict["inputs"]
diff --git a/lib/galaxy/tools/verify/asserts/hdf5.py b/lib/galaxy/tools/verify/asserts/hdf5.py
index f64314fe1ea..c4aae057c9e 100644
--- a/lib/galaxy/tools/verify/asserts/hdf5.py
+++ b/lib/galaxy/tools/verify/asserts/hdf5.py
@@ -24,10 +24,19 @@ def assert_has_h5_attribute(output_bytes, key, value):
def assert_has_h5_keys(output_bytes, keys):
- """ Asserts the specified HDF5 output has exactly the given keys."""
+ """ Asserts the specified HDF5 output has the given keys."""
_assert_h5py()
keys = [k.strip() for k in keys.strip().split(',')]
h5_keys = sorted(keys)
output_temp = io.BytesIO(output_bytes)
- local_keys = sorted(list(h5py.File(output_temp, 'r').keys()))
- assert local_keys == h5_keys, "Not a HDF5 file or H5 keys do not match:\n\t%s\n\t%s" % (local_keys, h5_keys)
+ local_keys = []
+
+ def append_keys(key):
+ local_keys.append(key)
+ return None
+ h5py.File(output_temp, 'r').visit(append_keys)
+ missing = 0
+ for key in h5_keys:
+ if key not in local_keys:
+ missing += 1
+ assert missing == 0, "Not a HDF5 file or H5 keys missing:\n\t%s\n\t%s" % (local_keys, h5_keys)
diff --git a/lib/galaxy/tools/verify/interactor.py b/lib/galaxy/tools/verify/interactor.py
index 0898b7c7c39..f330656c801 100644
--- a/lib/galaxy/tools/verify/interactor.py
+++ b/lib/galaxy/tools/verify/interactor.py
@@ -194,7 +194,7 @@ class GalaxyInteractorApi(object):
"""
metadata = attributes.get('metadata', {}).copy()
for key, value in metadata.copy().items():
- if key not in ['name', 'info', 'tags']:
+ if key not in ['name', 'info', 'tags', 'created_from_basename']:
new_key = "metadata_%s" % key
metadata[new_key] = metadata[key]
del metadata[key]
diff --git a/lib/galaxy/tools/wrappers.py b/lib/galaxy/tools/wrappers.py
index 474a54b60f5..8e23cb72585 100644
--- a/lib/galaxy/tools/wrappers.py
+++ b/lib/galaxy/tools/wrappers.py
@@ -322,7 +322,7 @@ class HasDatasets(object):
def _dataset_wrapper(self, dataset, dataset_paths, **kwargs):
wrapper_kwds = kwargs.copy()
- if dataset:
+ if dataset and dataset_paths:
real_path = dataset.file_name
if real_path in dataset_paths:
wrapper_kwds["dataset_path"] = dataset_paths[real_path]
diff --git a/lib/galaxy/tools/xsd/galaxy.xsd b/lib/galaxy/tools/xsd/galaxy.xsd
index 86eb7bbec48..a9c208e62ca 100644
--- a/lib/galaxy/tools/xsd/galaxy.xsd
+++ b/lib/galaxy/tools/xsd/galaxy.xsd
@@ -175,13 +175,22 @@ set this to have values such as ``data_source`` or ``manage_data``.
- This string specified the minimum Galaxy
+ This string specifies the minimum Galaxy
version that should be required to run this tool. Certain legacy behaviors such
as using standard error content to detect errors instead of exit code are disabled
automatically if profile is set to any version newer than ``16.01``, such as
``16.04``.
+
+
+ This string specifies the minimum Python
+version that is able to fill the Cheetah sections of the tool. If unset defaults
+to 2.7 if the profile is older than 19.01, otherwise defaults to 3.5. Galaxy will
+attempt to convert Python statements in Cheetah sections using [future](http://python-future.org/)
+if Galaxy is run on Python 3 and ``python_template_version`` is below 3.
+
+ This attribute indicates if
@@ -1660,7 +1669,7 @@ module.
- ``).]]>
+ ``).]]>
@@ -2098,7 +2107,7 @@ parameter being described. All the attributes for the ``param`` element are
documented below for completeness, but here are the common ones for each
type are as follows:
-$attribute_list:name,type,optional,label,help,argument,load_contents:refresh_on_change:4
+$attribute_list:name,type,optional,label,help,argument,load_contents,refresh_on_change:4
### Parameter Types
diff --git a/lib/galaxy/util/template.py b/lib/galaxy/util/template.py
index 1fb97f6ed59..c5d22066664 100644
--- a/lib/galaxy/util/template.py
+++ b/lib/galaxy/util/template.py
@@ -1,10 +1,51 @@
"""Entry point for the usage of Cheetah templating within Galaxy."""
+from __future__ import absolute_import
+
+import sys
+import traceback
+from lib2to3.refactor import RefactoringTool
+
+import packaging.version
+from Cheetah.Compiler import Compiler
+from Cheetah.NameMapper import NotFound
from Cheetah.Template import Template
+from past.translation import myfixes
from . import unicodify
+# Skip libpasteurize fixers, which make sure code is py2 and py3 compatible.
+# This is not needed, we only translate code on py3.
+myfixes = [f for f in myfixes if not f.startswith('libpasteurize')]
+refactoring_tool = RefactoringTool(myfixes, {'print_function': True})
-def fill_template(template_text, context=None, **kwargs):
+
+class FixedModuleCodeCompiler(Compiler):
+
+ module_code = None
+
+ def getModuleCode(self):
+ self._moduleDef = self.module_code
+ return self._moduleDef
+
+
+def create_compiler_class(module_code):
+
+ class CustomCompilerClass(FixedModuleCodeCompiler):
+ pass
+
+ setattr(CustomCompilerClass, 'module_code', module_code)
+
+ return CustomCompilerClass
+
+
+def fill_template(template_text,
+ context=None,
+ retry=10,
+ compiler_class=Compiler,
+ first_exception=None,
+ futurized=False,
+ python_template_version='3',
+ **kwargs):
"""Fill a cheetah template out for specified context.
If template_text is None, an exception will be thrown, if context
@@ -15,4 +56,62 @@ def fill_template(template_text, context=None, **kwargs):
raise TypeError("Template text specified as None to fill_template.")
if not context:
context = kwargs
- return unicodify(Template(source=template_text, searchList=[context]))
+ if isinstance(python_template_version, str):
+ python_template_version = packaging.version.parse(python_template_version)
+ klass = Template.compile(source=template_text, compilerClass=compiler_class)
+ t = klass(searchList=[context])
+ try:
+ return unicodify(t)
+ except NotFound as e:
+ if first_exception is None:
+ first_exception = e
+ if sys.version_info.major > 2 and python_template_version.release[0] < 3 and retry > 0:
+ tb = e.__traceback__
+ last_stack = traceback.extract_tb(tb)[-1]
+ if last_stack.name == '':
+ # On python 3 list, dict and set comprehensions as well as generator expressions
+ # have their own local scope, which prevents accessing frame variables in cheetah.
+ # We can work around this by replacing `$var` with `var`, but we only do this for
+ # list comprehensions, as this has never worked for dict or set comprehensions or
+ # generator expressions in Cheetah.
+ var_not_found = e.args[0].split("'")[1]
+ replace_str = 'VFFSL(SL,"%s",True)' % var_not_found
+ lineno = last_stack.lineno - 1
+ module_code = t._CHEETAH_generatedModuleCode.splitlines()
+ module_code[lineno] = module_code[lineno].replace(replace_str, var_not_found)
+ module_code = "\n".join(module_code)
+ compiler_class = create_compiler_class(module_code)
+ return fill_template(template_text=template_text,
+ context=context,
+ retry=retry - 1,
+ compiler_class=compiler_class,
+ first_exception=first_exception,
+ python_template_version=python_template_version,
+ )
+ raise first_exception or e
+ except Exception as e:
+ if first_exception is None:
+ first_exception = e
+ if sys.version_info.major > 2 and python_template_version.release[0] < 3 and not futurized:
+ # Possibly an error caused by attempting to run python 2
+ # template code on python 3. Run the generated module code
+ # through futurize and hope for the best.
+ module_code = t._CHEETAH_generatedModuleCode
+ module_code = futurize_preprocessor(module_code)
+ compiler_class = create_compiler_class(module_code)
+ return fill_template(template_text=template_text,
+ context=context,
+ retry=retry,
+ compiler_class=compiler_class,
+ first_exception=first_exception,
+ futurized=True,
+ python_template_version=python_template_version,
+ )
+ raise first_exception or e
+
+
+def futurize_preprocessor(source):
+ source = str(refactoring_tool.refactor_string(source, name='auto_translate_cheetah'))
+ # libfuturize.fixes.fix_unicode_keep_u' breaks from Cheetah.compat import unicode
+ source = source.replace('from Cheetah.compat import str', 'from Cheetah.compat import unicode')
+ return source
diff --git a/lib/galaxy/version.py b/lib/galaxy/version.py
index 0f46519cd48..701b600c93d 100644
--- a/lib/galaxy/version.py
+++ b/lib/galaxy/version.py
@@ -1,3 +1,3 @@
-VERSION_MAJOR = "19.09"
+VERSION_MAJOR = "19.05"
VERSION_MINOR = "dev"
VERSION = VERSION_MAJOR + ('.' + VERSION_MINOR if VERSION_MINOR else '')
diff --git a/lib/galaxy/visualization/plugins/interactive_environments.py b/lib/galaxy/visualization/plugins/interactive_environments.py
index b7c1016f41c..180345c7c1b 100644
--- a/lib/galaxy/visualization/plugins/interactive_environments.py
+++ b/lib/galaxy/visualization/plugins/interactive_environments.py
@@ -19,7 +19,10 @@ from galaxy import model, web
from galaxy.containers import ContainerPort
from galaxy.containers.docker_model import DockerVolume
from galaxy.managers import api_keys
-from galaxy.util import string_as_bool_or_none
+from galaxy.util import (
+ string_as_bool_or_none,
+ unicodify
+)
from galaxy.util.bunch import Bunch
@@ -432,6 +435,8 @@ class InteractiveEnvironmentRequest(object):
))
p = Popen(raw_cmd, stdout=PIPE, stderr=PIPE, close_fds=True)
stdout, stderr = p.communicate()
+ stdout = unicodify(stdout)
+ stderr = unicodify(stderr)
if p.returncode != 0:
log.error("Container Launch error\n\n%s\n%s" % (stdout, stderr))
return None
diff --git a/lib/galaxy/web/framework/webapp.py b/lib/galaxy/web/framework/webapp.py
index f64e0a1d05b..88ca95eb8df 100644
--- a/lib/galaxy/web/framework/webapp.py
+++ b/lib/galaxy/web/framework/webapp.py
@@ -650,24 +650,27 @@ class GalaxyWebTransaction(base.DefaultWebTransaction,
Associate the user's last accessed history (if exists) with their new session
"""
history = None
+ set_permissions = False
try:
users_last_session = user.galaxy_sessions[0]
- last_accessed = True
except Exception:
users_last_session = None
- last_accessed = False
if (prev_galaxy_session and
prev_galaxy_session.current_history and not
prev_galaxy_session.current_history.deleted and
- prev_galaxy_session.current_history.datasets):
- if prev_galaxy_session.current_history.user is None or prev_galaxy_session.current_history.user == user:
- # If the previous galaxy session had a history, associate it with the new
- # session, but only if it didn't belong to a different user.
- history = prev_galaxy_session.current_history
- if prev_galaxy_session.user is None:
- # Increase the user's disk usage by the amount of the previous history's datasets if they didn't already own it.
- for hda in history.datasets:
- user.adjust_total_disk_usage(hda.quota_amount(user))
+ prev_galaxy_session.current_history.datasets and
+ (prev_galaxy_session.current_history.user is None or
+ prev_galaxy_session.current_history.user == user)):
+ # If the previous galaxy session had a history, associate it with the new session, but only if it didn't
+ # belong to a different user.
+ history = prev_galaxy_session.current_history
+ if prev_galaxy_session.user is None:
+ # Increase the user's disk usage by the amount of the previous history's datasets if they didn't already
+ # own it.
+ for hda in history.datasets:
+ user.adjust_total_disk_usage(hda.quota_amount(user))
+ # Only set default history permissions if the history is from the previous session and anonymous
+ set_permissions = True
elif self.galaxy_session.current_history:
history = self.galaxy_session.current_history
if (not history and users_last_session and
@@ -681,8 +684,7 @@ class GalaxyWebTransaction(base.DefaultWebTransaction,
if history.user is None:
history.user = user
self.galaxy_session.current_history = history
- if not last_accessed:
- # Only set default history permissions if current history is not from a previous session
+ if set_permissions:
self.app.security_agent.history_set_default_permissions(history, dataset=True, bypass_manage_permission=True)
self.sa_session.add_all((prev_galaxy_session, self.galaxy_session, history))
diff --git a/lib/galaxy/web/proxy/js/lib/mapper.js b/lib/galaxy/web/proxy/js/lib/mapper.js
index 54fb097078e..78af6256dac 100644
--- a/lib/galaxy/web/proxy/js/lib/mapper.js
+++ b/lib/galaxy/web/proxy/js/lib/mapper.js
@@ -1,5 +1,6 @@
var fs = require('fs');
var sqlite3 = require('sqlite3')
+var watch = require('node-watch');
var endsWith = function(subjectString, searchString) {
@@ -67,7 +68,7 @@ var mapFor = function(path) {
}
console.log("Watching path " + path);
loadMap();
- fs.watch(path, loadMap);
+ watch(path, loadMap);
return map;
};
diff --git a/lib/galaxy/web/proxy/js/package.json b/lib/galaxy/web/proxy/js/package.json
index e74db4d284f..7a4aa13d4f6 100644
--- a/lib/galaxy/web/proxy/js/package.json
+++ b/lib/galaxy/web/proxy/js/package.json
@@ -14,6 +14,7 @@
"commander": "~2.19.0",
"eventemitter3": "3.1.0",
"http-proxy": "1.17.0",
+ "node-watch": "0.6.2",
"sqlite3": "4.0.4"
}
}
diff --git a/lib/galaxy/web/stack/handlers.py b/lib/galaxy/web/stack/handlers.py
index 124cf91d7fc..7754e8966b4 100644
--- a/lib/galaxy/web/stack/handlers.py
+++ b/lib/galaxy/web/stack/handlers.py
@@ -48,23 +48,58 @@ class ConfiguresHandlers(object):
else:
self.handlers[tag] = [handler_id]
- def _init_handlers(self, config_element):
+ @staticmethod
+ def xml_to_dict(config, config_element):
+ handling_config_dict = {}
+
+ processes = {}
+ handling_config_dict["processes"] = processes
+
# Parse handlers
if config_element is not None:
- for handler in self._findall_with_required(config_element, 'handler'):
+ for handler in ConfiguresHandlers._findall_with_required(config_element, 'handler'):
handler_id = handler.get('id')
- if handler_id in self.handlers:
+ if handler_id in processes:
log.error("Handler '%s' overlaps handler with the same name, ignoring", handler_id)
else:
log.debug("Read definition for handler '%s'", handler_id)
- self._parse_handler(handler_id, handler)
- self.add_handler(
- handler_id,
- [x.strip() for x in handler.get('tags', self.DEFAULT_HANDLER_TAG).split(',')]
- )
- self.default_handler_id = self._get_default(self.app.config, config_element, list(self.handlers.keys()))
+ plugins = []
+ for plugin in ConfiguresHandlers._findall_with_required(handler, 'plugin', ['id']):
+ plugins.append(plugin.get("id"))
+ tags = [x.strip() for x in handler.get('tags', ConfiguresHandlers.DEFAULT_HANDLER_TAG).split(',')]
+ handler_def = {"tags": tags}
+ if plugins:
+ handler_def["plugins"] = plugins
+ processes[handler_id] = handler_def
+ default_handler = ConfiguresHandlers.get_xml_default(config, config_element)
+ if default_handler:
+ handling_config_dict["default"] = default_handler
+
+ assign = listify(config_element.attrib.get('assign_with', []), do_strip=True)
+ if len(assign) > 0:
+ handling_config_dict["assign"] = assign
+ max_grap_str = config_element.attrib.get('max_grab', None)
+ if max_grap_str:
+ handling_config_dict["max_grab"] = int(max_grap_str)
+
+ return handling_config_dict
+
+ def _init_handlers(self, handling_config_dict=None):
+ handling_config_dict = handling_config_dict or {}
+ for handler_id, process in handling_config_dict.get("processes", {}).items():
+ process = process or {}
+ if handler_id in self.handlers:
+ log.error("Handler '%s' overlaps handler with the same name, ignoring", handler_id)
+ else:
+ log.debug("Read definition for handler '%s'", handler_id)
+ self._parse_handler(handler_id, process)
+ self.add_handler(handler_id, process.get("tags") or [self.DEFAULT_HANDLER_TAG])
+
+ self.default_handler_id = self._ensure_default_set(handling_config_dict.get("default"), list(self.handlers.keys()), required=False)
+
+ def _init_handler_assignment_methods(self, handling_config_dict=None):
+ handling_config_dict = handling_config_dict or {}
- def _init_handler_assignment_methods(self, config_element=None):
self.__is_handler = None
# This is set by the stack job handler init code
self.pool_for_tag = {}
@@ -76,8 +111,8 @@ class ConfiguresHandlers(object):
HANDLER_ASSIGNMENT_METHODS.DB_SKIP_LOCKED: self._assign_db_tag,
HANDLER_ASSIGNMENT_METHODS.UWSGI_MULE_MESSAGE: self._assign_uwsgi_mule_message_handler,
}
- if config_element is not None:
- for method in listify(config_element.attrib.get('assign_with', []), do_strip=True):
+ if handling_config_dict:
+ for method in handling_config_dict.get("assign", []):
method = method.lower()
assert method in HANDLER_ASSIGNMENT_METHODS, \
"Invalid job handler assignment method '%s', must be one of: %s" % (
@@ -89,7 +124,7 @@ class ConfiguresHandlers(object):
self.handler_assignment_methods = [method]
if self.handler_assignment_methods == [HANDLER_ASSIGNMENT_METHODS.MEM_SELF]:
self.app.config.track_jobs_in_database = False
- self.handler_max_grab = int(config_element.attrib.get('max_grab', self.handler_max_grab))
+ self.handler_max_grab = handling_config_dict.get('max_grab', self.handler_max_grab)
def _set_default_handler_assignment_methods(self):
if not self.handler_assignment_methods_configured:
@@ -116,7 +151,18 @@ class ConfiguresHandlers(object):
def _parse_handler(self, handler_id, handler_def):
pass
- def _get_default(self, config, parent, names, auto=False):
+ @staticmethod
+ def get_xml_default(config, parent):
+ rval = parent.get('default')
+ if 'default_from_environ' in parent.attrib:
+ environ_var = parent.attrib['default_from_environ']
+ rval = os.environ.get(environ_var, rval)
+ elif 'default_from_config' in parent.attrib:
+ config_val = parent.attrib['default_from_config']
+ rval = config.config_dict.get(config_val, rval)
+ return rval
+
+ def _get_default(self, config, parent, names, auto=False, required=True):
"""
Returns the default attribute set in a parent tag like or
, or return the ID of the child, if there is no explicit
@@ -128,29 +174,29 @@ class ConfiguresHandlers(object):
:type names: list of str
:param auto: Automatically set a default if there is no default in the parent tag and there is only one child.
:type auto: bool
+ :param required: Require a default to be set or determined automatically, else raise Exception
+ :type required: bool
:returns: str -- id or tag representing the default.
"""
+ rval = ConfiguresHandlers.get_xml_default(config, parent)
+ return self._ensure_default_set(rval, names, auto=auto, required=required)
- rval = parent.get('default')
- if 'default_from_environ' in parent.attrib:
- environ_var = parent.attrib['default_from_environ']
- rval = os.environ.get(environ_var, rval)
- elif 'default_from_config' in parent.attrib:
- config_val = parent.attrib['default_from_config']
- rval = config.config_dict.get(config_val, rval)
-
+ def _ensure_default_set(self, rval, names, auto=False, required=True):
if rval is not None:
# If the parent element has a 'default' attribute, use the id or tag in that attribute
- if self.deterministic_handler_assignment and rval not in names:
- raise Exception("<%s> default attribute '%s' does not match a defined id or tag in a child element" % (parent.tag, rval))
- log.debug("<%s> default set to child with id or tag '%s'" % (parent.tag, rval))
+ if required and rval not in names:
+ raise Exception("default attribute '%s' does not match a defined id or tag in a child element" % (rval))
+ log.debug("default set to child with id or tag '%s'" % (rval))
elif auto and len(names) == 1:
- log.info("Setting <%s> default to child with id '%s'" % (parent.tag, names[0]))
+ log.info("Setting default to child with id '%s'" % (names[0]))
rval = names[0]
+ elif required:
+ raise Exception("No default specified, please specify a valid id or tag with the 'default' attribute")
return rval
- def _findall_with_required(self, parent, match, attribs=None):
+ @staticmethod
+ def _findall_with_required(parent, match, attribs=None):
"""Like ``xml.etree.ElementTree.Element.findall()``, except only returns children that have the specified attribs.
:param parent: Parent element in which to find.
@@ -181,7 +227,7 @@ class ConfiguresHandlers(object):
@property
def deterministic_handler_assignment(self):
- return self.handler_assignment_methods and all(
+ return self.handler_assignment_methods and any(
filter(lambda x: x in (
HANDLER_ASSIGNMENT_METHODS.UWSGI_MULE_MESSAGE,
HANDLER_ASSIGNMENT_METHODS.DB_PREASSIGN,
diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py
index 6eb8fa7a76c..9289ab0b1f5 100644
--- a/lib/galaxy/webapps/galaxy/buildapp.py
+++ b/lib/galaxy/webapps/galaxy/buildapp.py
@@ -101,6 +101,7 @@ def app_factory(global_conf, load_app_kwds={}, **kwargs):
webapp.add_client_route('/admin/data_tables', 'admin')
webapp.add_client_route('/admin/data_types', 'admin')
+ webapp.add_client_route('/admin/jobs', 'admin')
webapp.add_client_route('/admin/data_manager{path_info:.*}', 'admin')
webapp.add_client_route('/admin/error_stack', 'admin')
webapp.add_client_route('/admin/users', 'admin')
diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml
index 7447f18be59..91b3482b4e1 100644
--- a/lib/galaxy/webapps/galaxy/config_schema.yml
+++ b/lib/galaxy/webapps/galaxy/config_schema.yml
@@ -102,10 +102,10 @@ mapping:
default: false
required: false
desc: |
- Enable's a per request sql debugging option. If this is set to true, append
- ?sql_debug=1 to web request URLs to enable detailed logging on the backend of SQL
- queries generated during that request. This is useful for debugging slow endpoints
- during development.
+ Enables a per request sql debugging option. If this is set to true,
+ append ?sql_debug=1 to web request URLs to enable detailed logging on
+ the backend of SQL queries generated during that request. This is
+ useful for debugging slow endpoints during development.
install_database_connection:
type: str
@@ -144,7 +144,7 @@ mapping:
database_wait_sleep:
type: float
- default: 1
+ default: 1.
required: false
desc: |
Time to sleep between attempts if database_wait is enabled (in seconds).
@@ -397,9 +397,9 @@ mapping:
default: ''
required: false
desc: |
- Container resolvers configuration (beta). Setup a file describing container
- resolvers to use when discovering containers for Galaxy. If this is set to
- None, the default containers loaded is determined by
+ Container resolvers configuration (beta). Set up a file describing
+ container resolvers to use when discovering containers for Galaxy. If
+ this is set to None, the default containers loaded is determined by
enable_beta_mulled_containers.
involucro_path:
@@ -874,7 +874,7 @@ mapping:
desc: |
Galaxy can display data at various external browsers. These options specify
which browsers should be available. URLs and builds available at these
- browsers are defined in the specifield files.
+ browsers are defined in the specified files.
If use_remote_user = True, display application servers will be denied access
to Galaxy and so displaying datasets in these sites will fail.
@@ -970,14 +970,15 @@ mapping:
default: http://localhost:8080
required: false
desc: |
- URL (with schema http/https) of the Galaxy instance as accessible within your
- local network - if specified used as a default by pulsar file staging and
- Jupyter Docker container for communicating back with Galaxy via the API.
+ URL (with schema http/https) of the Galaxy instance as accessible
+ within your local network - if specified used as a default by pulsar
+ file staging and Jupyter Docker container for communicating back with
+ Galaxy via the API.
- If you are attempting to setup GIEs on Mac OS X with Docker for Mac - this
- should likely be the IP address of your machine on the virtualbox network (vboxnet0)
- setup for the Docker host VM. This can found by running ifconfig and using the
- IP address of the network vboxnet0.
+ If you are attempting to set up GIEs on Mac OS X with Docker Desktop
+ for Mac and your Galaxy instance runs on port 8080 this should be
+ 'http://host.docker.internal:8080'. For more details see
+ https://docs.docker.com/docker-for-mac/networking/
galaxy_infrastructure_web_port:
type: int
@@ -1292,7 +1293,7 @@ mapping:
default: 8800
required: false
desc: |
- Set the port and IP for the the dynamic proxy to bind to, this must match
+ Set the port and IP for the dynamic proxy to bind to, this must match
the external configuration if dynamic_proxy_manage is False.
dynamic_proxy_bind_ip:
@@ -1300,7 +1301,7 @@ mapping:
default: 0.0.0.0
required: false
desc: |
- Set the port and IP for the the dynamic proxy to bind to, this must match
+ Set the port and IP for the dynamic proxy to bind to, this must match
the external configuration if dynamic_proxy_manage is False.
dynamic_proxy_debug:
@@ -2360,6 +2361,9 @@ mapping:
the system on which Galaxy is started. Advanced job running capabilities can
be configured through the job configuration file.
+ job_config:
+ !include job_config_schema.yml
+
default_job_resubmission_condition:
type: str
required: false
diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py
index f001821285a..175fff3589b 100644
--- a/lib/galaxy/webapps/galaxy/controllers/admin.py
+++ b/lib/galaxy/webapps/galaxy/controllers/admin.py
@@ -1600,15 +1600,26 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP
return {'message' : 'User \'%s\' has been updated with %d associated roles and %d associated groups (private roles are not displayed).' % (user.email, len(in_roles) - 1, len(in_groups))}
@web.expose
+ @web.json
@web.require_admin
- def jobs(self, trans, stop=[], stop_msg=None, cutoff=180, job_lock=None, ajl_submit=None, **kwd):
+ def jobs_control(self, trans, job_lock=None, **kwd):
+ if job_lock is not None:
+ job_lock = True if job_lock == 'true' else False
+ galaxy.queue_worker.send_control_task(trans.app, 'admin_job_lock', kwargs={'job_lock': job_lock}, get_response=True)
+ job_lock = trans.app.job_manager.job_lock
+ return {'job_lock': job_lock}
+
+ @web.expose
+ @web.json
+ @web.require_admin
+ def jobs_list(self, trans, stop=[], stop_msg=None, cutoff=180, **kwd):
deleted = []
- msg = None
- status = None
+ message = kwd.get('message', '')
+ status = kwd.get('status', 'info')
job_ids = util.listify(stop)
if job_ids and stop_msg in [None, '']:
- msg = 'Please enter an error message to display to the user describing why the job was terminated'
- status = 'error'
+ message = 'Please enter an error message to display to the user describing why the job was terminated'
+ return self.message_exception(trans, message)
elif job_ids:
if stop_msg[-1] not in PUNCTUATION:
stop_msg += '.'
@@ -1624,24 +1635,14 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP
trans.app.job_manager.stop(job, message=error_msg)
deleted.append(str(job_id))
if deleted:
- msg = 'Queued job'
+ message = 'Queued job'
if len(deleted) > 1:
- msg += 's'
- msg += ' for deletion: '
- msg += ', '.join(deleted)
+ message += 's'
+ message += ' for deletion: '
+ message += ', '.join(deleted)
status = 'done'
trans.sa_session.flush()
- if ajl_submit:
- if job_lock == 'on':
- galaxy.queue_worker.send_control_task(trans.app, 'admin_job_lock',
- kwargs={'job_lock': True})
- job_lock = True
- else:
- galaxy.queue_worker.send_control_task(trans.app, 'admin_job_lock',
- kwargs={'job_lock': False})
- job_lock = False
- else:
- job_lock = trans.app.job_manager.job_lock
+ job_lock = trans.app.job_manager.job_lock
cutoff_time = datetime.utcnow() - timedelta(seconds=int(cutoff))
jobs = trans.sa_session.query(trans.app.model.Job) \
.filter(and_(trans.app.model.Job.table.c.update_time < cutoff_time,
@@ -1655,33 +1656,44 @@ class AdminGalaxy(controller.JSAppLauncher, AdminActions, UsesQuotaMixin, QuotaP
or_(trans.app.model.Job.state == trans.app.model.Job.states.ERROR,
trans.app.model.Job.state == trans.app.model.Job.states.OK))) \
.order_by(trans.app.model.Job.table.c.update_time.desc()).all()
- last_updated = {}
- for job in jobs:
- delta = datetime.utcnow() - job.update_time
- if delta.days > 0:
- last_updated[job.id] = '%s hours' % (delta.days * 24 + int(delta.seconds / 60 / 60))
- elif delta > timedelta(minutes=59):
- last_updated[job.id] = '%s hours' % int(delta.seconds / 60 / 60)
- else:
- last_updated[job.id] = '%s minutes' % int(delta.seconds / 60)
- finished = {}
- for job in recent_jobs:
- delta = datetime.utcnow() - job.update_time
- if delta.days > 0:
- finished[job.id] = '%s hours' % (delta.days * 24 + int(delta.seconds / 60 / 60))
- elif delta > timedelta(minutes=59):
- finished[job.id] = '%s hours' % int(delta.seconds / 60 / 60)
- else:
- finished[job.id] = '%s minutes' % int(delta.seconds / 60)
- return trans.fill_template('/admin/jobs.mako',
- jobs=jobs,
- recent_jobs=recent_jobs,
- last_updated=last_updated,
- finished=finished,
- cutoff=cutoff,
- msg=msg,
- status=status,
- job_lock=job_lock)
+
+ def prepare_jobs_list(jobs):
+ res = []
+ for job in jobs:
+ delta = datetime.utcnow() - job.update_time
+ update_time = ""
+ if delta.days > 0:
+ update_time = '%s hours ago' % (delta.days * 24 + int(delta.seconds / 60 / 60))
+ elif delta > timedelta(minutes=59):
+ update_time = '%s hours ago' % int(delta.seconds / 60 / 60)
+ else:
+ update_time = '%s minutes ago' % int(delta.seconds / 60)
+ inputs = ""
+ try:
+ inputs = ", ".join(['{} {}'.format(da.dataset.id, da.dataset.state) for da in job.input_datasets])
+ except Exception:
+ inputs = 'Unable to determine inputs'
+ res.append({
+ 'job_info': {
+ 'id': job.id,
+ 'info_url': "{}?jobid={}".format(web.url_for(controller="admin", action="job_info"), job.id)
+ },
+ 'user': job.history.user.email if job.history and job.history.user else 'anonymous',
+ 'update_time': update_time,
+ 'tool_id': job.tool_id,
+ 'state': job.state,
+ 'input_dataset': inputs,
+ 'command_line': job.command_line,
+ 'job_runner_name': job.job_runner_name,
+ 'job_runner_external_id': job.job_runner_external_id
+ })
+ return res
+ return {'jobs': prepare_jobs_list(jobs),
+ 'recent_jobs': prepare_jobs_list(recent_jobs),
+ 'cutoff': cutoff,
+ 'message': message,
+ 'status': status,
+ 'job_lock': job_lock}
@web.expose
@web.require_admin
diff --git a/lib/galaxy/webapps/galaxy/controllers/tag.py b/lib/galaxy/webapps/galaxy/controllers/tag.py
index 25e74fdf576..0320f2be72e 100644
--- a/lib/galaxy/webapps/galaxy/controllers/tag.py
+++ b/lib/galaxy/webapps/galaxy/controllers/tag.py
@@ -90,7 +90,6 @@ class TagsController(BaseUIController, UsesTagsMixin):
user = trans.user
item_class = self.get_class(item_class)
q = '' if q is None else q
- q = q.encode('utf-8')
if q.find(":") == -1:
return self._get_tag_autocomplete_names(trans, q, limit, timestamp, user, item, item_class)
else:
diff --git a/lib/galaxy/webapps/galaxy/job_config_schema.yml b/lib/galaxy/webapps/galaxy/job_config_schema.yml
new file mode 100644
index 00000000000..f025bc85be1
--- /dev/null
+++ b/lib/galaxy/webapps/galaxy/job_config_schema.yml
@@ -0,0 +1,116 @@
+desc: Description of job running configuration, can be embedded into Galaxy configuration
+ or loaded from an additional file with the job_config_file option.
+type: map
+mapping:
+ runners:
+ type: map
+ mapping:
+ dynamic:
+ type: map
+ mapping:
+ rules_module:
+ type: str
+ regex;(.+):
+ type: map
+ allowempty: true
+ mapping:
+ workers:
+ type: int
+ load:
+ type: str
+ enabled:
+ type: bool
+ handling:
+ type: map
+ allowempty: true
+ execution:
+ type: map
+ allowempty: true
+ mapping:
+ default:
+ type: str
+ environments:
+ type: map
+ mapping:
+ regex;(.+):
+ type: map
+ allowempty: true
+ mapping:
+ runner:
+ type: str
+ tags:
+ type: seq
+ sequence:
+ - type: str
+ shell:
+ type: str
+ env:
+ type: seq
+ sequence:
+ - type: map
+ mapping:
+ name:
+ type: str
+ raw:
+ type: bool
+ value:
+ type: str
+ file:
+ type: str
+ execute:
+ type: str
+ resubmit:
+ type: seq
+ sequence:
+ - type: map
+ mapping:
+ delay:
+ type: any # str or number
+ condition:
+ type: str
+ environment:
+ type: str
+ handler:
+ type: str
+
+ tools:
+ type: seq
+ sequence:
+ - type: map
+ allowempty: true
+ mapping:
+ id:
+ type: str
+ handler:
+ type: str
+ environment:
+ type: str
+ resources:
+ type: str
+ resources:
+ type: map
+ mapping:
+ default:
+ type: str
+ groups:
+ type: map
+ mapping:
+ regex;(.+):
+ type: seq
+ sequence:
+ - type: str
+ limits:
+ type: seq
+ sequence:
+ - type: map
+ mapping:
+ type:
+ type: str
+ value:
+ type: any
+ window:
+ type: int
+ id:
+ type: str
+ tag:
+ type: str
diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py
index 7386e2c2f84..184a2b14cd0 100644
--- a/lib/galaxy/workflow/modules.py
+++ b/lib/galaxy/workflow/modules.py
@@ -1,9 +1,9 @@
"""
Modules used in building workflows
"""
+import json
import logging
import re
-from json import loads
from xml.etree.ElementTree import (
Element,
XML
@@ -31,6 +31,7 @@ from galaxy.tools.parameters import (
visit_input_values
)
from galaxy.tools.parameters.basic import (
+ BaseDataToolParameter,
BooleanToolParameter,
ConnectedValue,
DataCollectionToolParameter,
@@ -424,6 +425,8 @@ class SubWorkflowModule(WorkflowModule):
)
if step.type == 'data_collection_input':
input['collection_type'] = step.tool_inputs.get('collection_type') if step.tool_inputs else None
+ if step_type == 'parameter_input':
+ input['type'] = step.tool_inputs['parameter_type']
inputs.append(input)
return inputs
@@ -993,7 +996,9 @@ class ToolModule(WorkflowModule):
params = make_dict_copy(self.state.inputs)
params['on_string'] = 'input dataset(s)'
params['tool'] = self.tool
- extra_kwds['label'] = fill_template(tool_output.label, context=params)
+ extra_kwds['label'] = fill_template(tool_output.label,
+ context=params,
+ python_template_version=self.tool.python_template_version)
except Exception:
pass
data_outputs.append(
@@ -1157,7 +1162,7 @@ class ToolModule(WorkflowModule):
if self.tool:
state = super(ToolModule, self).decode_runtime_state(runtime_state)
if RUNTIME_STEP_META_STATE_KEY in runtime_state:
- self.__restore_step_meta_runtime_state(loads(runtime_state[RUNTIME_STEP_META_STATE_KEY]))
+ self.__restore_step_meta_runtime_state(json.loads(runtime_state[RUNTIME_STEP_META_STATE_KEY]))
return state
else:
raise ToolMissingException("Tool %s missing. Cannot recover runtime state." % self.tool_id,
@@ -1202,11 +1207,13 @@ class ToolModule(WorkflowModule):
input_dict = all_inputs_by_name[prefixed_name]
replacement = NO_REPLACEMENT
+ dataset_instance = None
if iteration_elements and prefixed_name in iteration_elements:
- if isinstance(input, DataToolParameter) and hasattr(iteration_elements[prefixed_name], 'dataset_instance'):
+ dataset_instance = getattr(iteration_elements[prefixed_name], 'dataset_instance', None)
+ if isinstance(input, DataToolParameter) and dataset_instance:
# Pull out dataset instance (=HDA) from element and set a temporary element_identifier attribute
# See https://github.com/galaxyproject/galaxy/pull/1693 for context.
- replacement = iteration_elements[prefixed_name].dataset_instance
+ replacement = dataset_instance
if hasattr(iteration_elements[prefixed_name], u'element_identifier') and iteration_elements[prefixed_name].element_identifier:
replacement.element_identifier = iteration_elements[prefixed_name].element_identifier
else:
@@ -1216,6 +1223,12 @@ class ToolModule(WorkflowModule):
replacement = progress.replacement_for_input(step, input_dict)
if replacement is not NO_REPLACEMENT:
+ if not isinstance(input, BaseDataToolParameter):
+ # Probably a parameter that can be replaced
+ dataset = dataset_instance or replacement
+ if getattr(dataset, 'extension', None) == 'expression.json':
+ with open(dataset.file_name, 'r') as f:
+ replacement = json.load(f)
found_replacement_keys.add(prefixed_name)
return replacement
diff --git a/lib/galaxy/workflow/run.py b/lib/galaxy/workflow/run.py
index 961a673e9f4..e9120f2529b 100644
--- a/lib/galaxy/workflow/run.py
+++ b/lib/galaxy/workflow/run.py
@@ -1,4 +1,3 @@
-import json
import logging
import uuid
@@ -369,15 +368,24 @@ class WorkflowProgress(object):
delayed_why = "dependent collection [%s] not yet populated with datasets" % replacement.id
raise modules.DelayedWorkflowEvaluation(why=delayed_why)
- is_hda = isinstance(replacement, model.HistoryDatasetAssociation)
- if not is_data and is_hda:
- if replacement.is_ok:
- with open(replacement.file_name, 'r') as f:
- replacement = json.load(f)
- elif replacement.is_pending:
- raise modules.DelayedWorkflowEvaluation()
+ data_inputs = (model.HistoryDatasetAssociation, model.HistoryDatasetCollectionAssociation, model.DatasetCollection)
+ if not is_data and isinstance(replacement, data_inputs):
+ if isinstance(replacement, model.HistoryDatasetAssociation):
+ if replacement.is_pending:
+ raise modules.DelayedWorkflowEvaluation()
+ if not replacement.is_ok:
+ raise modules.CancelWorkflowEvaluation()
else:
- raise modules.CancelWorkflowEvaluation()
+ if not replacement.collection.populated:
+ raise modules.DelayedWorkflowEvaluation()
+ pending = False
+ for dataset_instance in replacement.dataset_instances:
+ if dataset_instance.is_pending:
+ pending = True
+ elif not dataset_instance.is_ok:
+ raise modules.CancelWorkflowEvaluation()
+ if pending:
+ raise modules.DelayedWorkflowEvaluation()
return replacement
diff --git a/lib/galaxy/workflow/scheduling_manager.py b/lib/galaxy/workflow/scheduling_manager.py
index f88e9b23e78..33e5cfa193b 100644
--- a/lib/galaxy/workflow/scheduling_manager.py
+++ b/lib/galaxy/workflow/scheduling_manager.py
@@ -230,8 +230,9 @@ class WorkflowSchedulingManager(ConfiguresHandlers):
def __init_handlers(self, config_element=None):
assert not self.__handlers_configured
- self._init_handler_assignment_methods(config_element)
- self._init_handlers(config_element)
+ handling_config_dict = ConfiguresHandlers.xml_to_dict(self.app.config, config_element)
+ self._init_handler_assignment_methods(handling_config_dict)
+ self._init_handlers(handling_config_dict)
if not self.handler_assignment_methods_configured:
self._set_default_handler_assignment_methods()
else:
diff --git a/run_tests.sh b/run_tests.sh
index 5634c1d7793..0efac9e7111 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -362,6 +362,8 @@ do
fi
;;
-a|-api|--api)
+ GALAXY_TEST_USE_HIERARCHICAL_OBJECT_STORE="True" # Run these tests with a non-trivial object store.
+ export GALAXY_TEST_USE_HIERARCHICAL_OBJECT_STORE
GALAXY_TEST_TOOL_CONF="config/tool_conf.xml.sample,test/functional/tools/samples_tool_conf.xml"
test_script="pytest"
report_file="./run_api_tests.html"
diff --git a/scripts/cleanup_datasets/pgcleanup.py b/scripts/cleanup_datasets/pgcleanup.py
index 35492e4c2f0..260b65f5d5a 100755
--- a/scripts/cleanup_datasets/pgcleanup.py
+++ b/scripts/cleanup_datasets/pgcleanup.py
@@ -635,6 +635,8 @@ class PurgeDeletedUsers(PurgesHDAs, RemovesMetadataFiles, Action):
self.__zero_disk_usage_user_ids.add(row.zero_disk_usage_user_id)
def zero_disk_usage(self):
+ if not self.__zero_disk_usage_user_ids:
+ return
log.info('Zeroing disk usage for users who were purged')
sql = """
UPDATE galaxy_user
@@ -991,7 +993,7 @@ class Cleanup(object):
# TODO: is this per session or cursor?
if self.args.work_mem is not None:
log.info('Setting work_mem to %s' % self.args.work_mem)
- self._conn.cursor().execute('SET work_mem TO %s', (self.args.work_mem,))
+ self.__conn.cursor().execute('SET work_mem TO %s', (self.args.work_mem,))
return self.__conn
def __parse_args(self):
diff --git a/static/welcome.html.sample b/static/welcome.html.sample
index c3ef6200153..13346165b53 100644
--- a/static/welcome.html.sample
+++ b/static/welcome.html.sample
@@ -5,7 +5,7 @@
-
- Unfinished and recently finished jobs are displayed on this page. The
- 'cutoff' input box will do two things -- it will limit the display of
- unfinished jobs to only those jobs that have not had their job state
- updated recently, and it will limit the recently finished jobs list to only
- displaying jobs that have finished since the cutoff.
-
-
- If any jobs are displayed, you may choose to stop them. Your stop message
- will be displayed to the user as: "This job was stopped by an
- administrator: <YOUR MESSAGE> For more information or help,
- report this error".
-
-
-%if jobs:
-
-%else:
-
There are no unfinished jobs to show with current cutoff time.
-
-%endif
-
-%if recent_jobs:
-
- Recent Jobs: These jobs have completed in the previous ${cutoff} seconds.
-