opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge Galaxyproject Dev, Remove Modifications on Unrelated Files

Browse Source
This commit is contained in:
Matthew Spelchak
2018-03-14 09:22:21 -05:00
parent 5120ec6687 c6993caabf
commit 729714f940
139 changed files with 7207 additions and 5356 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/galaxy/scripts/apps/panels/tool-panel.js
+39 -53
View File
@@ -2,6 +2,7 @@ import Tools from "mvc/tool/tools";
import Upload from "mvc/upload/upload-view";
import _l from "utils/localization";
import ToolForm from "mvc/tool/tool-form-composite";
import _ from "libs/underscore";
var ToolPanel = Backbone.View.extend({
initialize: function(page, options) {
@@ -28,7 +29,8 @@ var ToolPanel = Backbone.View.extend({
// add upload modal
this.upload_button = new Upload({
nginx_upload_path: config.nginx_upload_path,
upload_path: config.nginx_upload_path || `${Galaxy.root}api/tools`,
chunk_upload_size: config.chunk_upload_size,
ftp_upload_site: config.ftp_upload_site,
default_genome: config.default_genome,
default_extension: config.default_extension
@@ -74,70 +76,54 @@ var ToolPanel = Backbone.View.extend({
/** build a link to one tool */
_templateTool: function(tool) {
return [
'<div class="toolTitle">',
'<a href="',
this.root,
tool.href,
'" target="galaxy_main">',
tool.title,
"</a>",
"</div>"
].join("");
return `<div class="toolTitle">
<a href="${Galaxy.root}${tool.href}" target="galaxy_main">
${tool.title}
</a>
</div>`;
},
/** build a link to 'All Workflows' */
_templateAllWorkflow: function(tool) {
return [
'<div class="toolTitle">',
// global
'<a href="',
Galaxy.root,
tool.href,
'">',
tool.title,
"</a>",
"</div>"
].join("");
return `<div class="toolTitle">
<a href="${Galaxy.root}${tool.href}">
${tool.title}
</a>
</div>`;
},
/** build links to workflows in toolpanel */
_templateWorkflowLink: function(wf) {
return [
'<div class="toolTitle">',
`<a class="${wf.cls} " href="`,
Galaxy.root,
wf.href,
'">',
wf.title,
"</a>",
"</div>"
].join("");
return `<div class="toolTitle">
<a class="${wf.cls}" href="${Galaxy.root}${wf.href}">
${_.escape(wf.title)}
</a>
</div>`;
},
/** override to include inital menu dom and workflow section */
_template: function() {
return [
'<div class="toolMenuContainer">',
'<div class="toolMenu" style="display: none">',
'<div id="search-no-results" style="display: none; padding-top: 5px">',
"<em><strong>",
_l("Search did not match any tools."),
"</strong></em>",
"</div>",
"</div>",
'<div class="toolSectionPad"/>',
'<div class="toolSectionPad"/>',
'<div class="toolSectionTitle" id="title_XXinternalXXworkflow">',
"<span>",
_l("Workflows"),
"</span>",
"</div>",
'<div id="internal-workflows" class="toolSectionBody">',
'<div class="toolSectionBg"/>',
"</div>",
"</div>"
].join("");
return `<div class="toolMenuContainer">
<div class="toolMenu" style="display: none">
<div id="search-no-results" style="display: none; padding-top: 5px">
<em>
<strong>
${_l("Search did not match any tools.")}
</strong>
</em>
</div>
</div>
<div class="toolSectionPad"/>
<div class="toolSectionPad"/>
<div class="toolSectionTitle" id="title_XXinternalXXworkflow">
<span>
${_l("Workflows")}
</span>
</div>
<div id="internal-workflows" class="toolSectionBody">
<div class="toolSectionBg"/>
</div>
</div>`;
},
toString: function() {
client/galaxy/scripts/mvc/form/form-data.js
+1 -1
View File
@@ -200,7 +200,7 @@ export var matchCase = (input, value) => {
* @param{dict} callback - Called with the mapped dictionary object and corresponding model node
*/
export var visitInputs = (inputs, callback, prefix, context) => {
context = $.extend(true, {}, context);
context = $.extend({}, context);
_.each(inputs, input => {
if (input && input.type && input.name) {
context[input.name] = input;
client/galaxy/scripts/mvc/form/form-section.js
+2 -3
View File
@@ -26,9 +26,8 @@ var View = Backbone.View.extend({
},
/** Add a new input element */
add: function(input) {
var input_def = jQuery.extend(true, {}, input);
input_def.id = input.id = Utils.uid();
add: function(input_def) {
input_def.id = Utils.uid();
this.app.input_list[input_def.id] = input_def;
switch (input_def.type) {
case "conditional":
client/galaxy/scripts/mvc/form/form-view.js
+1 -1
View File
@@ -80,7 +80,7 @@ export default Backbone.View.extend({
.first();
$panel.animate(
{
scrollTop: $panel.scrollTop() + input_element.$el.offset().top - 120
scrollTop: $panel.scrollTop() + input_element.$el.offset().top - $panel.position().top - 120
},
500
);
client/galaxy/scripts/mvc/grid/grid-template.js
+3 -1
View File
@@ -93,7 +93,9 @@ export default {
if (column.visible) {
tmpl += `<th id="${column.key}-header">`;
if (column.sortable) {
tmpl += `<a href="javascript:void(0)" class="sort-link" sort_key="${column.key}">${column.label}</a>`;
tmpl += `<a href="javascript:void(0)" class="sort-link" sort_key="${column.key}">${
column.label
}</a>`;
} else {
tmpl += column.label;
}
client/galaxy/scripts/mvc/tool/tool-form-composite.js
-28
View File
@@ -25,18 +25,6 @@ var View = Backbone.View.extend({
$("body").append(this.$el);
this._configure();
this.render();
$(window).resize(() => {
self._refresh();
});
},
/** Refresh height of scrollable div below header, handle scrolling by lazy loading steps */
_refresh: function(step_index) {
var margin =
_.reduce(this.$el.children(), (memo, child) => memo + $(child).outerHeight(), 0) -
this.$steps.height() +
90;
this.$steps.css("height", $(window).height() - margin);
},
/** Configures form/step options for each workflow step */
@@ -201,7 +189,6 @@ var View = Backbone.View.extend({
this._renderParameters();
this._renderHistory();
this._renderUseCachedJob();
this._renderResourceParameters();
_.each(this.steps, step => {
self._renderStep(step);
});
@@ -312,19 +299,6 @@ var View = Backbone.View.extend({
});
this._append(this.$steps, this.history_form.$el);
},
/** Render Workflow Options */
_renderResourceParameters: function() {
this.workflow_resource_parameters_form = null;
if(!_.isEmpty(this.model.get('workflow_resource_parameters'))){
this.workflow_resource_parameters_form = new Form({
cls : 'ui-portlet-narrow',
title : '<b>Workflow Resource Options</b>',
inputs : this.model.get('workflow_resource_parameters')
});
this._append( this.$steps, this.workflow_resource_parameters_form.$el );
}
},
/** Render job caching option */
_renderUseCachedJob: function() {
@@ -446,7 +420,6 @@ var View = Backbone.View.extend({
}
self.forms[step.index] = form;
self._append(self.$steps, form.$el);
self._refresh();
step.needs_refresh && self._refreshStep(step);
form.portlet[!self.show_progress ? "enable" : "disable"]();
self.show_progress &&
@@ -548,7 +521,6 @@ var View = Backbone.View.extend({
var job_def = {
new_history_name: history_form_data["new_history|name"] ? history_form_data["new_history|name"] : null,
history_id: !history_form_data["new_history|name"] ? this.model.get("history_id") : null,
resource_params: this.workflow_resource_parameters_form ? this.workflow_resource_parameters_form.data.create() : {},
replacement_params: this.wp_form ? this.wp_form.data.create() : {},
parameters: {},
// Tool form will submit flat maps for each parameter
client/galaxy/scripts/mvc/ui/ui-select-genomespace.js
+4 -16
View File
@@ -26,16 +26,10 @@ var View = Backbone.View.extend({
// create genomespace filepath textbox
this.filename_textbox = new Ui.Input();
// create genomespace token textbox
this.token_textbox = new Ui.Input({
type: "password"
});
// create elements
this.setElement(this._template(options));
this.$(".ui-gs-browse-button").append(this.browse_button.$el);
this.$(".ui-gs-filename-textbox").append(this.filename_textbox.$el);
this.$(".ui-gs-token-textbox").append(this.token_textbox.$el);
},
/** Browse GenomeSpace */
@@ -43,7 +37,7 @@ var View = Backbone.View.extend({
var self = this;
GenomespaceBrowser.openFileBrowser({
successCallback: function(data) {
self.value(`${data.destination}^${data.token}`);
self.value(data.destination);
}
});
},
@@ -56,15 +50,11 @@ var View = Backbone.View.extend({
'<span class="ui-gs-browse-button" />' +
'<span class="ui-gs-filename-textbox" />' +
"</div>" +
'<div class="ui-gs-token-field">' +
'<span class=ui-gs-label"><div class="ui-gs-token-label">Token</div></span>' +
'<span class="ui-gs-token-textbox" />' +
"</div>" +
"</div>"
);
},
/** Return/Set currently selected genomespace filename/token */
/** Return/Set currently selected genomespace filename */
value: function(new_value) {
// check if new_value is defined
if (new_value !== undefined) {
@@ -76,15 +66,13 @@ var View = Backbone.View.extend({
// get value
_getValue: function() {
return `${this.filename_textbox.value()}^${this.token_textbox.value()}`;
return this.filename_textbox.value();
},
// set value
_setValue: function(new_value) {
if (new_value) {
var values = new_value.split("^");
this.filename_textbox.value(values[0]);
this.token_textbox.value(values[1]);
this.filename_textbox.value(new_value);
}
this.options.onchange && this.options.onchange(new_value);
}
client/galaxy/scripts/mvc/upload/collection/collection-view.js
+1 -1
View File
@@ -116,7 +116,7 @@ export default Backbone.View.extend({
// file upload
this.uploadbox = this.$(".upload-box").uploadbox({
url: this.app.options.nginx_upload_path,
url: this.app.options.upload_path,
announce: function(index, file) {
self._eventAnnounce(index, file);
},
client/galaxy/scripts/mvc/upload/composite/composite-view.js
+1 -1
View File
@@ -133,7 +133,7 @@ export default Backbone.View.extend({
});
});
$.uploadpost({
url: this.app.options.nginx_upload_path,
url: this.app.options.upload_path,
data: this.app.toData(this.collection.filter()),
success: function(message) {
self._eventSuccess(message);
client/galaxy/scripts/mvc/upload/default/default-row.js
+8 -3
View File
@@ -11,6 +11,7 @@ export default Backbone.View.extend({
init: "upload-icon-button fa fa-trash-o",
queued: "upload-icon fa fa-spinner fa-spin",
running: "upload-icon fa fa-spinner fa-spin",
warning: "upload-icon fa fa-spinner fa-spin",
success: "upload-icon-button fa fa-check",
error: "upload-icon-button fa fa-exclamation-triangle"
},
@@ -183,7 +184,7 @@ export default Backbone.View.extend({
_refreshInfo: function() {
var info = this.model.get("info");
if (info) {
this.$info_text.html(`<strong>Failed: </strong>${info}`).show();
this.$info_text.html(`<strong>Warning: </strong>${info}`).show();
} else {
this.$info_text.hide();
}
@@ -213,13 +214,17 @@ export default Backbone.View.extend({
this.select_genome.disable();
this.select_extension.disable();
}
this.$info_progress.show();
this.$el.removeClass().addClass("upload-row");
if (status == "success") {
this.$el.addClass("success");
this.$percentage.html("100%");
}
if (status == "error") {
} else if (status == "error") {
this.$el.addClass("danger");
this.$info_progress.hide();
} else if (status == "warning") {
this.$el.addClass("warning");
this.$info_progress.hide();
}
},
client/galaxy/scripts/mvc/upload/default/default-view.js
+48 -36
View File
@@ -104,7 +104,7 @@ export default Backbone.View.extend({
// file upload
this.uploadbox = this.$uploadbox.uploadbox({
url: this.app.options.nginx_upload_path,
url: this.app.options.upload_path,
announce: function(index, file) {
self._eventAnnounce(index, file);
},
@@ -120,6 +120,9 @@ export default Backbone.View.extend({
error: function(index, message) {
self._eventError(index, message);
},
warning: function(index, message) {
self._eventWarning(index, message);
},
complete: function() {
self._eventComplete();
},
@@ -247,7 +250,7 @@ export default Backbone.View.extend({
/** Progress */
_eventProgress: function(index, percentage) {
var it = this.collection.get(index);
it.set("percentage", percentage);
it.set({ percentage: percentage, status: "running", info: "" });
this.ui_button.model.set("percentage", this._uploadPercentage(percentage, it.get("file_size")));
},
@@ -263,6 +266,12 @@ export default Backbone.View.extend({
Galaxy.currHistoryPanel.refreshContents();
},
/** Warning */
_eventWarning: function(index, message) {
var it = this.collection.get(index);
it.set({ status: "warning", info: message });
},
/** Error */
_eventError: function(index, message) {
var it = this.collection.get(index);
@@ -363,7 +372,10 @@ export default Backbone.View.extend({
this._uploadFtp();
// queue remaining files
this.uploadbox.start();
this.uploadbox.start({
id: Galaxy.user.id,
chunk_upload_size: this.app.options.chunk_upload_size
});
this.render();
}
},
@@ -430,7 +442,7 @@ export default Backbone.View.extend({
if (list.length > 0) {
$.uploadpost({
data: this.app.toData(list),
url: this.app.options.nginx_upload_path,
url: this.app.options.upload_path,
success: function(message) {
_.each(list, model => {
self._eventSuccess(model.id);
@@ -452,37 +464,37 @@ export default Backbone.View.extend({
/** Template */
_template: function() {
return (
'<div class="upload-view-default">' +
'<div class="upload-top">' +
'<h6 class="upload-top-info"/>' +
"</div>" +
'<div class="upload-box">' +
'<div class="upload-helper"><i class="fa fa-files-o"/>Drop files here</div>' +
'<table class="upload-table ui-table-striped" style="display: none;">' +
"<thead>" +
"<tr>" +
"<th>Name</th>" +
"<th>Size</th>" +
"<th>Type</th>" +
"<th>Genome</th>" +
"<th>Settings</th>" +
"<th>Status</th>" +
"<th/>" +
"</tr>" +
"</thead>" +
"<tbody/>" +
"</table>" +
"</div>" +
'<div class="upload-footer">' +
'<span class="upload-footer-title">Type (set all):</span>' +
'<span class="upload-footer-extension"/>' +
'<span class="upload-footer-extension-info upload-icon-button fa fa-search"/> ' +
'<span class="upload-footer-title">Genome (set all):</span>' +
'<span class="upload-footer-genome"/>' +
"</div>" +
'<div class="upload-buttons"/>' +
"</div>"
);
return `<div class="upload-view-default">
<div class="upload-top">
<h6 class="upload-top-info"/>
</div>
<div class="upload-box">
<div class="upload-helper">
<i class="fa fa-files-o"/>Drop files here
</div>
<table class="upload-table ui-table-striped" style="display: none;">
<thead>
<tr>
<th>Name</th>
<th>Size</th>
<th>Type</th>
<th>Genome</th>
<th>Settings</th>
<th>Status</th>
<th/>
</tr>
</thead>
<tbody/>
</table>
</div>
<div class="upload-footer">
<span class="upload-footer-title">Type (set all):</span>
<span class="upload-footer-extension"/>
<span class="upload-footer-extension-info upload-icon-button fa fa-search"/>
<span class="upload-footer-title">Genome (set all):</span>
<span class="upload-footer-genome"/>
</div>
<div class="upload-buttons"/>
</div>`;
}
});
client/galaxy/scripts/mvc/upload/upload-view.js
-1
View File
@@ -9,7 +9,6 @@ import UploadViewComposite from "mvc/upload/composite/composite-view";
import UploadViewCollection from "mvc/upload/collection/collection-view";
export default Backbone.View.extend({
options: {
nginx_upload_path: "",
ftp_upload_site: "n/a",
default_genome: "?",
default_extension: "auto",
client/galaxy/scripts/utils/config.js
+12 -9
View File
@@ -1,6 +1,10 @@
import * as _ from "libs/underscore";
import * as Backbone from "libs/backbone";
import util_mod from "viz/trackster/util";
import config_mod from "utils/config";
/* global $ */
/* global Galaxy */
/**
* A configuration setting. Currently key is used as id.
*/
@@ -307,23 +311,22 @@ var ConfigSettingCollectionView = Backbone.View.extend({
// No propagation to avoid triggering document click (and tip hiding) above.
e.stopPropagation();
});
var // Icon for setting a new random color; behavior set below.
new_color_icon = $("<a href='javascript:void(0)'/>")
// Icon for setting a new random color; behavior set below.
var new_color_icon = $("<a href='javascript:void(0)'/>")
.addClass("icon-button arrow-circle")
.appendTo(container_div)
.attr("title", "Set new random color")
.tooltip();
var // Color picker in tool tip style.
tip = $("<div class='tooltip right' style='position: absolute;' />")
// Color picker in tool tip style.
var tip = $("<div class='tooltip right' style='position: absolute;' />")
.appendTo(container_div)
.hide();
var // Inner div for padding purposes
tip_inner = $("<div class='tooltip-inner' style='text-align: inherit'></div>").appendTo(tip);
// Inner div for padding purposes
var tip_inner = $("<div class='tooltip-inner' style='text-align: inherit'></div>").appendTo(tip);
var tip_arrow = $("<div class='tooltip-arrow'></div>").appendTo(tip);
$("<div class='tooltip-arrow'></div>").appendTo(tip);
var farb_obj = $.farbtastic(tip_inner, {
width: 100,
client/galaxy/scripts/utils/uploadbox.js
+227 -93
View File
@@ -6,81 +6,29 @@
jQuery.event.props.push("dataTransfer");
/**
Posts file data to the API
xhr request helper
*/
$.uploadpost = config => {
// parse options
var _uploadrequest = config => {
var cnf = $.extend(
{},
{
data: {},
success: function() {},
error: function() {},
progress: function() {},
url: null,
maxfilesize: 2048,
error_filesize: "File exceeds 2GB. Please use a FTP client.",
error_default: "Please make sure the file is available.",
error_server: "Upload request failed.",
error_login: "Uploads require you to log in."
error_login: "Uploads require you to log in.",
error_retry: "Waiting for server to resume..."
},
config
);
// link data
var data = cnf.data;
// check errors
if (data.error_message) {
cnf.error(data.error_message);
return;
}
// construct form data
var form = new FormData();
for (var key in data.payload) {
form.append(key, data.payload[key]);
}
// add files to submission
var sizes = 0;
for (var key in data.files) {
var d = data.files[key];
form.append(d.name, d.file, d.file.name);
sizes += d.file.size;
}
// check file size, unless it's an ftp file
if (sizes > 1048576 * cnf.maxfilesize) {
cnf.error(cnf.error_filesize);
return;
}
// prepare request
console.debug(cnf);
var xhr = new XMLHttpRequest();
xhr.open("POST", cnf.url, true);
xhr.setRequestHeader("Accept", "application/json");
xhr.setRequestHeader("Cache-Control", "no-cache");
xhr.setRequestHeader("X-Requested-With", "XMLHttpRequest");
// captures state changes
xhr.setRequestHeader("Accept", "application/json");
xhr.onreadystatechange = () => {
// check for request completed, server connection closed
if (xhr.readyState == xhr.DONE) {
// parse response
var response = null;
var extra_info = "";
if (xhr.responseText) {
try {
response = jQuery.parseJSON(xhr.responseText);
extra_info = response.err_msg;
} catch (e) {
response = xhr.responseText;
extra_info = response;
}
}
// pass any error to the error option
if (xhr.status < 200 || xhr.status > 299) {
if ([502, 0].indexOf(xhr.status) !== -1 && cnf.warning) {
cnf.warning(cnf.error_retry);
} else if (xhr.status < 200 || xhr.status > 299) {
var text = xhr.statusText;
if (xhr.status == 403) {
text = cnf.error_login;
@@ -89,27 +37,191 @@
} else if (!text) {
text = cnf.error_default;
}
cnf.error(`${text} (${xhr.status}). ${extra_info}`);
cnf.error(`${text} (${xhr.status})`);
} else {
var response = null;
if (xhr.responseText) {
try {
response = jQuery.parseJSON(xhr.responseText);
} catch (e) {
response = xhr.responseText;
}
}
cnf.success(response);
}
}
};
xhr.upload.addEventListener("progress", cnf.progress, false);
xhr.send(cnf.data);
};
// prepare upload progress
xhr.upload.addEventListener(
"progress",
e => {
/**
Posts chunked files to the API.
*/
$.uploadchunk = function(config) {
// parse options
var cnf = $.extend(
{},
{
data: {},
success: () => {},
error: () => {},
warning: () => {},
progress: () => {},
attempts: 70000,
timeout: 5000,
url: null,
error_file: "File not provided.",
error_attempt: "Maximum number of attempts reached.",
error_tool: "Tool submission failed."
},
config
);
// initial validation
var data = cnf.data;
if (data.error_message) {
cnf.error(data.error_message);
return;
}
var file_data = data.files && data.files[0];
if (!file_data) {
cnf.error(cnf.error_file);
return;
}
var file = file_data.file;
var attempts = cnf.attempts;
var session_id = `${cnf.session.id}-${new Date().valueOf()}-${file.size}`;
var chunk_size = cnf.session.chunk_upload_size;
console.debug(`Starting chunked uploads [size=${chunk_size}].`);
// chunk processing helper
function process(start) {
start = start || 0;
var slicer = file.mozSlice || file.webkitSlice || file.slice;
if (!slicer) {
cnf.error("Browser does not support chunked uploads.");
return;
}
var end = Math.min(start + chunk_size, file.size);
var size = file.size;
console.debug(`Submitting chunk at ${start} bytes...`);
var form = new FormData();
form.append("session_id", session_id);
form.append("session_start", start);
form.append("session_chunk", slicer.bind(file)(start, end));
_uploadrequest({
url: `${Galaxy.root}api/uploads`,
data: form,
success: upload_response => {
var new_start = start + chunk_size;
if (new_start < size) {
attempts = cnf.attempts;
process(new_start);
} else {
console.debug("Upload completed.");
data.payload.inputs = JSON.parse(data.payload.inputs);
data.payload.inputs["files_0|file_data"] = {
session_id: session_id,
name: file.name
};
data.payload.inputs = JSON.stringify(data.payload.inputs);
$.ajax({
url: `${Galaxy.root}api/tools`,
method: "POST",
data: data.payload,
success: tool_response => {
cnf.success(tool_response);
},
error: tool_response => {
var err_msg =
tool_response && tool_response.responseJSON && tool_response.responseJSON.err_msg;
cnf.error(err_msg || cnf.error_tool);
}
});
}
},
warning: upload_response => {
if (--attempts > 0) {
console.debug("Retrying last chunk...");
cnf.warning(upload_response);
setTimeout(() => process(start), cnf.timeout);
} else {
console.debug(cnf.error_attempt);
cnf.error(cnf.error_attempt);
}
},
error: upload_response => {
console.debug(upload_response);
cnf.error(upload_response);
},
progress: e => {
if (e.lengthComputable) {
cnf.progress(Math.min(Math.round((start + e.loaded) * 100 / file.size), 100));
}
}
});
}
// initiate processing queue for chunks
process();
};
/**
Posts multiple files without chunking to the API.
*/
$.uploadpost = function(config) {
var cnf = $.extend(
{},
{
data: {},
success: () => {},
error: () => {},
progress: () => {},
url: null,
maxfilesize: 1048576 * 2048,
error_filesize: "File exceeds 2GB. Please use a FTP client."
},
config
);
var data = cnf.data;
if (data.error_message) {
cnf.error(data.error_message);
return;
}
// construct form data
var form = new FormData();
for (let key in data.payload) {
form.append(key, data.payload[key]);
}
// add files to submission
var sizes = 0;
for (let key in data.files) {
var d = data.files[key];
form.append(d.name, d.file, d.file.name);
sizes += d.file.size;
}
// check file size, unless it's an ftp file
if (sizes > cnf.maxfilesize) {
cnf.error(cnf.error_filesize);
return;
}
// submit request
_uploadrequest({
url: cnf.url,
data: form,
success: cnf.success,
error: cnf.error,
progress: e => {
if (e.lengthComputable) {
cnf.progress(Math.round(e.loaded * 100 / e.total));
}
},
false
);
// send request
Galaxy.emit.debug("uploadbox::uploadpost()", "Posting following data.", cnf);
xhr.send(form);
}
});
};
/**
@@ -121,9 +233,9 @@
var opts = $.extend(
{},
{
ondragover: function() {},
ondragleave: function() {},
onchange: function() {},
ondragover: () => {},
ondragleave: () => {},
onchange: () => {},
multiple: false
},
options
@@ -132,7 +244,7 @@
// append hidden upload field
var $input = $(`<input type="file" style="display: none" ${(opts.multiple && "multiple") || ""}/>`);
el.append(
$input.change(function(e) {
$input.change(e => {
opts.onchange(e.target.files);
$(this).val("");
})
@@ -157,7 +269,7 @@
// exports
return {
dialog: function() {
dialog: () => {
$input.trigger("click");
}
};
@@ -171,16 +283,17 @@
var opts = $.extend(
{},
{
dragover: function() {},
dragleave: function() {},
announce: function(d) {},
initialize: function(d) {},
progress: function(d, m) {},
success: function(d, m) {},
error: function(d, m) {
dragover: () => {},
dragleave: () => {},
announce: d => {},
initialize: d => {},
progress: (d, m) => {},
success: (d, m) => {},
warning: (d, m) => {},
error: (d, m) => {
alert(m);
},
complete: function() {}
complete: () => {}
},
options
);
@@ -188,6 +301,9 @@
// file queue
var queue = {};
// session options
var session = null;
// queue index/length counter
var queue_index = 0;
var queue_length = 0;
@@ -199,7 +315,10 @@
// element
var uploadinput = $(this).uploadinput({
multiple: true,
onchange: function(files) {
onchange: files => {
_.each(files, file => {
file.chunk_mode = true;
});
add(files);
},
ondragover: options.ondragover,
@@ -252,7 +371,7 @@
// get an identifier from the queue
var index = -1;
for (var key in queue) {
for (let key in queue) {
index = key;
break;
}
@@ -264,18 +383,32 @@
remove(index);
// create and submit data
$.uploadpost({
var submitter = $.uploadpost;
if (
file.chunk_mode &&
session &&
session.id &&
session.chunk_upload_size &&
session.chunk_upload_size > 0
) {
submitter = $.uploadchunk;
}
submitter({
url: opts.url,
data: opts.initialize(index),
success: function(message) {
session: session,
success: message => {
opts.success(index, message);
process();
},
error: function(message) {
warning: message => {
opts.warning(index, message);
},
error: message => {
opts.error(index, message);
process();
},
progress: function(percentage) {
progress: percentage => {
opts.progress(index, percentage);
}
});
@@ -298,7 +431,8 @@
}
// initiate upload process
function start() {
function start(_session) {
session = _session;
if (!queue_running) {
queue_running = true;
process();
client/galaxy/scripts/viz/trackster.js
+170 -183
View File
@@ -1,15 +1,15 @@
import _l from "utils/localization";
/**
* Top-level trackster code, used for creating/loading visualizations and user interface elements.
*/
//import * as $ from 'jquery';
import _l from "utils/localization";
import * as _ from "libs/underscore";
import * as Backbone from "libs/backbone";
import tracks from "viz/trackster/tracks";
import visualization from "viz/visualization";
import mod_icon_btn from "mvc/ui/icon-button";
import IconButton from "mvc/ui/icon-button";
import query_string from "utils/query-string-parsing";
import GridView from "mvc/grid/grid-view";
import mod_utils from "utils/utils";
import Utils from "utils/utils";
import "libs/jquery/jquery.event.drag";
import "libs/jquery/jquery.event.hover";
import "libs/jquery/jquery.mousewheel";
@@ -20,44 +20,36 @@ import "libs/jquery/jquery.form";
import "libs/jquery/jquery.rating";
import "ui/editable-text";
// trackster global variables
var ui = null;
var view = null;
var browser_router = null;
/**
* Base Object/Model for inhertiance.
*/
var Base = function() {
if (this.initialize) {
this.initialize.apply(this, arguments);
}
};
Base.extend = Backbone.Model.extend;
/* global Galaxy */
/* global $ */
/**
* User interface controls for trackster
*/
var TracksterUI = Base.extend({
initialize: function(baseURL) {
mod_utils.cssLoadFile("static/style/jquery.rating.css");
mod_utils.cssLoadFile("static/style/autocomplete_tagging.css");
mod_utils.cssLoadFile("static/style/jquery-ui/smoothness/jquery-ui.css");
mod_utils.cssLoadFile("static/style/library.css");
mod_utils.cssLoadFile("static/style/trackster.css");
class TracksterUI extends Backbone.Model {
constructor(options) {
super(options);
}
initialize(baseURL) {
this.baseURL = baseURL;
},
Utils.cssLoadFile("static/style/jquery.rating.css");
Utils.cssLoadFile("static/style/autocomplete_tagging.css");
Utils.cssLoadFile("static/style/jquery-ui/smoothness/jquery-ui.css");
Utils.cssLoadFile("static/style/library.css");
Utils.cssLoadFile("static/style/trackster.css");
}
/**
* Save visualization, returning a Deferred object for the remote call to save.
*/
save_viz: function() {
save_viz() {
// show dialog
Galaxy.modal.show({ title: "Saving...", body: "progress" });
// Save bookmarks.
var bookmarks = [];
$(".bookmark").each(function() {
$(".bookmark").each(() => {
bookmarks.push({
position: $(this)
.children(".position")
@@ -69,14 +61,16 @@ var TracksterUI = Base.extend({
});
// FIXME: give unique IDs to Drawables and save overview as ID.
var overview_track_name = view.overview_drawable ? view.overview_drawable.config.get_value("name") : null;
var overview_track_name = this.view.overview_drawable
? this.view.overview_drawable.config.get_value("name")
: null;
var viz_config = {
view: view.to_dict(),
view: this.view.to_dict(),
viewport: {
chrom: view.chrom,
start: view.low,
end: view.high,
chrom: this.view.chrom,
start: this.view.low,
end: this.view.high,
overview: overview_track_name
},
bookmarks: bookmarks
@@ -88,17 +82,17 @@ var TracksterUI = Base.extend({
type: "POST",
dataType: "json",
data: {
id: view.vis_id,
title: view.config.get_value("name"),
dbkey: view.dbkey,
id: this.view.vis_id,
title: this.view.config.get_value("name"),
dbkey: this.view.dbkey,
type: "trackster",
vis_json: JSON.stringify(viz_config)
}
})
.success(vis_info => {
Galaxy.modal.hide();
view.vis_id = vis_info.vis_id;
view.has_changes = false;
this.view.vis_id = vis_info.vis_id;
this.view.has_changes = false;
// Needed to set URL when first saving a visualization.
window.history.pushState({}, "", vis_info.url + window.location.hash);
@@ -109,29 +103,27 @@ var TracksterUI = Base.extend({
title: _l("Could Not Save"),
body: "Could not save visualization. Please try again later.",
buttons: {
Cancel: function() {
Cancel: () => {
Galaxy.modal.hide();
}
}
});
});
},
}
/**
* Create button menu
*/
createButtonMenu: function() {
var self = this;
var menu = mod_icon_btn.create_icon_buttons_menu(
createButtonMenu() {
var menu = IconButton.create_icon_buttons_menu(
[
{
icon_class: "plus-button",
title: _l("Add tracks"),
on_click: function() {
visualization.select_datasets({ dbkey: view.dbkey }, new_tracks => {
on_click: () => {
visualization.select_datasets({ dbkey: this.view.dbkey }, new_tracks => {
_.each(new_tracks, track => {
view.add_drawable(tracks.object_from_template(track, view, view));
this.view.add_drawable(tracks.object_from_template(track, this.view, this.view));
});
});
}
@@ -139,9 +131,9 @@ var TracksterUI = Base.extend({
{
icon_class: "block--plus",
title: _l("Add group"),
on_click: function() {
view.add_drawable(
new tracks.DrawableGroup(view, view, {
on_click: () => {
this.view.add_drawable(
new tracks.DrawableGroup(this.view, this.view, {
name: "New Group"
})
);
@@ -150,7 +142,7 @@ var TracksterUI = Base.extend({
{
icon_class: "bookmarks",
title: _l("Bookmarks"),
on_click: function() {
on_click: () => {
// HACK -- use style to determine if panel is hidden and hide/show accordingly.
window.force_right_panel($("div#right").css("right") == "0px" ? "hide" : "show");
}
@@ -158,22 +150,22 @@ var TracksterUI = Base.extend({
{
icon_class: "globe",
title: _l("Circster"),
on_click: function() {
window.location = `${self.baseURL}visualization/circster?id=${view.vis_id}`;
on_click: () => {
window.location = `${this.baseURL}visualization/circster?id=${this.view.vis_id}`;
}
},
{
icon_class: "disk--arrow",
title: _l("Save"),
on_click: function() {
self.save_viz();
on_click: () => {
this.save_viz();
}
},
{
icon_class: "cross-circle",
title: _l("Close"),
on_click: function() {
self.handle_unsaved_changes(view);
on_click: () => {
this.handle_unsaved_changes(this.view);
}
}
],
@@ -184,12 +176,12 @@ var TracksterUI = Base.extend({
this.buttonMenu = menu;
return menu;
},
}
/**
* Add bookmark.
*/
add_bookmark: function(position, annotation, editable) {
add_bookmark(position, annotation, editable) {
// Create HTML.
var bookmarks_container = $("#right .unified-panel-body");
@@ -201,11 +193,12 @@ var TracksterUI = Base.extend({
.addClass("position")
.appendTo(new_bookmark);
var position_link = $("<a href=''/>")
//position_link
$("<a href=''/>")
.text(position)
.appendTo(position_div)
.click(() => {
view.go_to(position);
this.view.go_to(position);
return false;
});
@@ -222,11 +215,12 @@ var TracksterUI = Base.extend({
// Remove bookmark.
new_bookmark.slideUp("fast");
new_bookmark.remove();
view.has_changes = true;
this.view.has_changes = true;
return false;
});
var delete_icon = $("<a href=''/>")
// delete_icon
$("<a href=''/>")
.addClass("icon-button delete")
.appendTo(delete_icon_container);
@@ -239,56 +233,50 @@ var TracksterUI = Base.extend({
.addClass("annotation");
}
view.has_changes = true;
this.view.has_changes = true;
return new_bookmark;
},
}
/**
* Create a complete Trackster visualization. Returns view.
*/
create_visualization: function(view_config, viewport_config, drawables_config, bookmarks_config, editable) {
create_visualization(view_config, viewport_config, drawables_config, bookmarks_config, editable) {
// Create view.
var self = this;
this.view = new tracks.TracksterView(_.extend(view_config, { header: false }));
this.view.editor = true;
view = new tracks.TracksterView(_.extend(view_config, { header: false }));
view.editor = true;
$.when(view.load_chroms_deferred).then(chrom_info => {
$.when(this.view.load_chroms_deferred).then(chrom_info => {
var overview_drawable_name = null;
// Viewport config.
if (viewport_config) {
var chrom = viewport_config.chrom;
var start = viewport_config.start;
var end = viewport_config.end;
var overview_drawable_name = viewport_config.overview;
overview_drawable_name = viewport_config.overview;
if (chrom && start !== undefined && end) {
view.change_chrom(chrom, start, end);
this.view.change_chrom(chrom, start, end);
} else {
// No valid viewport, so use first chromosome.
view.change_chrom(chrom_info[0].chrom);
this.view.change_chrom(chrom_info[0].chrom);
}
} else {
// No viewport, so use first chromosome.
view.change_chrom(chrom_info[0].chrom);
this.view.change_chrom(chrom_info[0].chrom);
}
// Add drawables to view.
if (drawables_config) {
// FIXME: can from_dict() be used to create view and add drawables?
var drawable_config;
var drawable_type;
var drawable;
for (var i = 0; i < drawables_config.length; i++) {
view.add_drawable(tracks.object_from_template(drawables_config[i], view, view));
for (let i = 0; i < drawables_config.length; i++) {
this.view.add_drawable(tracks.object_from_template(drawables_config[i], this.view, this.view));
}
}
// Set overview.
var overview_drawable;
for (var i = 0; i < view.drawables.length; i++) {
if (view.drawables[i].config.get_value("name") === overview_drawable_name) {
view.set_overview(view.drawables[i]);
for (let i = 0; i < this.view.drawables.length; i++) {
if (this.view.drawables[i].config.get_value("name") === overview_drawable_name) {
this.view.set_overview(this.view.drawables[i]);
break;
}
}
@@ -296,34 +284,35 @@ var TracksterUI = Base.extend({
// Load bookmarks.
if (bookmarks_config) {
var bookmark;
for (var i = 0; i < bookmarks_config.length; i++) {
for (let i = 0; i < bookmarks_config.length; i++) {
bookmark = bookmarks_config[i];
self.add_bookmark(bookmark["position"], bookmark["annotation"], editable);
this.add_bookmark(bookmark.position, bookmark.annotation, editable);
}
}
// View has no changes as of yet.
view.has_changes = false;
this.view.has_changes = false;
});
// Final initialization.
this.set_up_router({ view: view });
this.set_up_router({ view: this.view });
return view;
},
// TODO: This is hopefully not necessary anymore, since we're using the instance view. Do it for compatibility for now.
return this.view;
}
/**
* Set up location router to use hashes as track browser locations.
*/
set_up_router: function(options) {
set_up_router(options) {
new visualization.TrackBrowserRouter(options);
Backbone.history.start();
},
}
/**
* Set up keyboard navigation for a visualization.
*/
init_keyboard_nav: function(view) {
init_keyboard_nav(view) {
// Keyboard navigation. Scroll ~7% of height when scrolling up/down.
$(document).keyup(e => {
// Do not navigate if arrow keys used in input element.
@@ -337,39 +326,38 @@ var TracksterUI = Base.extend({
view.move_fraction(0.25);
break;
case 38:
var change = Math.round(view.viewport_container.height() / 15.0);
// var change = Math.round(view.viewport_container.height() / 15.0);
view.viewport_container.scrollTop(view.viewport_container.scrollTop() - 20);
break;
case 39:
view.move_fraction(-0.25);
break;
case 40:
var change = Math.round(view.viewport_container.height() / 15.0);
// var change = Math.round(view.viewport_container.height() / 15.0);
view.viewport_container.scrollTop(view.viewport_container.scrollTop() + 20);
break;
}
});
},
}
/**
* Handle unsaved changes in visualization.
*/
handle_unsaved_changes: function(view) {
handle_unsaved_changes(view) {
if (view.has_changes) {
var self = this;
Galaxy.modal.show({
title: _l("Close visualization"),
body: "There are unsaved changes to your visualization which will be lost if you do not save them.",
buttons: {
Cancel: function() {
Cancel: () => {
Galaxy.modal.hide();
},
"Leave without Saving": function() {
"Leave without Saving": () => {
$(window).off("beforeunload");
window.location = `${Galaxy.root}visualization`;
},
Save: function() {
$.when(self.save_viz()).then(() => {
Save: () => {
$.when(this.save_viz()).then(() => {
window.location = `${Galaxy.root}visualization`;
});
}
@@ -379,22 +367,25 @@ var TracksterUI = Base.extend({
window.location = `${Galaxy.root}visualization`;
}
}
});
}
var TracksterView = Backbone.View.extend({
class TracksterUIView extends Backbone.View {
constructor(options) {
super(options);
}
// initalize trackster
initialize: function() {
initialize() {
// load ui
ui = new TracksterUI(Galaxy.root);
this.ui = new TracksterUI(Galaxy.root);
// create button menu
ui.createButtonMenu();
this.ui.createButtonMenu();
// attach the button menu to the panel header and float it left
ui.buttonMenu.$el.attr("style", "float: right");
this.ui.buttonMenu.$el.attr("style", "float: right");
// add to center panel
$("#center .unified-panel-header-inner").append(ui.buttonMenu.$el);
$("#center .unified-panel-header-inner").append(this.ui.buttonMenu.$el);
// configure right panel
$("#right .unified-panel-title").append("Bookmarks");
@@ -404,7 +395,7 @@ var TracksterView = Backbone.View.extend({
// resize view when showing/hiding right panel (bookmarks for now).
$("#right-border").click(() => {
view.resize_window();
this.ui.view.resize_window();
});
// hide right panel
@@ -418,10 +409,9 @@ var TracksterView = Backbone.View.extend({
} else {
this.view_new();
}
},
}
choose_existing_or_new: function() {
var self = this;
choose_existing_or_new() {
var dbkey = query_string.get("dbkey");
var listTracksParams = {};
@@ -442,21 +432,21 @@ var TracksterView = Backbone.View.extend({
body: `<p><ul style='list-style: disc inside none'>You can add this dataset as:<li>a new track to one of your existing, saved Trackster sessions if they share the genome build: <b>${dbkey ||
"Not available."}</b></li><li>or create a new session with this dataset as the only track</li></ul></p>`,
buttons: {
Cancel: function() {
Cancel: () => {
window.location = `${Galaxy.root}visualizations/list`;
},
"View in saved visualization": function() {
self.view_in_saved(dataset_params);
"View in saved visualization": () => {
this.view_in_saved(dataset_params);
},
"View in new visualization": function() {
self.view_new();
"View in new visualization": () => {
this.view_new();
}
}
});
},
}
// view
view_in_saved: function(dataset_params) {
view_in_saved(dataset_params) {
var tracks_grid = new GridView({
url_base: `${Galaxy.root}visualization/list_tracks`,
embedded: true
@@ -465,28 +455,28 @@ var TracksterView = Backbone.View.extend({
title: _l("Add Data to Saved Visualization"),
body: tracks_grid.$el,
buttons: {
Cancel: function() {
Cancel: () => {
window.location = `${Galaxy.root}visualizations/list`;
},
"Add to visualization": function() {
$(parent.document)
"Add to visualization": () => {
$(window.parent.document)
.find("input[name=id]:checked")
.each(function() {
.each(() => {
dataset_params.id = $(this).val();
window.location = `${Galaxy.root}visualization/trackster?${$.param(dataset_params)}`;
});
}
}
});
},
}
// view
view_existing: function() {
view_existing() {
// get config
var viz_config = window.galaxy_config.app.viz_config;
// view
view = ui.create_visualization(
this.ui.create_visualization(
{
container: $("#center .unified-panel-body"),
name: viz_config.title,
@@ -501,31 +491,28 @@ var TracksterView = Backbone.View.extend({
// initialize editor
this.init_editor();
},
}
// view
view_new: function() {
// reference this
var self = this;
view_new() {
// ajax
$.ajax({
url: `${Galaxy.root}api/genomes?chrom_info=True`,
data: {},
error: function() {
error: () => {
alert("Couldn't create new browser.");
},
success: function(response) {
success: response => {
// show dialog
Galaxy.modal.show({
title: _l("New Visualization"),
body: self.template_view_new(response),
body: this.template_view_new(response),
buttons: {
Cancel: function() {
Cancel: () => {
window.location = `${Galaxy.root}visualizations/list`;
},
Create: function() {
self.create_browser($("#new-title").val(), $("#new-dbkey").val());
Create: () => {
this.create_browser($("#new-title").val(), $("#new-dbkey").val());
Galaxy.modal.hide();
}
}
@@ -548,10 +535,10 @@ var TracksterView = Backbone.View.extend({
$("#overlay").css("overflow", "auto");
}
});
},
}
// new browser form
template_view_new: function(response) {
template_view_new(response) {
// start template
var html =
'<form id="new-browser-form" action="javascript:void(0);" method="post" onsubmit="return false;">' +
@@ -568,7 +555,7 @@ var TracksterView = Backbone.View.extend({
'<select name="dbkey" id="new-dbkey">';
// add dbkeys
for (var i = 0; i < response.length; i++) {
for (let i = 0; i < response.length; i++) {
html += `<option value="${response[i][1]}">${response[i][0]}</option>`;
}
@@ -579,13 +566,48 @@ var TracksterView = Backbone.View.extend({
// return
return html;
},
}
// initialization for editor-specific functions.
init_editor() {
// set title
$("#center .unified-panel-title").text(`${this.ui.view.config.get_value("name")} (${this.ui.view.dbkey})`);
// add dataset
if (window.galaxy_config.app.add_dataset)
$.ajax({
url: `${Galaxy.root}api/datasets/${window.galaxy_config.app.add_dataset}`,
data: { hda_ldda: "hda", data_type: "track_config" },
dataType: "json",
success: track_data => {
this.ui.view.add_drawable(tracks.object_from_template(track_data, this.ui.view, this.ui.view));
}
});
// initialize icons
$("#add-bookmark-button").click(() => {
// add new bookmark.
var position = `${this.ui.view.chrom}:${this.ui.view.low}-${this.ui.view.high}`;
var annotation = "Bookmark description";
return this.ui.add_bookmark(position, annotation, true);
});
// initialize keyboard
this.ui.init_keyboard_nav(this.ui.view);
$(window).on("beforeunload", () => {
if (this.ui.view.has_changes) {
return "There are unsaved changes to your visualization that will be lost if you leave this page.";
}
});
}
// create
create_browser: function(name, dbkey) {
create_browser(name, dbkey) {
$(document).trigger("convert_to_values");
view = ui.create_visualization(
this.ui.create_visualization(
{
container: $("#center .unified-panel-body"),
name: name,
@@ -598,46 +620,11 @@ var TracksterView = Backbone.View.extend({
this.init_editor();
// modify view setting
view.editor = true;
},
// initialization for editor-specific functions.
init_editor: function() {
// set title
$("#center .unified-panel-title").text(`${view.config.get_value("name")} (${view.dbkey})`);
// add dataset
if (window.galaxy_config.app.add_dataset)
$.ajax({
url: `${Galaxy.root}api/datasets/${window.galaxy_config.app.add_dataset}`,
data: { hda_ldda: "hda", data_type: "track_config" },
dataType: "json",
success: function(track_data) {
view.add_drawable(tracks.object_from_template(track_data, view, view));
}
});
// initialize icons
$("#add-bookmark-button").click(() => {
// add new bookmark.
var position = `${view.chrom}:${view.low}-${view.high}`;
var annotation = "Bookmark description";
return ui.add_bookmark(position, annotation, true);
});
// initialize keyboard
ui.init_keyboard_nav(view);
$(window).on("beforeunload", () => {
if (view.has_changes) {
return "There are unsaved changes to your visualization that will be lost if you leave this page.";
}
});
this.ui.view.editor = true;
}
});
}
export default {
TracksterUI: TracksterUI,
GalaxyApp: TracksterView
GalaxyApp: TracksterUIView
};
client/galaxy/scripts/viz/trackster/filters.js
+3 -5
View File
@@ -151,7 +151,7 @@ var NumberFilter = function(obj_dict) {
.addClass("elt-label")
.appendTo(filter.parent_div);
var name_span = $("<span/>")
$("<span/>")
.addClass("slider-name")
.text(`${filter.name} `)
.appendTo(filter_label);
@@ -341,9 +341,6 @@ extend(NumberFilter.prototype, {
return true;
}
// Keep value function.
var filter = this;
// Do filtering.
var to_filter = element[this.index];
if (to_filter instanceof Array) {
@@ -482,7 +479,7 @@ var FiltersManager = function(track, obj_dict) {
filter.height_icon.addClass("active").show();
}
} else {
console.log("ERROR: unsupported filter: ", name, type);
console.log("ERROR: unsupported filter: ", filters_dict[i]);
}
}
@@ -656,6 +653,7 @@ extend(FiltersManager.prototype, {
// Remove current filter.
filters = filters.slice(1);
// DBTODO: This will never work, run_tool_url doesn't exist?
$.getJSON(run_tool_url, url_params, response => {
if (response.error) {
// General error.
client/galaxy/scripts/viz/trackster/painters.js
+518 -535
View File
File diff suppressed because it is too large Load Diff
client/galaxy/scripts/viz/trackster/slotting.js
+5 -5
View File
@@ -45,7 +45,7 @@ extend(FeatureSlotter.prototype, {
var draw_end = Math.ceil(feature[2] * this.w_scale);
var f_name = feature[3];
var text_align;
//var text_align;
// Update start, end drawing locations to include feature name.
// Try to put the name on the left, if not, put on right.
@@ -55,10 +55,10 @@ extend(FeatureSlotter.prototype, {
var text_len = this.measureText(f_name).width + (LABEL_SPACING + PACK_SPACING);
if (draw_start - text_len >= 0) {
draw_start -= text_len;
text_align = "left";
//text_align = "left";
} else {
draw_end += text_len;
text_align = "right";
//text_align = "right";
}
}
@@ -130,7 +130,7 @@ extend(FeatureSlotter.prototype, {
// Loop through features to (a) find those that are not yet slotted and (b) update
// those that are slotted if new information is availabe. For (a), features already
// slotted (based on slotting from other tiles) will retain their current slot.
for (var i = 0, len = features.length; i < len; i++) {
for (let i = 0, len = features.length; i < len; i++) {
feature = features[i];
feature_uid = feature[0];
var slotted_info = this.slots[feature_uid];
@@ -163,7 +163,7 @@ extend(FeatureSlotter.prototype, {
// Slot unslotted features.
// Do slotting.
for (var i = 0, len = undone.length; i < len; i++) {
for (let i = 0, len = undone.length; i < len; i++) {
feature = features[undone[i]];
feature_uid = feature[0];
var draw_coords = this._get_draw_coords(feature);
client/galaxy/scripts/viz/trackster/tracks.js
+167 -173
View File
@@ -1,5 +1,6 @@
import _l from "utils/localization";
import * as _ from "libs/underscore";
import * as Backbone from "libs/backbone";
import visualization from "viz/visualization";
import viz_views from "viz/viz_views";
import util from "viz/trackster/util";
@@ -13,6 +14,9 @@ import bbi from "viz/bbi-data-manager";
import "ui/editable-text";
var extend = _.extend;
/* global $ */
/* global Galaxy */
// ---- Web UI specific utilities ----
/**
@@ -24,9 +28,9 @@ var html_elt_js_obj_dict = {};
/**
* Designates an HTML as a container.
*/
var is_container = (element, obj) => {
function is_container(element, obj) {
html_elt_js_obj_dict[element.attr("id")] = obj;
};
}
/**
* Make `element` moveable within parent and sibling elements by dragging `handle` (a selector).
@@ -37,7 +41,7 @@ var is_container = (element, obj) => {
* @param container_selector selector used to identify possible containers for this element
* @param element_js_obj JavaScript object associated with element; used
*/
var moveable = (element, handle_class, container_selector, element_js_obj) => {
function moveable(element, handle_class, container_selector, element_js_obj) {
// HACK: set default value for container selector.
container_selector = ".group";
@@ -48,11 +52,9 @@ var moveable = (element, handle_class, container_selector, element_js_obj) => {
element
.bind("drag", { handle: `.${handle_class}`, relative: true }, function(e, d) {
var element = $(this);
var parent = $(this).parent();
var // Only sorting amongst tracks and groups.
children = parent.children(".track,.group");
var parent = element.parent();
// Only sorting amongst tracks and groups.
var children = parent.children(".track,.group");
var this_obj = html_elt_js_obj_dict[$(this).attr("id")];
var child;
var container;
@@ -144,46 +146,47 @@ var moveable = (element, handle_class, container_selector, element_js_obj) => {
.bind("dragend", function() {
$(this).removeClass("dragging");
});
};
}
/**
* Init constants & functions used throughout trackster.
*/
var // Padding at the top of tracks for error messages
ERROR_PADDING = 20;
var // Maximum number of rows un a slotted track
MAX_FEATURE_DEPTH = 100;
// Padding at the top of tracks for error messages
const ERROR_PADDING = 20;
var // Minimum width for window for squish to be used.
MIN_SQUISH_VIEW_WIDTH = 12000;
// Maximum number of rows un a slotted track
const MAX_FEATURE_DEPTH = 100;
var // Number of pixels per tile, not including left offset.
TILE_SIZE = 400;
// Minimum width for window for squish to be used.
const MIN_SQUISH_VIEW_WIDTH = 12000;
var DEFAULT_DATA_QUERY_WAIT = 5000;
// Number of pixels per tile, not including left offset.
const TILE_SIZE = 400;
var // Maximum number of chromosomes that are selectable at any one time.
MAX_CHROMS_SELECTABLE = 100;
const DEFAULT_DATA_QUERY_WAIT = 5000;
var DATA_ERROR = "Cannot display dataset due to an error. ";
// Maximum number of chromosomes that are selectable at any one time.
const MAX_CHROMS_SELECTABLE = 100;
var DATA_NOCONVERTER = "A converter for this dataset is not installed. Please check your datatypes_conf.xml file.";
const DATA_ERROR = "Cannot display dataset due to an error. ";
var DATA_NONE = "No data for this chrom/contig.";
const DATA_NOCONVERTER = "A converter for this dataset is not installed. Please check your datatypes_conf.xml file.";
var DATA_PENDING =
const DATA_NONE = "No data for this chrom/contig.";
const DATA_PENDING =
"Preparing data. This can take a while for a large dataset. " +
"If the visualization is saved and closed, preparation will continue in the background.";
var DATA_CANNOT_RUN_TOOL = "Tool cannot be rerun: ";
var DATA_LOADING = "Loading data...";
var DATA_OK = "Ready for display";
var TILE_CACHE_SIZE = 10;
var DATA_CACHE_SIZE = 20;
var // Numerical/continuous data display modes.
CONTINUOUS_DATA_MODES = ["Histogram", "Line", "Filled", "Intensity"];
const DATA_CANNOT_RUN_TOOL = "Tool cannot be rerun: ";
//var DATA_LOADING = "Loading data...";
const DATA_OK = "Ready for display";
const TILE_CACHE_SIZE = 10;
//var DATA_CACHE_SIZE = 20;
//
// Numerical/continuous data display modes.
const CONTINUOUS_DATA_MODES = ["Histogram", "Line", "Filled", "Intensity"];
/**
* Round a number to a given number of decimal places.
@@ -248,7 +251,10 @@ var Drawable = function(view, container, obj_dict) {
this.action_icons = {};
// -- Set up drawable configuration. --
this.config = config_mod.ConfigSettingCollection.from_models_and_saved_values(this.config_params, obj_dict.prefs);
this.config = config_mod.ConfigSettingCollection.from_models_and_saved_values(
this.build_config_params(),
obj_dict.prefs
);
// If there's no saved name, use object name.
if (!this.config.get_value("name")) {
@@ -342,6 +348,10 @@ extend(Drawable.prototype, {
}
],
build_config_params: function() {
return this.config_params;
},
config_onchange: function() {},
init: function() {},
@@ -723,7 +733,6 @@ extend(DrawableGroup.prototype, Drawable.prototype, DrawableCollection.prototype
for (i = 0; i < num_drawables; i++) {
drawable = this.drawables[i];
if (drawable.get_type() !== a_type) {
can_composite = false;
break;
}
if (drawable instanceof FeatureTrack) {
@@ -772,16 +781,12 @@ extend(DrawableGroup.prototype, Drawable.prototype, DrawableCollection.prototype
// manager.
//
this.filters_manager.remove_all();
var filters;
var new_filter;
var min;
var max;
for (var filter_name in shared_filters) {
filters = shared_filters[filter_name];
let filters = shared_filters[filter_name];
if (filters.length === num_feature_tracks) {
// Add new filter.
// FIXME: can filter.copy() be used?
new_filter = new filters_mod.NumberFilter({
let new_filter = new filters_mod.NumberFilter({
name: filters[0].name,
index: filters[0].index
});
@@ -834,7 +839,7 @@ extend(DrawableGroup.prototype, Drawable.prototype, DrawableCollection.prototype
name: this.config.get_value("name"),
drawables: this.drawables
});
var index = this.container.replace_drawable(this, composite_track, true);
this.container.replace_drawable(this, composite_track, true);
composite_track.request_draw();
},
@@ -981,7 +986,8 @@ var TracksterView = Backbone.View.extend({
this.intro_div = $("<div/>")
.addClass("intro")
.appendTo(this.viewport_container);
var add_tracks_button = $("<div/>")
// Add tracks button
$("<div/>")
.text("Add Datasets to Visualization")
.addClass("action-button")
.appendTo(this.intro_div)
@@ -1270,11 +1276,11 @@ var TracksterView = Backbone.View.extend({
$(window).bind("resize", function() {
// Stop previous timer.
if (this.resize_timer) {
clearTimeout(this.resize_timer);
window.clearTimeout(this.resize_timer);
}
// When function activated, resize window and redraw.
this.resize_timer = setTimeout(() => {
this.resize_timer = window.setTimeout(() => {
view.resize_window();
}, 500);
});
@@ -1309,18 +1315,17 @@ extend(TracksterView.prototype, DrawableCollection.prototype, {
trigger_navigate: function(new_chrom, new_low, new_high, delay) {
// Stop previous timer.
if (this.timer) {
clearTimeout(this.timer);
window.clearTimeout(this.timer);
}
if (delay) {
// To aggregate calls, use timer and only navigate once
// location has stabilized.
var self = this;
this.timer = setTimeout(() => {
self.trigger("navigate", `${new_chrom}:${new_low}-${new_high}`);
this.timer = window.setTimeout(() => {
this.trigger("navigate", `${new_chrom}:${new_low}-${new_high}`);
}, 500);
} else {
view.trigger("navigate", `${new_chrom}:${new_low}-${new_high}`);
this.trigger("navigate", `${new_chrom}:${new_low}-${new_high}`);
}
},
@@ -1343,13 +1348,12 @@ extend(TracksterView.prototype, DrawableCollection.prototype, {
load_chroms: function(url_parms) {
url_parms.num = MAX_CHROMS_SELECTABLE;
var view = this;
var chrom_data = $.Deferred();
$.ajax({
url: `${Galaxy.root}api/genomes/${this.dbkey}`,
data: url_parms,
dataType: "json",
success: function(result) {
success: result => {
// Do nothing if could not load chroms.
if (result.chrom_info.length === 0) {
return;
@@ -1357,34 +1361,34 @@ extend(TracksterView.prototype, DrawableCollection.prototype, {
// Load chroms.
if (result.reference) {
var ref_track = new ReferenceTrack(view);
view.add_label_track(ref_track);
view.reference_track = ref_track;
var ref_track = new ReferenceTrack(this);
this.add_label_track(ref_track);
this.reference_track = ref_track;
}
view.chrom_data = result.chrom_info;
this.chrom_data = result.chrom_info;
view.chrom_select.html("");
view.chrom_select.append($('<option value="">Select Chrom/Contig</option>'));
this.chrom_select.html("");
this.chrom_select.append($('<option value="">Select Chrom/Contig</option>'));
for (var i = 0, len = view.chrom_data.length; i < len; i++) {
var chrom = view.chrom_data[i].chrom;
for (var i = 0, len = this.chrom_data.length; i < len; i++) {
var chrom = this.chrom_data[i].chrom;
var chrom_option = $("<option>");
chrom_option.text(chrom);
chrom_option.val(chrom);
view.chrom_select.append(chrom_option);
this.chrom_select.append(chrom_option);
}
if (result.prev_chroms) {
view.chrom_select.append($(`<option value="previous">Previous ${MAX_CHROMS_SELECTABLE}</option>`));
this.chrom_select.append($(`<option value="previous">Previous ${MAX_CHROMS_SELECTABLE}</option>`));
}
if (result.next_chroms) {
view.chrom_select.append($(`<option value="next">Next ${MAX_CHROMS_SELECTABLE}</option>`));
this.chrom_select.append($(`<option value="next">Next ${MAX_CHROMS_SELECTABLE}</option>`));
}
view.chrom_start_index = result.start_index;
this.chrom_start_index = result.start_index;
chrom_data.resolve(result.chrom_info);
},
error: function() {
alert(`Could not load chroms for this dbkey: ${view.dbkey}`);
alert(`Could not load chroms for this dbkey: ${this.dbkey}`);
}
});
return chrom_data;
@@ -1491,7 +1495,7 @@ extend(TracksterView.prototype, DrawableCollection.prototype, {
str = str.replace(/,/g, "");
// Replace colons and hyphens with space for easy parsing.
str = str.replace(/:|\-/g, " ");
str = str.replace(/:|-/g, " ");
// Parse new location.
var chrom_pos = str.split(/\s+/);
@@ -1537,10 +1541,10 @@ extend(TracksterView.prototype, DrawableCollection.prototype, {
// Set up timeout to redraw with more data when moving stops.
if (this.redraw_on_move_fn) {
clearTimeout(this.redraw_on_move_fn);
window.clearTimeout(this.redraw_on_move_fn);
}
this.redraw_on_move_fn = setTimeout(() => {
this.redraw_on_move_fn = window.setTimeout(() => {
view.request_redraw();
}, 200);
@@ -1617,7 +1621,7 @@ extend(TracksterView.prototype, DrawableCollection.prototype, {
// Set up redraw if it has not been requested since last redraw.
if (!this.requested_redraw) {
requestAnimationFrame(() => {
window.requestAnimationFrame(() => {
view._redraw();
});
this.requested_redraw = true;
@@ -1779,8 +1783,8 @@ extend(TracksterView.prototype, DrawableCollection.prototype, {
this.overview_box.height(this.default_overview_height);
this.overview_close.hide();
this.overview_highlight.hide();
view.resize_window();
view.overview_drawable = null;
this.resize_window();
this.overview_drawable = null;
}
});
@@ -1828,7 +1832,7 @@ var ToolParameterView = Backbone.View.extend({
var param = this.model;
// Param label.
var label_div = $("<div>")
$("<div>")
.addClass("param-label")
.text(param.get("label"))
.appendTo(param_div);
@@ -1881,7 +1885,7 @@ var TracksterToolView = Backbone.View.extend({
});
// Add name, inputs.
var name_div = $("<div class='tool-name'>")
$("<div class='tool-name'>")
.appendTo(parent_div)
.text(tool.get("name"));
tool.get("inputs").each(param => {
@@ -1989,44 +1993,42 @@ var TracksterToolView = Backbone.View.extend({
regions: [region.toJSON()]
};
var current_track = track;
var // Set name of track to include tool name, parameters, and region used.
track_name = tool.get("name") + current_track.tool_region_and_parameters_str(region);
// Set name of track to include tool name, parameters, and region used.
var track_name = tool.get("name") + track.tool_region_and_parameters_str(region);
var container;
// If track not in a group, create a group for it and add new track to group. If track
// already in group, add track to group.
if (current_track.container === view) {
if (track.container === track.view) {
// Create new group.
var group = new DrawableGroup(view, view, {
var group = new DrawableGroup(track.view, track.view, {
name: track.config.get_value("name")
});
// Replace track with group.
var index = current_track.container.replace_drawable(current_track, group, false);
var index = track.container.replace_drawable(track, group, false);
// Update HTML.
// FIXME: this is ugly way to replace a track with a group -- make this easier via
// a Drawable or DrawableCollection function.
group.container_div.insertBefore(current_track.view.content_div.children()[index]);
group.add_drawable(current_track);
current_track.container_div.appendTo(group.content_div);
group.container_div.insertBefore(track.view.content_div.children()[index]);
group.add_drawable(track);
track.container_div.appendTo(group.content_div);
container = group;
} else {
// Use current group.
container = current_track.container;
container = track.container;
}
// Create and init new track.
var new_track = new current_track.constructor(view, container, {
var new_track = new track.constructor(track.view, container, {
name: track_name,
hda_ldda: "hda"
});
new_track.init_for_tool_data();
new_track.change_mode(current_track.mode);
new_track.set_filters_manager(current_track.filters_manager.copy(new_track));
new_track.change_mode(track.mode);
new_track.set_filters_manager(track.filters_manager.copy(new_track));
new_track.update_icons();
container.add_drawable(new_track);
new_track.tiles_div.text("Starting job.");
@@ -2441,12 +2443,17 @@ extend(Track.prototype, Drawable.prototype, {
name: "param_space_viz_icon",
title: _l("Tool parameter space visualization"),
css_class: "arrow-split",
on_click_fn: function(track) {
var html = `<strong>Tool</strong>:${track.tool.get(
"name"
)}<br/><strong>Dataset</strong>:${track.config.get_value(
"name"
)}<br/><strong>Region(s)</strong>: <select name="regions"><option value="cur">current viewing area</option><option value="bookmarks">bookmarks</option><option value="both">current viewing area and bookmarks</option></select>`;
on_click_fn: track => {
var html = `
<strong>Tool</strong>:${track.tool.get("name")}<br/>
<strong>Dataset</strong>:${track.config.get_value("name")}<br/>
<strong>Region(s)</strong>:
<select name="regions">
<option value="cur">current viewing area</option>
<option value="bookmarks">bookmarks</option>
<option value="both">current viewing area and bookmarks</option>
</select>
`;
var cancel_fn = () => {
Galaxy.modal.hide();
@@ -2454,22 +2461,22 @@ extend(Track.prototype, Drawable.prototype, {
};
var ok_fn = () => {
var regions_to_use = $('select[name="regions"] option:selected').val(),
regions,
view_region = new visualization.GenomeRegion({
chrom: view.chrom,
start: view.low,
end: view.high
}),
bookmarked_regions = _.map(
$(".bookmark"),
elt =>
new visualization.GenomeRegion({
from_str: $(elt)
.children(".position")
.text()
})
);
var regions_to_use = $('select[name="regions"] option:selected').val();
var regions;
var view_region = new visualization.GenomeRegion({
chrom: this.view.chrom,
start: this.view.low,
end: this.view.high
});
var bookmarked_regions = _.map(
$(".bookmark"),
elt =>
new visualization.GenomeRegion({
from_str: $(elt)
.children(".position")
.text()
})
);
// Get regions for visualization.
if (regions_to_use === "cur") {
@@ -2493,6 +2500,8 @@ extend(Track.prototype, Drawable.prototype, {
})}`;
};
/*
* TODO: Re-enable this when functional.
var check_enter_esc = e => {
if ((e.keyCode || e.which) === 27) {
// Escape key
@@ -2502,6 +2511,7 @@ extend(Track.prototype, Drawable.prototype, {
ok_fn();
}
};
*/
// show dialog
Galaxy.modal.show({
@@ -2732,7 +2742,7 @@ extend(Track.prototype, Drawable.prototype, {
track.container_div.addClass("pending");
track.show_message(DATA_PENDING);
//$("<img/>").attr("src", image_path + "/yui/rel_interstitial_loading.gif").appendTo(track.tiles_div);
setTimeout(() => {
window.setTimeout(() => {
track.init();
}, track.data_query_wait);
} else if (result === "data" || result.status === "data") {
@@ -3040,7 +3050,7 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
var clear_after = options && options.clear_after;
var low = this.view.low;
var high = this.view.high;
var range = high - low;
//var range = high - low;
var width = this.view.container.width();
var w_scale = this.view.resolution_px_b;
var resolution = 1 / w_scale;
@@ -3049,7 +3059,7 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
if (this.is_overview) {
low = this.view.max_low;
high = this.view.max_high;
w_scale = width / (view.max_high - view.max_low);
w_scale = width / (this.view.max_high - this.view.max_low);
resolution = 1 / w_scale;
}
@@ -3105,17 +3115,16 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
}
// When all tiles are drawn, call post-draw actions.
var track = this;
$.when.apply($, tile_promises).then(() => {
// Step (c) for (re)moving tiles when clear_after is true:
track.tiles_div.children(".remove").remove();
this.tiles_div.children(".remove").remove();
// Only do postdraw actions for tiles; instances where tiles may not be drawn include:
// (a) ReferenceTrack without sufficient resolution;
// (b) data_fetch = false.
tiles = _.filter(tiles, t => t !== null);
if (tiles.length !== 0) {
track.postdraw_actions(tiles, width, w_scale, clear_after);
this.postdraw_actions(tiles, width, w_scale, clear_after);
}
});
},
@@ -3124,18 +3133,17 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
* Add a maximum/minimum label to track.
*/
_add_yaxis_label: function(type, on_change) {
var track = this;
var css_class = type === "max" ? "top" : "bottom";
var text = type === "max" ? "max" : "min";
var pref_name = type === "max" ? "max_value" : "min_value";
var label = this.container_div.find(`.yaxislabel.${css_class}`);
var value = round(track.config.get_value(pref_name), 1);
var value = round(this.config.get_value(pref_name), 1);
// Default action for on_change is to redraw track.
on_change =
on_change ||
(() => {
track.request_draw({ clear_tile_cache: true });
this.request_draw({ clear_tile_cache: true });
});
if (label.length !== 0) {
@@ -3149,7 +3157,7 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
num_cols: 12,
on_finish: function(new_val) {
$(".tooltip").remove();
track.config.set_value(pref_name, round(new_val, 1));
this.config.set_value(pref_name, round(new_val, 1));
on_change();
},
help_text: `Set ${text} value`
@@ -3176,18 +3184,17 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
// Clear because this is set when drawing.
this.max_height_px = 0;
var track = this;
_.each(tiles, tile => {
if (!(tile instanceof LineTrackTile)) {
tile.html_elt.remove();
track.draw_helper(tile.region, w_scale, {
this.draw_helper(tile.region, w_scale, {
force: true,
mode: "Coverage"
});
}
});
track._add_yaxis_label("max");
this._add_yaxis_label("max");
} else {
// -- Drawing in non-Coverage mode. --
@@ -3247,19 +3254,16 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
var mode = options.mode || this.mode;
var resolution = 1 / w_scale;
var // Useful vars.
track = this;
var drawables = this._get_drawables();
var key = this._gen_tile_cache_key(w_scale, region);
var is_tile = o => o && "track" in o;
// Check tile cache, if found show existing tile in correct position
var tile = force ? undefined : track.tile_cache.get_elt(key);
var tile = force ? undefined : this.tile_cache.get_elt(key);
if (tile) {
if (is_tile(tile)) {
track.show_tile(tile, w_scale);
this.show_tile(tile, w_scale);
}
return tile;
}
@@ -3277,16 +3281,16 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
// Map drawable object to data needed for drawing.
var tile_data = _.map(drawables, (
d // Get the track data/promise.
) => d.data_manager.get_data(region, data_mode, resolution, track.data_url_extra_params));
) => d.data_manager.get_data(region, data_mode, resolution, this.data_url_extra_params));
// Get reference data/promise.
if (view.reference_track) {
if (this.view.reference_track) {
tile_data.push(
view.reference_track.data_manager.get_data(
this.view.reference_track.data_manager.get_data(
region,
mode,
resolution,
view.reference_track.data_url_extra_params
this.view.reference_track.data_url_extra_params
)
);
}
@@ -3298,7 +3302,7 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
// When data is available, draw tile.
//
var tile_drawn = $.Deferred();
track.tile_cache.set_elt(key, tile_drawn);
this.tile_cache.set_elt(key, tile_drawn);
$.when.apply($, get_tile_data()).then(() => {
var tile_data = get_tile_data();
var tracks_data = tile_data;
@@ -3309,41 +3313,36 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
// Deferred, try again from the top. NOTE: this condition could (should?) be handled by the
// GenomeDataManager in visualization module.
if (_.find(tile_data, d => util.is_deferred(d))) {
track.tile_cache.set_elt(key, undefined);
$.when(track.draw_helper(region, w_scale, options)).then(tile => {
this.tile_cache.set_elt(key, undefined);
$.when(this.draw_helper(region, w_scale, options)).then(tile => {
tile_drawn.resolve(tile);
});
return;
}
// If sequence data is available, subset to get only data in region.
if (view.reference_track) {
seq_data = view.reference_track.data_manager.subset_entry(tile_data.pop(), region);
if (this.view.reference_track) {
seq_data = this.view.reference_track.data_manager.subset_entry(tile_data.pop(), region);
}
// Get drawing modes, heights for all tracks.
var drawing_modes = [];
var drawing_heights = [];
_.each(drawables, (d, i) => {
var mode = d.mode;
var data = tracks_data[i];
if (mode === "Auto") {
mode = d.get_mode(data);
d.update_auto_mode(mode);
if (d.mode === "Auto") {
d.mode = d.get_mode(data);
d.update_auto_mode(d.mode);
}
drawing_modes.push(mode);
drawing_heights.push(d.get_canvas_height(data, mode, w_scale, width));
drawing_modes.push(d.mode);
drawing_heights.push(d.get_canvas_height(data, d.mode, w_scale, width));
});
var canvas = track.view.canvas_manager.new_canvas();
var canvas = this.view.canvas_manager.new_canvas();
var tile_low = region.get("start");
var tile_high = region.get("end");
var all_data_index = 0;
var width = Math.ceil((tile_high - tile_low) * w_scale) + track.left_offset;
var width = Math.ceil((tile_high - tile_low) * w_scale) + this.left_offset;
var height = _.max(drawing_heights);
var tile;
@@ -3354,7 +3353,7 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
// Height is specified in options or is the height found above.
canvas.height = options.height || height;
var ctx = canvas.getContext("2d");
ctx.translate(track.left_offset, 0);
ctx.translate(this.left_offset, 0);
if (drawables.length > 1) {
ctx.globalAlpha = 0.5;
ctx.globalCompositeOperation = "source-over";
@@ -3365,8 +3364,8 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
// Don't cache, show if no tile.
if (tile !== undefined) {
track.tile_cache.set_elt(key, tile);
track.show_tile(tile, w_scale);
this.tile_cache.set_elt(key, tile);
this.show_tile(tile, w_scale);
}
tile_drawn.resolve(tile);
@@ -3428,7 +3427,6 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
* an existing tile rather than reshowing it.
*/
show_tile: function(tile, w_scale) {
var track = this;
var tile_element = tile.html_elt;
// -- Show/move tile element. --
@@ -3785,7 +3783,7 @@ extend(CompositeTrack.prototype, TiledTrack.prototype, {
}
// Replace track with group.
var index = this.container.replace_drawable(this, group, true);
this.container.replace_drawable(this, group, true);
group.request_draw({ clear_tile_cache: true });
},
@@ -3819,7 +3817,6 @@ extend(CompositeTrack.prototype, TiledTrack.prototype, {
* Update minimum, maximum for component tracks.
*/
update_all_min_max: function() {
var track = this;
var min_value = this.config.get_value("min_value");
var max_value = this.config.get_value("max_value");
_.each(this.drawables, d => {
@@ -4127,7 +4124,8 @@ extend(DiagonalHeatmapTrack.prototype, Drawable.prototype, TiledTrack.prototype,
hda_ldda: track.dataset.get("hda_ldda")
},
result => {
var data = result.data;
// What does this do? Is it meant to be attached to some higher scope state object?
// var data = result.data;
}
);
},
@@ -4163,10 +4161,9 @@ var FeatureTrack = function(view, container, obj_dict) {
this.slotters = {};
this.start_end_dct = {};
this.left_offset = 200;
// this.painter = painters.LinkedFeaturePainter;
this.set_painter_from_config();
};
extend(FeatureTrack.prototype, Drawable.prototype, TiledTrack.prototype, {
display_modes: ["Auto", "Coverage", "Dense", "Squish", "Pack"],
@@ -4258,9 +4255,6 @@ extend(FeatureTrack.prototype, Drawable.prototype, TiledTrack.prototype, {
postdraw_actions: function(tiles, width, w_scale, clear_after) {
TiledTrack.prototype.postdraw_actions.call(this, tiles, width, w_scale, clear_after);
var track = this;
var i;
var line_track_tiles = _.filter(tiles, t => t instanceof LineTrackTile);
//
@@ -4277,7 +4271,6 @@ extend(FeatureTrack.prototype, Drawable.prototype, TiledTrack.prototype, {
});
// Draw incomplete features on each tile.
var self = this;
_.each(tiles, tile => {
// Remove features already drawn on tile originally.
var tile_incomplete_features = _.omit(
@@ -4296,16 +4289,16 @@ extend(FeatureTrack.prototype, Drawable.prototype, TiledTrack.prototype, {
data: _.values(tile_incomplete_features)
};
var new_canvas = self.view.canvas_manager.new_canvas();
var new_canvas = this.view.canvas_manager.new_canvas();
var new_canvas_ctx = new_canvas.getContext("2d");
new_canvas.height = Math.max(
tile.canvas.height,
self.get_canvas_height(features, tile.mode, tile.w_scale, 100)
this.get_canvas_height(features, tile.mode, tile.w_scale, 100)
);
new_canvas.width = tile.canvas.width;
new_canvas_ctx.drawImage(tile.canvas, 0, 0);
new_canvas_ctx.translate(track.left_offset, 0);
var new_tile = self.draw_tile(
new_canvas_ctx.translate(this.left_offset, 0);
var new_tile = this.draw_tile(
features,
new_canvas_ctx,
tile.mode,
@@ -4348,8 +4341,8 @@ extend(FeatureTrack.prototype, Drawable.prototype, TiledTrack.prototype, {
//
// Update filtering UI.
if (track.filters_manager) {
var filters = track.filters_manager.filters;
if (this.filters_manager) {
var filters = this.filters_manager.filters;
var f;
for (f = 0; f < filters.length; f++) {
filters[f].update_ui_elt();
@@ -4361,7 +4354,7 @@ extend(FeatureTrack.prototype, Drawable.prototype, TiledTrack.prototype, {
var example_feature;
var filter;
for (i = 0; i < tiles.length; i++) {
for (let i = 0; i < tiles.length; i++) {
if (tiles[i].data.length) {
example_feature = tiles[i].data[0];
for (f = 0; f < filters.length; f++) {
@@ -4375,12 +4368,12 @@ extend(FeatureTrack.prototype, Drawable.prototype, TiledTrack.prototype, {
}
// If filter availability changed, hide filter div if necessary and update menu.
if (track.filters_available !== filters_available) {
track.filters_available = filters_available;
if (!track.filters_available) {
track.filters_manager.hide();
if (this.filters_available !== filters_available) {
this.filters_available = filters_available;
if (!this.filters_available) {
this.filters_manager.hide();
}
track.update_icons();
this.update_icons();
}
}
@@ -4389,7 +4382,7 @@ extend(FeatureTrack.prototype, Drawable.prototype, TiledTrack.prototype, {
//
if (tiles[0] instanceof FeatureTrackTile) {
var all_slotted = true;
for (i = 0; i < tiles.length; i++) {
for (let i = 0; i < tiles.length; i++) {
if (!tiles[i].all_slotted) {
all_slotted = false;
break;
@@ -4557,6 +4550,7 @@ extend(FeatureTrack.prototype, Drawable.prototype, TiledTrack.prototype, {
);
var feature_mapper = null;
var incomplete_features = null;
ctx.fillStyle = this.config.get_value("block_color");
ctx.font = ctx.canvas.manager.default_font;
client/galaxy/scripts/viz/visualization.js
+3 -5
View File
@@ -5,7 +5,6 @@ import util_mod from "viz/trackster/util";
import config_mod from "utils/config";
import GridView from "mvc/grid/grid-view";
import Tabs from "mvc/ui/ui-tabs";
import Ui from "mvc/ui/ui-misc";
/**
* Mixin for returning custom JSON representation from toJSON. Class attribute to_json_keys defines a set of attributes
* to include in the representation; to_json_mappers defines mappers for returned objects.
@@ -1109,12 +1108,11 @@ var TrackBrowserRouter = Backbone.Router.extend({
// Can't put regular expression in routes dictionary.
// NOTE: parentheses are used to denote parameters returned to callback.
this.route(/([\w]+)$/, "change_location");
this.route(/([\w\+]+\:[\d,]+-[\d,]+)$/, "change_location");
this.route(/([\w+]+:[\d,]+-[\d,]+)$/, "change_location");
// Handle navigate events from view.
var self = this;
self.view.on("navigate", new_loc => {
self.navigate(new_loc);
this.view.on("navigate", new_loc => {
this.navigate(new_loc);
});
},
client/galaxy/style/less/ui.less
+6 -13
View File
@@ -486,6 +486,9 @@
.ui-form-composite {
max-width: 900px;
height: 100%;
flex-direction: column;
display: flex;
.ui-form-header {
&:extend(h3);
margin-top: 0px;
@@ -906,10 +909,6 @@
float: right;
width: ~'calc(100% - 76px)';
}
.ui-gs-token-textbox {
float: right;
width: ~'calc(100% - 76px)';
}
.ui-gs-browse-button {
float: left;
.ui-button-icon {
@@ -917,15 +916,9 @@
margin-right: 5px;
}
}
.ui-gs-token-label {
margin-top: 5px;
margin-left: 32px;
margin-right: 5px;
float: left;
}
.ui-gs-token-field {
clear: both;
padding-top: 5px;
.ui-gs-browse-field {
height: @input-height-base;
line-height: @line-height-base;
}
}
config/datatypes_conf.xml.sample
+2 -2
View File
@@ -21,7 +21,7 @@
</datatype>
<datatype extension="qname_sorted.bam" type="galaxy.datatypes.binary:BamQuerynameSorted" mimetype="application/octet-stream" display_in_upload="true" description="A binary file compressed in the BGZF format with a '.bam' file extension and sorted by queryname." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#BAM">
</datatype>
<datatype extension="bam_native" type="galaxy.datatypes.binary:BamNative" mimetype="application/octet-stream" display_in_upload="true" description="A binary file compressed in the BGZF format with a '.bam' file extension." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#BAM">
<datatype extension="unsorted.bam" type="galaxy.datatypes.binary:BamNative" mimetype="application/octet-stream" display_in_upload="true" description="A binary file compressed in the BGZF format with a '.bam' file extension." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#BAM">
<converter file="bam_to_bigwig_converter.xml" target_datatype="bigwig"/>
<converter file="to_coordinate_sorted_bam.xml" target_datatype="bam"/>
<converter file="to_qname_sorted_bam.xml" target_datatype="qname_sorted.bam"/>
@@ -299,7 +299,7 @@
<datatype extension="qual454" type="galaxy.datatypes.qualityscore:QualityScore454" display_in_upload="true"/>
<datatype extension="roadmaps" type="galaxy.datatypes.assembly:Roadmaps" display_in_upload="false"/>
<datatype extension="sam" type="galaxy.datatypes.tabular:Sam" display_in_upload="true">
<converter file="sam_to_bam_native.xml" target_datatype="bam_native"/>
<converter file="sam_to_unsorted_bam.xml" target_datatype="unsorted.bam"/>
<converter file="to_coordinate_sorted_bam.xml" target_datatype="bam"/>
<converter file="to_qname_sorted_bam.xml" target_datatype="qname_sorted.bam"/>
<converter file="sam_to_bigwig_converter.xml" target_datatype="bigwig"/>
config/galaxy.yml.sample
+14 -17
View File
@@ -812,6 +812,11 @@ galaxy:
# documentation for the corresponding nginx configuration.
#nginx_upload_job_files_path: false
# Galaxy can upload user files in chunks without using nginx. Enable
# the chunk uploader by specifying a chunk size larger than 0. The
# chunk size is specified in bytes (default: 100MB).
#chunk_upload_size: 104857600
# Have Galaxy manage dynamic proxy component for routing requests to
# other services based on Galaxy's session cookie. It will attempt to
# do this by default though you do need to install node+npm and do an
@@ -1167,6 +1172,15 @@ galaxy:
# Set maximum size of ngrams
#tool_ngram_maxsize: 4
# Set tool test data directory. The test framework sets this value to
# 'test-data,https://github.com/galaxyproject/galaxy-test-data.git'
# which will cause Galaxy to clone down extra test data on the fly for
# certain tools distributed with Galaxy but this is likely not
# appropriate for production systems. Instead one can simply clone
# that repository directly and specify a path here instead of a Git
# HTTP repository.
#tool_test_data_directories: test-data
# Galaxy encodes various internal values when these values will be
# output in some format (for example, in a URL or cookie). You should
# set a key to be used by the algorithm that encodes and decodes these
@@ -1621,23 +1635,6 @@ galaxy:
# processors, memory and walltime.
#job_resource_params_file: config/job_resource_params_conf.xml
# Similar to the above parameter, workflows can describe parameters
# used to influence scheduling of jobs within the workflow. This
# requires both a description of the fields available (which defaults
# to the definitions in job_resource_params_file if not set).
#workflow_resource_params_file: config/workflow_resource_params_conf.xml
# This parameter describes how to map users and workflows to a set of
# workflow resource parameter to present (typically input IDs from
# workflow_resource_params_file). If this this is a function reference
# it will be passed various inputs (workflow model object and user)
# and it should produce a list of input IDs. If it is a path it is
# expected to an XML or YAML file describing how to map group names to
# parameter descriptions (additional types of mappings via these files
# could be implemented but haven't yet - for instance using workflow
# tags to do the mapping).
#workflow_resource_params_mapper: config/workflow_resource_mapper_conf.yml
# If using job concurrency limits (configured in job_config_file),
# several extra database queries must be performed to determine the
# number of jobs a user has dispatched to a given destination. By
config/job_metrics_conf.xml.sample
+17 -12
View File
@@ -15,7 +15,7 @@
<!-- Core plugin captures Galaxy slots, start and end of job (in seconds
since epoch) and computes runtime in seconds. -->
<core />
<!-- Uncomment to dump processor count for each job - linux only. -->
<!-- <cpuinfo /> -->
<!-- Uncomment to dump information about all processors for for each
@@ -30,13 +30,18 @@
only. -->
<!-- <uname /> -->
<!-- Uncomment following to enable plugin dumping complete environment
<!-- Uncomment following to enable plugin dumping complete environment
for each job, potentially useful for debuging -->
<!-- <env /> -->
<!-- env plugin can also record more targetted, obviously useful variables
as well. -->
<!-- <env variables="HOSTNAME,SLURM_CPUS_ON_NODE,SLURM_JOBID" /> -->
<!-- If galaxy jobs are run in cgroups, like slurm does if memory limits
are enforced, we can try to grep some information from this. -->
<!-- <cgroup /> -->
<!-- <cgroup verbose="true" /> -->
<!-- <collectl /> -->
<!-- Collectl (http://collectl.sourceforge.net/) is a powerful monitoring
utility capable of gathering numerous system and process level
@@ -46,7 +51,7 @@
is highly customiziable - both using the attributes documented below
or simply hacking up the code in lib/galaxy/jobs/metrics.
Warning: In order to use this plugin collectl must be available on the
Warning: In order to use this plugin collectl must be available on the
compute server the job runs on and on the local Galaxy server as well
(unless in this latter case summarize_process_data is set to False).
@@ -64,12 +69,12 @@
full time-series data corresponding to a job run.
'subsystems': Comma separated list of collectl subystems to collect
data for. Plugin doesn't currently expose all of them or offer
summary data for any of them except 'process' but extensions
would be welcome. May seem pointless to include subsystems
data for. Plugin doesn't currently expose all of them or offer
summary data for any of them except 'process' but extensions
would be welcome. May seem pointless to include subsystems
beside process since they won't be processed online by Galaxy -
but if 'saved_logs_path' these files can be played back at anytime.
Available subsystems - 'process', 'cpu', 'memory', 'network',
'disk', 'network'. (Default 'process').
@@ -108,13 +113,13 @@
'flush': Interval (in seconds I think) between when collectl will
flush its buffer to disk. Galaxy overrides this to disable
flushing by default if not set.
flushing by default if not set.
'local_collectl_path', 'remote_collectl_path', 'collectl_path':
By default, jobs will just assume collectl is on the PATH, but
it can be overridden with 'local_collectl_path' and
'remote_collectl_path' (or simply 'collectl_path' if it is not
on the path but installed in the same location both locally and
By default, jobs will just assume collectl is on the PATH, but
it can be overridden with 'local_collectl_path' and
'remote_collectl_path' (or simply 'collectl_path' if it is not
on the path but installed in the same location both locally and
remotely).
There are more and more increasingly obsecure options including -
config/plugins/webhooks/demo/tour_generator/__init__.py
+4 -17
View File
@@ -64,26 +64,13 @@ class TourGenerator(object):
self._use_datasets = False
return
test_data_paths = [os.path.abspath('test-data')]
test_data_cache_dir = os.path.abspath(
os.environ.get('GALAXY_TEST_DATA_REPO_CACHE', 'test-data-cache'))
test_data_paths.extend([
x[0] for x in os.walk(test_data_cache_dir) if '.git' not in x[0]])
if self._tool.tool_shed:
test_data_paths.append(os.path.abspath(os.path.join(
self._tool.tool_dir, 'test-data')))
# Upload all test datasets
for input_name, input in self._data_inputs.items():
if input_name in test_datasets.keys():
for i, data_path in enumerate(test_data_paths):
input_path = os.path.join(data_path,
test_datasets[input_name])
if os.path.exists(input_path):
break
elif i + 1 == len(test_data_paths): # the last path
raise ValueError('Test dataset "%s" doesn\'t exist.' %
input_name)
filename = test_datasets[input_name]
input_path = self._tool.test_data_path(filename)
if not input_path:
raise ValueError('Test dataset "%s" doesn\'t exist.' % input_name)
upload_tool = self._trans.app.toolbox.get_tool('upload1')
filename = os.path.basename(input_path)
doc/source/admin/galaxy_options.rst
+28
View File
@@ -1643,6 +1643,18 @@
:Type: bool
~~~~~~~~~~~~~~~~~~~~~
``chunk_upload_size``
~~~~~~~~~~~~~~~~~~~~~
:Description:
Galaxy can upload user files in chunks without using nginx. Enable
the chunk uploader by specifying a chunk size larger than 0. The
chunk size is specified in bytes (default: 100MB).
:Default: ``104857600``
:Type: int
~~~~~~~~~~~~~~~~~~~~~~~~
``dynamic_proxy_manage``
~~~~~~~~~~~~~~~~~~~~~~~~
@@ -2439,6 +2451,22 @@
:Type: int
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
``tool_test_data_directories``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:Description:
Set tool test data directory. The test framework sets this value
to 'test-data,https://github.com/galaxyproject/galaxy-test-
data.git' which will cause Galaxy to clone down extra test data on
the fly for certain tools distributed with Galaxy but this is
likely not appropriate for production systems. Instead one can
simply clone that repository directly and specify a path here
instead of a Git HTTP repository.
:Default: ``test-data``
:Type: str
~~~~~~~~~~~~~
``id_secret``
~~~~~~~~~~~~~
doc/source/releases/18.01.rst
+1184
View File
File diff suppressed because it is too large Load Diff
doc/source/releases/18.01_announce.rst
+221 -3
View File
@@ -3,8 +3,226 @@
January 2018 Galaxy Release (v 18.01)
===========================================================
.. include:: _header.rst
Schedule
Highlights
===========================================================
* Planned Freeze Date: 2018-01-08
* Planned Release Date: 2018-01-29
**Performance and User Experience Improvements**
We made Galaxy more lively and responsive. Homepage, published workflows, published/saved histories, and data libraries should all load much faster now. Importing data from FTP will also take less of your time.
We achieved this by optimizing database queries, implementing cache, rethinking presentation, and adding progress bars and loading indicators where needed. See the `list of related performance pull requests`_.
**Web Server and Configuration**
The default web server used by Galaxy has changed from Paste to `uWSGI <https://uwsgi-docs.readthedocs.io/>`__ and the default configuration file for Galaxy is now ``config/galaxy.yml`` instead of ``config/galaxy.ini``.
uWSGI is more production ready and allows Galaxy to scale better in its default
configuration. In the future uWSGI will allow Galaxy to setup GIE proxies without additional
configuration and use modern web technologies such as web sockets.
Read more about the server, configuration, and documentation changes in the `uWSGI details`_ section of this document.
**Dataset Collection Usability**
This release has significantly improved the usability of Galaxy dataset collections. Dozens of improvements
to collections have been made, some of the key highlights include:
- Data library folders can now be sent to histories as a dataset collection. `Pull Request 4998`_, `Pull Request 5080`_
- Failed dataset collection elements can now be fixed using job re-running (thanks to `@mvdbeek <https://github.com/mvdbeek>`__). `Pull Request 5321`_, `Pull Request 5247`_, `Pull Request 5270`_
- Collections now appear with state and progress bars in the history panel and contained datasets
are hidden by default (with help from `@mvdbeek <https://github.com/mvdbeek>`__).
`Pull Request 5013`_, `Pull Request 5078`_
- We added intuitive workflow post job actions for dataset collections. `Pull Request 5416`_, `Pull Request 5418`_, `Pull Request 5414`_
- The web interface now supports collections with arbitrary nesting and size (with help
from `@mvdbeek <https://github.com/mvdbeek>`__). `Pull Request 4942`_, `Pull Request 4934`_, `Pull Request 5091`_
- More robust nametag discovery and propagation when using collections. `Pull Request 5403`_
**Client Architecture**
The architecture for the client code that powers the Galaxy user interface has been significantly
overhauled. The code base has been converted to `ES6 <http://es6-features.org/>`__,
`Yarn <https://github.com/yarnpkg/yarn>`__ now powers the build and dependency management of the code,
`Prettier <https://prettier.io/>`__ is now used to ensure consistent code formatting,
and the `VueJS <https://vuejs.org/>`__ framework has been integrated. Taken together, these changes are
enabling Galaxy developers to write usable, responsive client code more quickly and concisely than
previously possible. A big thanks goes out to community contributions from
`@anuprulez <https://github.com/anuprulez>`__ and `@anatskiy <https://github.com/anatskiy>`__ that are
already converting existing Galaxy components to reactive VueJS ones. See the `list of related client pull requests`_.
**New BAM datatypes**
Previously Galaxy only supported coordinate sorted BAM files by default (the ``bam`` datatype). In addition, this
release of Galaxy now supports three new types of BAM:
- ``qname_sorted.bam``, that ensures that the file is queryname sorted (e.g. ``SO:queryname``);
- ``qname_input_sorted.bam``, that can be used to describe the output of aligners which generally keep mate pairs adjacent
- ``unsorted.bam``, that makes no assumptions about the sort order of the file.
A huge thanks goes out to `@bgruening <https://github.com/bgruening>`__ and `@mvdbeek <https://github.com/mvdbeek>`__ who
implemented these datatypes.
`Pull Request 5180`_, `Pull Request 5589`_, `Pull Request 5532`_, `Pull Request 5644`_, `Pull Request 5674`_
**Experimental Job Caching**
Galaxy can now be configured to allow users the option of skipping duplicated jobs if one with
identical parameters has been previously executed and simply reuse the previously generated outputs.
This contribution is thanks to `@mvdbeek <https://github.com/mvdbeek>`__.
`Pull Request 4690`_
Get Galaxy
==========
The code lives at `Github <https://github.com/galaxyproject/galaxy>`__ and you should have `Git <https://git-scm.com/>`__ to obtain it.
To get a new Galaxy repository run:
.. code-block:: shell
$ git clone -b release_18.01 https://github.com/galaxyproject/galaxy.git
To update an existing Galaxy repository run:
.. code-block:: shell
$ git checkout release_18.01 && git pull --ff-only origin release_18.01
See the `community hub <https://galaxyproject.org/develop/source-code/>`__ for additional details regarding the source code locations.
Security
========
Unsecure GenomeSpace token exposure
-----------------------------------
Tracked as ``GX-2018-0002``.
We have found and fixed a medium-level security issue concering the GenomeSpace importer/exporter tools that were updated in the Galaxy release 17.09. These tools did not handle the GenomeSpace access token securely and stored it as a job parameter which made it accessible to anybody with access to the datasets created by these tools.
This means that any user that used a GenomeSpace token to access these tools and subsequently shared the output dataset (or history that contains it) with another user shared their GenomeSpace token also.
These tools are both included in the ``tool_conf.xml.sample`` and are therefore *enabled on every new Galaxy by default*.
Administrators please see the `GenomeSpace security sanitization`_ section of this document for the details on how to sanitize the tokens stored in the Galaxy database created prior to this fix.
The vulnerability has been resolved by removing the token functionality until a proper implementation is in place. The GenomeSpace tools continue to work using the OpenID authentication as before.
The fix for this issue has been applied back to Galaxy release 17.09 and can be found in this `pull request <https://github.com/galaxyproject/galaxy/pull/5631>`__.
Breaking Changes
================
We have reworked the **Galaxy Webhook** interface so if you have custom webhooks at your instance
you need to take the following steps in order to make them work:
1. Rename the main config file: <name>.yml -> config.yml
2. In the config file, rename the name attribute: name -> id
3. Put all files into the plugin's root folder, which should contain only four files: ``config.yml`` (mandatory), ``__init__.py`` (optional), ``script.js`` (optional), ``styles.css`` (optional)
Deprecation Notices
===================
* This is the last release we are shipping JavaScript source maps for Galaxy client.
Removal Notices
===============
The following features that have been deprecated in the past releases are being removed in 18.01:
* Sample tracking
* Sample request and external services management
* Legacy library interface for Administrators
In addition to that, the PlantTribes datatypes have been commented out in the source code. Uncomment them if you want to re-enable it.
uWSGI details
=============
To minimize the impact of this change on existing Galaxy instances, if a Galaxy has a ``galaxy.ini``
file configured, it will continue to use Paste by default unless additional steps are
taken by the administrator (Galaxy can be forced to start under uWSGI even with an older configuration file
by setting ``APP_WEBSERVER=uwsgi`` in the environment). As part of the transition to YAML-based
configuration files, we have implemented a schema to validate Galaxy configuration files. Run
``make config-validate`` from Galaxy's root directory to validate a schema and ``make config-lint`` to
check for best practices. While there is no need to convert your configuration file (``galaxy.ini`` hasn't
been deprecated), you can run ``make config-convert-dry-run`` and ``make config-convert``
to respectively test and perform the conversion of an ``ini`` configuration file to a YAML one.
These are big changes that affect many parts of Galaxy's administration documentation and makes
this documentation very dependent on which Galaxy version they are targeting. To address this, we have
moved a significant amount of administration documentation into Galaxy's code
base and made it available on a per-release basis. The latest administration documentation for
the previous release of Galaxy (17.09) which reference ``galaxy.ini`` files and Paste servers can
be found `here <https://docs.galaxyproject.org/en/release_17.09/admin/index.html>`__, while documentation for this release can be found `here <https://docs.galaxyproject.org/en/release_18.01/admin/index.html>`__.
`Pull Request 4475`_, `Pull Request 5135`_, `Pull Request 5390`_, `Pull Request 5373`_,
`Pull Request 5105`_, `Pull Request 5441`_
Release Notes
=============
.. include:: 18.01.rst
:start-after: announce_start
GenomeSpace security sanitization
=================================
Outputs of these tools may require sanitization: ``genomespace_importer`` and ``genomespace_exporter``
The following SQL commands will help you **identify** the datasets in your Galaxy's database.
Finding bad GenomeSpace importer params:
.. code-block:: sql
SELECT j.id,
j.create_time,
j.user_id,
jp.value
FROM job_parameter jp
JOIN job j ON j.id = jp.job_id
WHERE jp.job_id IN
(SELECT id
FROM job
WHERE tool_id = 'genomespace_importer')
AND jp.name = 'URL'
AND jp.value LIKE '%^%'
ORDER BY j.id DESC;
Finding bad GenomeSpace exporter params:
.. code-block:: sql
SELECT j.id,
j.create_time,
j.user_id,
jp.value
FROM job_parameter jp
JOIN job j ON j.id = jp.job_id
WHERE jp.job_id IN
(SELECT id
FROM job
WHERE tool_id = 'genomespace_exporter')
AND jp.name = 'genomespace_browser'
AND jp.value LIKE '%^%'
ORDER BY j.id DESC;
The following SQL commands will help you **sanitize** the datasets in your Galaxy's database.
Sanitizing GenomeSpace importer params:
.. code-block:: sql
UPDATE job_parameter jp
SET value = split_part(jp.value, '^', 1) || '"'
FROM job j
WHERE jp.job_id = j.id
AND j.tool_id = 'genomespace_importer'
AND jp.name = 'URL'
AND jp.value LIKE '%^%';
Sanitizing GenomeSpace exporter params:
.. code-block:: sql
UPDATE job_parameter jp
SET value = split_part(jp.value, '^', 1) || '"'
FROM job j
WHERE jp.job_id = j.id
AND j.tool_id = 'genomespace_exporter'
AND jp.name = 'genomespace_browser'
AND jp.value LIKE '%^%';
.. include:: _thanks.rst
doc/source/releases/18.05_announce.rst
+10
View File
@@ -0,0 +1,10 @@
===========================================================
May 2018 Galaxy Release (v 18.05)
===========================================================
Schedule
===========================================================
* Planned Freeze Date: 2018-05-07
* Planned Release Date: 2018-05-28
doc/source/releases/index.rst
+1
View File
@@ -4,6 +4,7 @@ Releases
.. toctree::
:maxdepth: 1
18.01_announce
17.09_announce
17.05_announce
17.01_announce
lib/galaxy/actions/library.py
+2 -4
View File
@@ -249,16 +249,14 @@ class LibraryActions(object):
uploaded_dataset.to_posix_lines = params.get('to_posix_lines', None)
uploaded_dataset.space_to_tab = params.get('space_to_tab', None)
uploaded_dataset.tag_using_filenames = params.get('tag_using_filenames', True)
uploaded_dataset.purge_source = getattr(trans.app.config, 'ftp_upload_purge', True)
if in_folder:
uploaded_dataset.in_folder = in_folder
uploaded_dataset.data = upload_common.new_upload(trans, 'api', uploaded_dataset, library_bunch)
uploaded_dataset.link_data_only = link_data_only
uploaded_dataset.uuid = uuid_str
if link_data_only == 'link_to_files':
uploaded_dataset.data.file_name = os.path.abspath(path)
# Since we are not copying the file into Galaxy's managed
# default file location, the dataset should never be purgable.
uploaded_dataset.data.dataset.purgable = False
uploaded_dataset.data.link_to(path)
trans.sa_session.add_all((uploaded_dataset.data, uploaded_dataset.data.dataset))
trans.sa_session.flush()
return uploaded_dataset
lib/galaxy/app.py
+10
View File
@@ -12,6 +12,9 @@ import galaxy.security
from galaxy import config, jobs
from galaxy.jobs import metrics as job_metrics
from galaxy.managers.collections import DatasetCollectionManager
from galaxy.managers.folders import FolderManager
from galaxy.managers.histories import HistoryManager
from galaxy.managers.libraries import LibraryManager
from galaxy.managers.tags import GalaxyTagManager
from galaxy.openid.providers import OpenIDProviders
from galaxy.queue_worker import GalaxyQueueWorker
@@ -20,8 +23,10 @@ from galaxy.tools.cache import (
ToolShedRepositoryCache
)
from galaxy.tools.data_manager.manager import DataManagers
from galaxy.tools.deps.views import DependencyResolversView
from galaxy.tools.error_reports import ErrorReports
from galaxy.tools.special_tools import load_lib_tools
from galaxy.tools.verify import test_data
from galaxy.tours import ToursRegistry
from galaxy.util import (
ExecutionTimer,
@@ -90,6 +95,11 @@ class UniverseApplication(config.ConfiguresGalaxyMixin):
self.tag_handler = GalaxyTagManager(self.model.context)
# Dataset Collection Plugins
self.dataset_collections_service = DatasetCollectionManager(self)
self.history_manager = HistoryManager(self)
self.dependency_resolvers_view = DependencyResolversView(self)
self.test_data_resolver = test_data.TestDataResolver(file_dirs=self.config.tool_test_data_directories)
self.library_folder_manager = FolderManager()
self.library_manager = LibraryManager()
# Tool Data Tables
self._configure_tool_data_tables(from_shed_config=False)
lib/galaxy/config.py
+3 -10
View File
@@ -47,7 +47,6 @@ PATH_DEFAULTS = dict(
error_report_file=['config/error_report.yml', 'config/error_report.yml.sample'],
dependency_resolvers_config_file=['config/dependency_resolvers_conf.xml', 'dependency_resolvers_conf.xml'],
job_resource_params_file=['config/job_resource_params_conf.xml', 'job_resource_params_conf.xml'],
workflow_resource_params_file=['config/workflow_resource_params_conf.xml', 'workflow_resource_params_conf.xml'],
migrated_tools_config=['migrated_tools_conf.xml', 'config/migrated_tools_conf.xml'],
object_store_config_file=['config/object_store_conf.xml', 'object_store_conf.xml'],
openid_config_file=['config/openid_conf.xml', 'openid_conf.xml', 'config/openid_conf.xml.sample'],
@@ -392,15 +391,6 @@ class Configuration(object):
self.maximum_workflow_invocation_duration = int(kwargs.get("maximum_workflow_invocation_duration", 2678400))
self.maximum_workflow_jobs_per_scheduling_iteration = int(kwargs.get("maximum_workflow_jobs_per_scheduling_iteration", -1))
workflow_resource_params_mapper = kwargs.get("workflow_resource_params_mapper", None)
if not workflow_resource_params_mapper:
workflow_resource_params_mapper = None
elif ":" not in workflow_resource_params_mapper:
# Assume it is not a Python function, so a file
workflow_resource_params_mapper = self.resolve_path(workflow_resource_params_mapper)
# else: a Python a function!
self.workflow_resource_params_mapper = workflow_resource_params_mapper
self.cache_user_job_count = string_as_bool(kwargs.get('cache_user_job_count', False))
self.pbs_application_server = kwargs.get('pbs_application_server', "")
self.pbs_dataset_server = kwargs.get('pbs_dataset_server', "")
@@ -446,6 +436,7 @@ class Configuration(object):
self.user_library_import_check_permissions = string_as_bool(kwargs.get('user_library_import_check_permissions', False))
self.user_library_import_dir_auto_creation = string_as_bool(kwargs.get('user_library_import_dir_auto_creation', False)) if self.user_library_import_dir else False
# Searching data libraries
self.chunk_upload_size = int(kwargs.get('chunk_upload_size', 104857600))
self.ftp_upload_dir = kwargs.get('ftp_upload_dir', None)
self.ftp_upload_dir_identifier = kwargs.get('ftp_upload_dir_identifier', 'email') # attribute on user - email, username, id, etc...
self.ftp_upload_dir_template = kwargs.get('ftp_upload_dir_template', '${ftp_upload_dir}%s${ftp_upload_dir_identifier}' % os.path.sep)
@@ -472,6 +463,8 @@ class Configuration(object):
self.tool_enable_ngram_search = kwargs.get("tool_enable_ngram_search", False)
self.tool_ngram_minsize = kwargs.get("tool_ngram_minsize", 3)
self.tool_ngram_maxsize = kwargs.get("tool_ngram_maxsize", 4)
default_tool_test_data_directories = os.environ.get("GALAXY_TEST_FILE_DIR", resolve_path("test-data", self.root))
self.tool_test_data_directories = kwargs.get("tool_test_data_directories", default_tool_test_data_directories)
# Location for tool dependencies.
use_tool_dependencies, tool_dependency_dir, use_cached_dependency_manager, tool_dependency_cache_dir, precache_dependencies = \
parse_dependency_options(kwargs, self.root, self.dependency_resolvers_config_file)
lib/galaxy/containers/docker_model.py
+12 -2
View File
@@ -85,7 +85,12 @@ class DockerContainer(Container):
# }
# ]
rval = []
port_mappings = self.inspect[0]['NetworkSettings']['Ports']
try:
port_mappings = self.inspect[0]['NetworkSettings']['Ports']
except (IndexError, KeyError) as exc:
log.warning("Failed to get ports for container %s from `docker inspect` output at "
"[0]['NetworkSettings']['Ports']: %s: %s", self.id, exc.__class__.__name__, str(exc))
return None
for port_name in port_mappings:
for binding in port_mappings[port_name]:
rval.append(ContainerPort(
@@ -165,7 +170,12 @@ class DockerService(Container):
# }
# ]
rval = []
port_mappings = self.inspect[0]['Endpoint']['Ports']
try:
port_mappings = self.inspect[0]['Endpoint']['Ports']
except (IndexError, KeyError) as exc:
log.warning("Failed to get ports for container %s from `docker service inspect` output at "
"[0]['Endpoint']['Ports']: %s: %s", self.id, exc.__class__.__name__, str(exc))
return None
for binding in port_mappings:
rval.append(ContainerPort(
binding['TargetPort'],
lib/galaxy/datatypes/binary.py
+1 -1
View File
@@ -192,7 +192,7 @@ class BamNative(Binary):
"""Class describing a BAM binary file that is not necessarily sorted"""
edam_format = "format_2572"
edam_data = "data_0863"
file_ext = "bam_native"
file_ext = "unsorted.bam"
sort_flag = None
MetadataElement(name="bam_index", desc="BAM Index File", param=metadata.FileParameter, file_ext="bai", readonly=True, no_value=None, visible=False, optional=True)
lib/galaxy/datatypes/converters/bam_to_bigwig_converter.xml
+1 -1
View File
@@ -17,7 +17,7 @@
> temp.bg && bedGraphToBigWig temp.bg '$chromInfo' '$output']]>
</command>
<inputs>
<param format="bam,bam_native" name="input" type="data" label="Choose BAM file"/>
<param format="bam,unsorted.bam" name="input" type="data" label="Choose BAM file"/>
</inputs>
<outputs>
<data format="bigwig" name="output"/>
lib/galaxy/datatypes/converters/sam_to_bam_native.xml → lib/galaxy/datatypes/converters/sam_to_unsorted_bam.xml
+2 -2
View File
@@ -1,4 +1,4 @@
<tool id="CONVERTER_sam_to_bam_native" name="Convert SAM to BAM native - without sorting" version="1.0.0" profile="18.01">
<tool id="CONVERTER_sam_to_unsorted_bam" name="Convert SAM to BAM without sorting" version="1.0.0" profile="18.01">
<requirements>
<requirement type="package" version="1.6">samtools</requirement>
</requirements>
@@ -15,7 +15,7 @@
<param name="input" type="data" format="sam" label="SAM file"/>
</inputs>
<outputs>
<data name="output" format="bam_native"/>
<data name="output" format="unsorted.bam"/>
</outputs>
<help>
</help>
lib/galaxy/datatypes/converters/to_coordinate_sorted_bam.xml
+1 -1
View File
@@ -12,7 +12,7 @@
]]>
</command>
<inputs>
<param format="sam,bam_native,qname_sorted.bam" name="input" type="data" label="Choose a BAM native or queryname sortedfile"/>
<param format="sam,unsorted.bam,qname_sorted.bam" name="input" type="data" label="Choose a BAM native or queryname sortedfile"/>
</inputs>
<outputs>
<data format="bam" name="output"/>
lib/galaxy/datatypes/converters/to_qname_sorted_bam.xml
+1 -1
View File
@@ -13,7 +13,7 @@
]]>
</command>
<inputs>
<param format="sam,bam_native" name="input" type="data" label="Choose a BAM native or queryname sortedfile"/>
<param format="sam,unsorted.bam" name="input" type="data" label="Choose a BAM native or queryname sortedfile"/>
</inputs>
<outputs>
<data format="qname_sorted.bam" name="output"/>
lib/galaxy/datatypes/sniff.py
+18 -6
View File
@@ -14,6 +14,7 @@ import tempfile
import zipfile
from six import text_type
from six.moves.urllib.request import urlopen
from galaxy import util
from galaxy.util import compression_utils
@@ -39,6 +40,12 @@ def get_test_fname(fname):
return full_path
def stream_url_to_file(path):
page = urlopen(path) # page will be .close()ed in stream_to_file
temp_name = stream_to_file(page, prefix='url_paste', source_encoding=util.get_charset_from_http_headers(page.headers))
return temp_name
def stream_to_open_named_file(stream, fd, filename, source_encoding=None, source_error='strict', target_encoding=None, target_error='strict'):
"""Writes a stream to the provided file descriptor, returns the file name. Closes file descriptor"""
# signature and behavor is somewhat odd, due to backwards compatibility, but this can/should be done better
@@ -131,7 +138,7 @@ def convert_newlines(fname, in_place=True, tmp_dir=None, tmp_prefix="gxupload"):
return (i, temp_name)
def sep2tabs(fname, in_place=True, patt="\\s+"):
def sep2tabs(fname, in_place=True, patt="\\s+", tmp_dir=None, tmp_prefix="gxupload"):
"""
Transforms in place a 'sep' separated file to a tab separated one
@@ -143,13 +150,18 @@ def sep2tabs(fname, in_place=True, patt="\\s+"):
'1\\t2\\n3\\t4\\n'
"""
regexp = re.compile(patt)
fd, temp_name = tempfile.mkstemp()
fd, temp_name = tempfile.mkstemp(prefix=tmp_prefix, dir=tmp_dir)
with os.fdopen(fd, "wt") as fp:
i = None
for i, line in enumerate(open(fname)):
line = line.rstrip('\r\n')
elems = regexp.split(line)
fp.write("%s\n" % '\t'.join(elems))
if line.endswith("\r"):
line = line.rstrip('\r')
elems = regexp.split(line)
fp.write("%s\r" % '\t'.join(elems))
else:
line = line.rstrip('\n')
elems = regexp.split(line)
fp.write("%s\n" % '\t'.join(elems))
if i is None:
i = 0
else:
@@ -332,7 +344,7 @@ def guess_ext(fname, sniff_order):
'bam'
>>> fname = get_test_fname('3unsorted.bam')
>>> guess_ext(fname, sniff_order)
'bam_native'
'unsorted.bam'
>>> fname = get_test_fname('test.idpDB')
>>> guess_ext(fname, sniff_order)
'idpdb'
lib/galaxy/datatypes/upload_util.py
+47
View File
@@ -0,0 +1,47 @@
from galaxy.datatypes import sniff
from galaxy.datatypes.binary import Binary
class UploadProblemException(Exception):
def __init__(self, message):
self.message = message
def handle_unsniffable_binary_check(data_type, ext, path, name, is_binary, requested_ext, check_content, registry):
"""Return modified values of data_type and ext if unsniffable binary encountered.
Throw UploadProblemException if content problems or extension mismatches occur.
Precondition: check_binary called returned True.
"""
if is_binary or registry.is_extension_unsniffable_binary(requested_ext):
# We have a binary dataset, but it is not Bam, Sff or Pdf
data_type = 'binary'
parts = name.split(".")
if len(parts) > 1:
ext = parts[-1].strip().lower()
is_ext_unsniffable_binary = registry.is_extension_unsniffable_binary(ext)
if check_content and not is_ext_unsniffable_binary:
raise UploadProblemException('The uploaded binary file contains inappropriate content')
elif is_ext_unsniffable_binary and requested_ext != ext:
err_msg = "You must manually set the 'File Format' to '%s' when uploading %s files." % (ext, ext)
raise UploadProblemException(err_msg)
return data_type, ext
def handle_sniffable_binary_check(data_type, ext, path, registry):
"""Return modified values of data_type and ext if sniffable binary encountered.
Precondition: check_binary called returned True.
"""
# Sniff the data type
guessed_ext = sniff.guess_ext(path, registry.sniff_order)
# Set data_type only if guessed_ext is a binary datatype
datatype = registry.get_datatype_by_extension(guessed_ext)
if isinstance(datatype, Binary):
data_type = guessed_ext
ext = guessed_ext
return data_type, ext
lib/galaxy/dependencies/pinned-requirements.txt
+2
View File
@@ -15,6 +15,8 @@ uWSGI==2.0.15
pysam==0.14
# pure Python packages
bdbag==1.1.1
bleach==2.1.3
bz2file==0.98; python_version < '3.3'
ipaddress==1.0.18; python_version < '3.3'
boltons==17.1.0
lib/galaxy/dependencies/requirements.txt
+1
View File
@@ -13,6 +13,7 @@ pycrypto
pysam>=0.13
# pure Python packages
bleach
bz2file; python_version < '3.3'
ipaddress; python_version < '3.3'
boltons
lib/galaxy/jobs/__init__.py
+12 -2
View File
@@ -397,7 +397,17 @@ class JobConfiguration(ConfiguresHandlers):
return conditional_element
def __parse_resource_parameters(self):
self.resource_parameters = util.parse_resource_parameters(self.app.config.job_resource_params_file)
if os.path.exists(self.app.config.job_resource_params_file):
resource_param_file = self.app.config.job_resource_params_file
try:
resource_definitions = util.parse_xml(resource_param_file)
except Exception as e:
raise config_exception(e, resource_param_file)
resource_definitions_root = resource_definitions.getroot()
# TODO: Also handling conditionals would be awesome!
for parameter_elem in resource_definitions_root.findall("param"):
name = parameter_elem.get("name")
self.resource_parameters[name] = parameter_elem
def __get_params(self, parent):
"""Parses any child <param> tags in to a dictionary suitable for persistence.
@@ -1370,7 +1380,7 @@ class JobWrapper(HasResourceParameters):
collected_datasets = {
'primary': self.tool.collect_primary_datasets(out_data, self.get_tool_provided_job_metadata(), tool_working_directory, input_ext, input_dbkey)
}
self.tool.collect_dynamic_collections(
self.tool.collect_dynamic_outputs(
out_collections,
self.get_tool_provided_job_metadata(),
job_working_directory=tool_working_directory,
lib/galaxy/jobs/metrics/instrumenters/cgroup.py
+89
View File
@@ -0,0 +1,89 @@
"""The module describes the ``cgroup`` job metrics plugin."""
import logging
from galaxy.util import asbool, nice_size
from ..instrumenters import InstrumentPlugin
from ...metrics import formatting
log = logging.getLogger(__name__)
TITLES = {
"memory.memsw.max_usage_in_bytes": "Max memory usage (MEM+SWP)",
"memory.max_usage_in_bytes": "Max memory usage (MEM)",
"memory.limit_in_bytes": "Memory limit on cgroup (MEM)",
"memory.memsw.limit_in_bytes": "Memory limit on cgroup (MEM+SWP)",
"memory.soft_limit_in_bytes": "Memory softlimit on cgroup",
"memory.failcnt": "Failed to allocate memory count",
"memory.oom_control": "OOM Control enabled",
"under_oom": "Was OOM Killer active?",
"cpuacct.usage": "CPU Time"
}
CONVERSION = {
"memory.memsw.max_usage_in_bytes": nice_size,
"memory.max_usage_in_bytes": nice_size,
"memory.limit_in_bytes": nice_size,
"memory.memsw.limit_in_bytes": nice_size,
"memory.soft_limit_in_bytes": nice_size,
"under_oom": lambda x: "Yes" if x == "1" else "No",
"cpuacct.usage": lambda x: formatting.seconds_to_str(int(x) / 10**9) # convert nanoseconds
}
class CgroupPluginFormatter(formatting.JobMetricFormatter):
def format(self, key, value):
title = TITLES.get(key, key)
if key in CONVERSION:
return title, CONVERSION[key](value)
elif key.endswith("_bytes"):
try:
return title, nice_size(key)
except ValueError:
pass
return title, value
class CgroupPlugin(InstrumentPlugin):
""" Plugin that collects memory and cpu utilization from within a cgroup.
"""
plugin_type = "cgroup"
formatter = CgroupPluginFormatter()
def __init__(self, **kwargs):
self.verbose = asbool(kwargs.get("verbose", False))
def post_execute_instrument(self, job_directory):
commands = []
commands.append(self.__record_cgroup_cpu_usage(job_directory))
commands.append(self.__record_cgroup_memory_usage(job_directory))
return commands
def job_properties(self, job_id, job_directory):
metrics = self.__read_metrics(self.__cgroup_metrics_file(job_directory))
return metrics
def __record_cgroup_cpu_usage(self, job_directory):
return """if [ `command -v cgget` ] && [ -e /proc/$$/cgroup ]; then cat /proc/$$/cgroup | awk -F':' '$2=="cpuacct,cpu"{print $2":"$3}' | xargs -I{} cgget -g {} > %(metrics)s ; else echo "" > %(metrics)s; fi""" % {"metrics": self.__cgroup_metrics_file(job_directory)}
def __record_cgroup_memory_usage(self, job_directory):
return """if [ `command -v cgget` ] && [ -e /proc/$$/cgroup ]; then cat /proc/$$/cgroup | awk -F':' '$2=="memory"{print $2":"$3}' | xargs -I{} cgget -g {} >> %(metrics)s ; else echo "" > %(metrics)s; fi""" % {"metrics": self.__cgroup_metrics_file(job_directory)}
def __cgroup_metrics_file(self, job_directory):
return self._instrument_file_path(job_directory, "_metrics")
def __read_metrics(self, path):
metrics = {}
with open(path, "r") as infile:
for line in infile:
line = line.strip()
try:
key, value = line.split(": ")
if key in TITLES or self.verbose:
metrics[key] = value
except ValueError:
if line.startswith("under_oom"):
metrics["under_oom"] = line.split(" ")[1]
return metrics
__all__ = ('CgroupPlugin', )
lib/galaxy/managers/collections.py
+12 -7
View File
@@ -46,17 +46,22 @@ class DatasetCollectionManager(object):
self.tag_manager = tags.GalaxyTagManager(app.model.context)
self.ldda_manager = lddas.LDDAManager(app)
def precreate_dataset_collection_instance(self, trans, parent, name, implicit_inputs, implicit_output_name, structure):
def precreate_dataset_collection_instance(self, trans, parent, name, structure, implicit_inputs=None, implicit_output_name=None):
# TODO: prebuild all required HIDs and send them in so no need to flush in between.
dataset_collection = self.precreate_dataset_collection(structure)
dataset_collection = self.precreate_dataset_collection(structure, allow_unitialized_element=implicit_output_name is not None)
instance = self._create_instance_for_collection(
trans, parent, name, dataset_collection, implicit_inputs=implicit_inputs, implicit_output_name=implicit_output_name, flush=False
)
return instance
def precreate_dataset_collection(self, structure):
if structure.is_leaf or not structure.children_known:
return model.DatasetCollectionElement.UNINITIALIZED_ELEMENT
def precreate_dataset_collection(self, structure, allow_unitialized_element=True):
has_structure = not structure.is_leaf and structure.children_known
if not has_structure and allow_unitialized_element:
dataset_collection = model.DatasetCollectionElement.UNINITIALIZED_ELEMENT
elif not has_structure:
collection_type_description = structure.collection_type_description
dataset_collection = model.DatasetCollection(populated=False)
dataset_collection.collection_type = collection_type_description.collection_type
else:
collection_type_description = structure.collection_type_description
dataset_collection = model.DatasetCollection(populated=False)
@@ -67,7 +72,7 @@ class DatasetCollectionManager(object):
if substructure.is_leaf:
element = model.DatasetCollectionElement.UNINITIALIZED_ELEMENT
else:
element = self.precreate_dataset_collection(substructure)
element = self.precreate_dataset_collection(substructure, allow_unitialized_element=allow_unitialized_element)
element = model.DatasetCollectionElement(
element=element,
@@ -78,7 +83,7 @@ class DatasetCollectionManager(object):
dataset_collection.elements = elements
dataset_collection.element_count = len(elements)
return dataset_collection
return dataset_collection
def create(self, trans, parent, name, collection_type, element_identifiers=None,
elements=None, implicit_collection_info=None, trusted_identifiers=None,
lib/galaxy/managers/configuration.py
+2 -1
View File
@@ -73,7 +73,8 @@ class ConfigSerializer(base.ModelSerializer):
# TODO: is there no 'correct' way to get an api url? controller='api', action='tools' is a hack
# at any rate: the following works with path_prefix but is still brittle
# TODO: change this to (more generic) upload_path and incorporate config.nginx_upload_path into building it
'nginx_upload_path' : lambda i, k, **c: getattr(i, k, False) or self.url_for('/api/tools'),
'nginx_upload_path' : lambda i, k, **c: getattr(i, k, False),
'chunk_upload_size' : _defaults_to(104857600),
'ftp_upload_dir' : _defaults_to(None),
'ftp_upload_site' : _defaults_to(None),
'version_major' : _defaults_to(None),
lib/galaxy/managers/libraries.py
+4 -4
View File
@@ -139,10 +139,10 @@ class LibraryManager(object):
library_add_action = trans.app.security_agent.permitted_actions.LIBRARY_ADD.action
library_modify_action = trans.app.security_agent.permitted_actions.LIBRARY_MODIFY.action
library_manage_action = trans.app.security_agent.permitted_actions.LIBRARY_MANAGE.action
accessible_restricted_library_ids = []
allowed_library_add_ids = set([])
allowed_library_modify_ids = set([])
allowed_library_manage_ids = set([])
accessible_restricted_library_ids = set()
allowed_library_add_ids = set()
allowed_library_modify_ids = set()
allowed_library_manage_ids = set()
for action in all_actions:
if action.action == library_access_action:
accessible_restricted_library_ids.add(action.library_id)
lib/galaxy/managers/workflows.py
+8 -16
View File
@@ -36,7 +36,6 @@ from galaxy.workflow.modules import (
ToolModule,
WorkflowModuleInjector
)
from galaxy.workflow.resources import get_resource_mapper_function
from galaxy.workflow.steps import attach_ordered_steps
from .base import decode_id
@@ -206,7 +205,6 @@ class WorkflowContentsManager(UsesAnnotations):
def __init__(self, app):
self.app = app
self._resource_mapper_function = get_resource_mapper_function(app)
def build_workflow_from_dict(
self,
@@ -243,7 +241,7 @@ class WorkflowContentsManager(UsesAnnotations):
stored.user = trans.user
stored.published = publish
if data['annotation']:
annotation = sanitize_html(data['annotation'], 'utf-8', 'text/html')
annotation = sanitize_html(data['annotation'])
self.add_item_annotation(trans.sa_session, stored.user, stored, annotation)
workflow_tags = data.get('tags', [])
trans.app.tag_handler.set_tags_from_list(user=trans.user, item=stored, new_tags_list=workflow_tags)
@@ -434,20 +432,14 @@ class WorkflowContentsManager(UsesAnnotations):
step_model['messages'] = step.upgrade_messages
step_models.append(step_model)
return {
'id': trans.app.security.encode_id(stored.id),
'history_id': trans.app.security.encode_id(trans.history.id) if trans.history else None,
'name': stored.name,
'steps': step_models,
'step_version_changes': step_version_changes,
'has_upgrade_messages': has_upgrade_messages,
'workflow_resource_parameters': self._workflow_resource_parameters(trans, stored, workflow),
'id' : trans.app.security.encode_id(stored.id),
'history_id' : trans.app.security.encode_id(trans.history.id) if trans.history else None,
'name' : stored.name,
'steps' : step_models,
'step_version_changes' : step_version_changes,
'has_upgrade_messages' : has_upgrade_messages
}
def _workflow_resource_parameters(self, trans, stored, workflow):
"""Get workflow scheduling resource parameters for this user and workflow or None if unconfigured.
"""
return self._resource_mapper_function(trans=trans, stored_workflow=stored, workflow=workflow)
def _workflow_to_dict_editor(self, trans, stored):
workflow = stored.latest_workflow
# Pack workflow data into a dictionary and return
@@ -872,7 +864,7 @@ class WorkflowContentsManager(UsesAnnotations):
annotation = step_dict['annotation']
if annotation:
annotation = sanitize_html(annotation, 'utf-8', 'text/html')
annotation = sanitize_html(annotation)
self.add_item_annotation(trans.sa_session, trans.get_user(), step, annotation)
# Stick this in the step temporarily
lib/galaxy/model/__init__.py
+9 -14
View File
@@ -2035,6 +2035,12 @@ class DatasetInstance(object):
return self.dataset.set_file_name(filename)
file_name = property(get_file_name, set_file_name)
def link_to(self, path):
self.file_name = os.path.abspath(path)
# Since we are not copying the file into Galaxy's managed
# default file location, the dataset should never be purgable.
self.dataset.purgable = False
@property
def extra_files_path(self):
return self.dataset.extra_files_path
@@ -4067,7 +4073,7 @@ class WorkflowInvocation(UsesCreateAndUpdateTime, Dictifiable):
output_assoc.dataset_collection = output_object
self.output_dataset_collections.append(output_assoc)
else:
raise Exception("Uknown output type encountered")
raise Exception("Unknown output type encountered")
def to_dict(self, view='collection', value_mapper=None, step_details=False):
rval = super(WorkflowInvocation, self).to_dict(view=view, value_mapper=value_mapper)
@@ -4144,16 +4150,6 @@ class WorkflowInvocation(UsesCreateAndUpdateTime, Dictifiable):
request_to_content.workflow_step_id = step_id
self.input_step_parameters.append(request_to_content)
@property
def resource_parameters(self):
resource_type = WorkflowRequestInputParameter.types.RESOURCE_PARAMETERS
_resource_parameters = {}
for input_parameter in self.input_parameters:
if input_parameter.type == resource_type:
_resource_parameters[input_parameter.name] = input_parameter.value
return _resource_parameters
def has_input_for_step(self, step_id):
for content in self.input_datasets:
if content.workflow_step_id == step_id:
@@ -4201,7 +4197,7 @@ class WorkflowInvocationStep(Dictifiable):
output_assoc.output_name = output_name
self.output_dataset_collections.append(output_assoc)
else:
raise Exception("Uknown output type encountered")
raise Exception("Unknown output type encountered")
@property
def jobs(self):
@@ -4262,8 +4258,7 @@ class WorkflowRequestInputParameter(Dictifiable):
dict_collection_visible_keys = ['id', 'name', 'value', 'type']
types = Bunch(
REPLACEMENT_PARAMETERS='replacements',
META_PARAMETERS='meta',
RESOURCE_PARAMETERS='resource',
META_PARAMETERS='meta', #
)
def __init__(self, name=None, value=None, type=None):
lib/galaxy/tools/__init__.py
+56 -10
View File
@@ -26,7 +26,6 @@ from galaxy import (
model
)
from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper
from galaxy.managers import histories
from galaxy.managers.jobs import JobSearch
from galaxy.managers.tags import GalaxyTagManager
from galaxy.queue_worker import send_control_task
@@ -36,7 +35,6 @@ from galaxy.tools.actions.data_source import DataSourceToolAction
from galaxy.tools.actions.model_operations import ModelOperationToolAction
from galaxy.tools.deps import (
CachedDependencyManager,
views
)
from galaxy.tools.fetcher import ToolLocationFetcher
from galaxy.tools.parameters import (
@@ -69,6 +67,7 @@ from galaxy.tools.test import parse_tests
from galaxy.tools.toolbox import BaseGalaxyToolBox
from galaxy.util import (
ExecutionTimer,
in_directory,
listify,
Params,
rst_to_html,
@@ -104,6 +103,7 @@ MODEL_TOOLS_PATH = os.path.abspath(os.path.dirname(__file__))
# Tools that require Galaxy's Python environment to be preserved.
GALAXY_LIB_TOOLS_UNVERSIONED = [
"upload1",
"__DATA_FETCH__",
# Legacy tools bundled with Galaxy.
"vcf_to_maf_customtrack1",
"laj_1",
@@ -439,13 +439,19 @@ class Tool(Dictifiable):
except Exception as e:
global_tool_errors.add_error(config_file, "Tool Loading", e)
raise e
self.history_manager = histories.HistoryManager(app)
self._view = views.DependencyResolversView(app)
# The job search is only relevant in a galaxy context, and breaks
# loading tools into the toolshed for validation.
if self.app.name == 'galaxy':
self.job_search = JobSearch(app=self.app)
@property
def history_manager(self):
return self.app.history_manager
@property
def _view(self):
return self.app.dependency_resolvers_view
@property
def version_object(self):
return packaging.version.parse(self.version)
@@ -826,6 +832,44 @@ class Tool(Dictifiable):
self.__tests_populated = True
return self.__tests
@property
def _repository_dir(self):
"""If tool shed installed tool, the base directory of the repository installed."""
repository_dir = None
if hasattr(self, 'tool_shed') and self.tool_shed:
repository_dir = self.tool_dir
while True:
repository_dir_name = os.path.basename(repository_dir)
if repository_dir_name == self.repository_name:
break
parent_repository_dir = os.path.dirname(repository_dir)
if repository_dir == parent_repository_dir:
log.error("Problem finding repository dir for tool [%s]" % self.id)
repository_dir = None
return repository_dir
def test_data_path(self, filename):
repository_dir = self._repository_dir
if repository_dir:
for root, dirs, files in os.walk(repository_dir):
if '.hg' in dirs:
dirs.remove('.hg')
if 'test-data' in dirs:
test_data_dir = os.path.join(root, 'test-data')
result = os.path.abspath(os.path.join(test_data_dir, filename))
if not in_directory(result, test_data_dir):
# Don't raise an explicit exception and reveal details about what
# files are or are not on the path, simply return None and let the
# API raise a 404.
return None
else:
return result
else:
return self.app.test_data_resolver.get_filename(filename)
def tool_provided_metadata(self, job_wrapper):
meta_file = os.path.join(job_wrapper.tool_working_directory, self.provided_metadata_file)
# LEGACY: Remove in 17.XX
@@ -874,8 +918,7 @@ class Tool(Dictifiable):
# If we have an nginx upload, save the action as a tuple instead of
# a string. The actual action needs to get url_for run to add any
# prefixes, and we want to avoid adding the prefix to the
# nginx_upload_path. This logic is handled in the tool_form.mako
# template.
# nginx_upload_path.
if self.nginx_upload and self.app.config.nginx_upload_path:
if '?' in unquote_plus(self.action):
raise Exception('URL parameters in a non-default tool action can not be used '
@@ -1064,7 +1107,10 @@ class Tool(Dictifiable):
group.file_type_name = elem.get('file_type_name', group.file_type_name)
group.default_file_type = elem.get('default_file_type', group.default_file_type)
group.metadata_ref = elem.get('metadata_ref', group.metadata_ref)
rval[group.file_type_name].refresh_on_change = True
try:
rval[group.file_type_name].refresh_on_change = True
except KeyError:
pass
group_page_source = XmlPageSource(elem)
group.inputs = self.parse_input_elem(group_page_source, enctypes, context)
rval[group.name] = group
@@ -1615,10 +1661,10 @@ class Tool(Dictifiable):
"""
return output_collect.collect_primary_datasets(self, output, tool_provided_metadata, job_working_directory, input_ext, input_dbkey=input_dbkey)
def collect_dynamic_collections(self, output, tool_provided_metadata, **kwds):
""" Find files corresponding to dynamically structured collections.
def collect_dynamic_outputs(self, output, tool_provided_metadata, **kwds):
"""Collect dynamic outputs associated with a job from this tool.
"""
return output_collect.collect_dynamic_collections(self, output, tool_provided_metadata, **kwds)
return output_collect.collect_dynamic_outputs(self, output, tool_provided_metadata, **kwds)
def to_archive(self):
tool = self
lib/galaxy/tools/actions/__init__.py
+17 -6
View File
@@ -7,7 +7,8 @@ from six import string_types
from galaxy import model
from galaxy.exceptions import ObjectInvalid
from galaxy.model import LibraryDatasetDatasetAssociation
from galaxy.jobs.actions.post import ActionBox
from galaxy.model import LibraryDatasetDatasetAssociation, WorkflowRequestInputParameter
from galaxy.tools.parameters import update_dataset_ids
from galaxy.tools.parameters.basic import DataCollectionToolParameter, DataToolParameter, RuntimeValue
from galaxy.tools.parameters.wrapped import WrappedParameters
@@ -36,7 +37,8 @@ class ToolAction(object):
been converted and validated).
"""
def execute(self, tool, trans, incoming={}, set_output_hid=True):
def execute(self, tool, trans, incoming=None, set_output_hid=True):
incoming = incoming or {}
raise TypeError("Abstract method")
@@ -213,12 +215,13 @@ class DefaultToolAction(object):
return history, inp_data, inp_dataset_collections, preserved_tags
def execute(self, tool, trans, incoming={}, return_job=False, set_output_hid=True, history=None, job_params=None, rerun_remap_job_id=None, execution_cache=None, dataset_collection_elements=None, completed_job=None):
def execute(self, tool, trans, incoming=None, return_job=False, set_output_hid=True, history=None, job_params=None, rerun_remap_job_id=None, execution_cache=None, dataset_collection_elements=None, completed_job=None):
"""
Executes a tool, creating job and tool outputs, associating them, and
submitting the job to the job queue. If history is not specified, use
trans.history as destination for tool's output datasets.
"""
incoming = incoming or {}
self._check_access(tool, trans)
app = trans.app
if execution_cache is None:
@@ -491,7 +494,6 @@ class DefaultToolAction(object):
rerun_remap_job_id=rerun_remap_job_id,
current_job=job,
out_data=out_data)
log.info("Setup for job %s complete, ready to flush %s" % (job.log_str(), job_setup_timer))
job_flush_timer = ExecutionTimer()
@@ -544,6 +546,15 @@ class DefaultToolAction(object):
# Duplicate PJAs before remap.
for pjaa in old_job.post_job_actions:
current_job.add_post_job_action(pjaa.post_job_action)
if old_job.workflow_invocation_step:
replacement_dict = {}
for parameter in old_job.workflow_invocation_step.workflow_invocation.input_parameters:
if parameter.type == WorkflowRequestInputParameter.types.REPLACEMENT_PARAMETERS:
replacement_dict[parameter.name] = parameter.value
for pja in old_job.workflow_invocation_step.workflow_step.post_job_actions:
# execute immediate actions here, with workflow context.
if pja.action_type in ActionBox.immediate_actions:
ActionBox.execute(trans.app, trans.sa_session, pja, current_job, replacement_dict)
for p in old_job.parameters:
if p.name.endswith('|__identifier__'):
current_job.parameters.append(p.copy())
@@ -598,7 +609,7 @@ class DefaultToolAction(object):
def _get_on_text(self, inp_data):
input_names = []
for name, data in reversed(inp_data.items()):
for data in reversed(inp_data.values()):
if getattr(data, "hid", None):
input_names.append('data %s' % data.hid)
@@ -782,7 +793,7 @@ class OutputCollections(object):
if "elements" in element_kwds:
elements = element_kwds["elements"]
if hasattr(elements, "items"): # else it is ELEMENTS_UNINITIALIZED object.
for key, value in elements.items():
for value in elements.values():
# Either a HDA (if) or a DatasetCollection (the else)
if getattr(value, "history_content_type", None) == "dataset":
assert value.history is not None
lib/galaxy/tools/actions/upload.py
+121 -9
View File
@@ -1,15 +1,20 @@
import json
import logging
import os
from galaxy.dataset_collections.structure import UnitializedTree
from galaxy.exceptions import RequestParameterMissingException
from galaxy.tools.actions import upload_common
from galaxy.util import ExecutionTimer
from galaxy.util.bunch import Bunch
from . import ToolAction
log = logging.getLogger(__name__)
class UploadToolAction(ToolAction):
class BaseUploadToolAction(ToolAction):
def execute(self, tool, trans, incoming={}, set_output_hid=True, history=None, **kwargs):
def execute(self, tool, trans, incoming={}, history=None, **kwargs):
dataset_upload_inputs = []
for input_name, input in tool.inputs.items():
if input.type == "upload_dataset":
@@ -19,18 +24,125 @@ class UploadToolAction(ToolAction):
persisting_uploads_timer = ExecutionTimer()
incoming = upload_common.persist_uploads(incoming, trans)
log.debug("Persisted uploads %s" % persisting_uploads_timer)
rval = self._setup_job(tool, trans, incoming, dataset_upload_inputs, history)
return rval
def _setup_job(self, tool, trans, incoming, dataset_upload_inputs, history):
"""Take persisted uploads and create a job for given tool."""
def _create_job(self, *args, **kwds):
"""Wrapper around upload_common.create_job with a timer."""
create_job_timer = ExecutionTimer()
rval = upload_common.create_job(*args, **kwds)
log.debug("Created upload job %s" % create_job_timer)
return rval
class UploadToolAction(BaseUploadToolAction):
def _setup_job(self, tool, trans, incoming, dataset_upload_inputs, history):
check_timer = ExecutionTimer()
# We can pass an empty string as the cntrller here since it is used to check whether we
# are in an admin view, and this tool is currently not used there.
uploaded_datasets = upload_common.get_uploaded_datasets(trans, '', incoming, dataset_upload_inputs, history=history)
if not uploaded_datasets:
return None, 'No data was entered in the upload form, please go back and choose data to upload.'
log.debug("Checked uploads %s" % check_timer)
create_job_timer = ExecutionTimer()
json_file_path = upload_common.create_paramfile(trans, uploaded_datasets)
data_list = [ud.data for ud in uploaded_datasets]
rval = upload_common.create_job(trans, incoming, tool, json_file_path, data_list, history=history)
log.debug("Created upload job %s" % create_job_timer)
return rval
log.debug("Checked uploads %s" % check_timer)
return self._create_job(
trans, incoming, tool, json_file_path, data_list, history=history
)
class FetchUploadToolAction(BaseUploadToolAction):
def _setup_job(self, tool, trans, incoming, dataset_upload_inputs, history):
# Now replace references in requests with these.
files = incoming.get("files", [])
files_iter = iter(files)
request = json.loads(incoming.get("request_json"))
def replace_file_srcs(request_part):
if isinstance(request_part, dict):
if request_part.get("src", None) == "files":
path_def = next(files_iter)
if path_def is None or path_def["file_data"] is None:
raise RequestParameterMissingException("Failed to find uploaded file matching target with src='files'")
request_part["path"] = path_def["file_data"]["local_filename"]
if "name" not in request_part:
request_part["name"] = path_def["file_data"]["filename"]
request_part["src"] = "path"
else:
for key, value in request_part.items():
replace_file_srcs(value)
elif isinstance(request_part, list):
for value in request_part:
replace_file_srcs(value)
replace_file_srcs(request)
outputs = []
for target in request.get("targets", []):
destination = target.get("destination")
destination_type = destination.get("type")
# Start by just pre-creating HDAs.
if destination_type == "hdas":
if target.get("elements_from"):
# Dynamic collection required I think.
continue
_precreate_fetched_hdas(trans, history, target, outputs)
if destination_type == "hdca":
_precreate_fetched_collection_instance(trans, history, target, outputs)
incoming["request_json"] = json.dumps(request)
return self._create_job(
trans, incoming, tool, None, outputs, history=history
)
def _precreate_fetched_hdas(trans, history, target, outputs):
for item in target.get("elements", []):
name = item.get("name", None)
if name is None:
src = item.get("src", None)
if src == "url":
url = item.get("url")
if name is None:
name = url.split("/")[-1]
elif src == "path":
path = item["path"]
if name is None:
name = os.path.basename(path)
file_type = item.get("ext", "auto")
dbkey = item.get("dbkey", "?")
uploaded_dataset = Bunch(
type='file', name=name, file_type=file_type, dbkey=dbkey
)
data = upload_common.new_upload(trans, '', uploaded_dataset, library_bunch=None, history=history)
outputs.append(data)
item["object_id"] = data.id
def _precreate_fetched_collection_instance(trans, history, target, outputs):
collection_type = target.get("collection_type")
if not collection_type:
# Can't precreate collections of unknown type at this time.
return
name = target.get("name")
if not name:
return
collections_service = trans.app.dataset_collections_service
collection_type_description = collections_service.collection_type_descriptions.for_collection_type(collection_type)
structure = UnitializedTree(collection_type_description)
hdca = collections_service.precreate_dataset_collection_instance(
trans, history, name, structure=structure
)
outputs.append(hdca)
# Following flushed needed for an ID.
trans.sa_session.flush()
target["destination"]["object_id"] = hdca.id
lib/galaxy/tools/actions/upload_common.py
+28 -20
View File
@@ -16,7 +16,7 @@ except ImportError:
from urllib.parse import urlparse
from galaxy import datatypes, util
from galaxy.exceptions import ObjectInvalid
from galaxy.exceptions import ConfigDoesNotAllowException, ObjectInvalid
from galaxy.managers import tags
from galaxy.util import unicodify
from galaxy.util.odict import odict
@@ -102,7 +102,7 @@ def validate_url(url, ip_whitelist):
pass
else:
# Otherwise, we deny access.
raise Exception("Access to this address in not permitted by server configuration")
raise ConfigDoesNotAllowException("Access to this address in not permitted by server configuration")
return url
@@ -123,7 +123,7 @@ def persist_uploads(params, trans):
local_filename=local_filename)
elif type(f) == dict and 'local_filename' not in f:
raise Exception('Uploaded file was encoded in a way not understood by Galaxy.')
if upload_dataset['url_paste'] and upload_dataset['url_paste'].strip() != '':
if 'url_paste' in upload_dataset and upload_dataset['url_paste'] and upload_dataset['url_paste'].strip() != '':
upload_dataset['url_paste'] = datatypes.sniff.stream_to_file(
StringIO(validate_url(upload_dataset['url_paste'], trans.app.config.fetch_url_whitelist_ips)),
prefix="strio_url_paste_"
@@ -380,7 +380,7 @@ def create_paramfile(trans, uploaded_datasets):
return json_file_path
def create_job(trans, params, tool, json_file_path, data_list, folder=None, history=None, job_params=None):
def create_job(trans, params, tool, json_file_path, outputs, folder=None, history=None, job_params=None):
"""
Create the upload job.
"""
@@ -408,21 +408,28 @@ def create_job(trans, params, tool, json_file_path, data_list, folder=None, hist
job.add_parameter(name, value)
job.add_parameter('paramfile', dumps(json_file_path))
object_store_id = None
for i, dataset in enumerate(data_list):
if folder:
job.add_output_library_dataset('output%i' % i, dataset)
for i, output_object in enumerate(outputs):
output_name = "output%i" % i
if hasattr(output_object, "collection"):
job.add_output_dataset_collection(output_name, output_object)
output_object.job = job
else:
job.add_output_dataset('output%i' % i, dataset)
# Create an empty file immediately
if not dataset.dataset.external_filename:
dataset.dataset.object_store_id = object_store_id
try:
trans.app.object_store.create(dataset.dataset)
except ObjectInvalid:
raise Exception('Unable to create output dataset: object store is full')
object_store_id = dataset.dataset.object_store_id
trans.sa_session.add(dataset)
# open( dataset.file_name, "w" ).close()
dataset = output_object
if folder:
job.add_output_library_dataset(output_name, dataset)
else:
job.add_output_dataset(output_name, dataset)
# Create an empty file immediately
if not dataset.dataset.external_filename:
dataset.dataset.object_store_id = object_store_id
try:
trans.app.object_store.create(dataset.dataset)
except ObjectInvalid:
raise Exception('Unable to create output dataset: object store is full')
object_store_id = dataset.dataset.object_store_id
trans.sa_session.add(output_object)
job.object_store_id = object_store_id
job.set_state(job.states.NEW)
job.set_handler(tool.get_job_handler(None))
@@ -436,8 +443,9 @@ def create_job(trans, params, tool, json_file_path, data_list, folder=None, hist
trans.app.job_manager.job_queue.put(job.id, job.tool_id)
trans.log_event("Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id)
output = odict()
for i, v in enumerate(data_list):
output['output%i' % i] = v
for i, v in enumerate(outputs):
if not hasattr(output_object, "collection_type"):
output['output%i' % i] = v
return job, output
lib/galaxy/tools/data_fetch.py
+325
View File
@@ -0,0 +1,325 @@
import argparse
import errno
import json
import os
import shutil
import sys
import tempfile
import bdbag.bdbag_api
from galaxy.datatypes import sniff
from galaxy.datatypes.registry import Registry
from galaxy.datatypes.upload_util import (
handle_sniffable_binary_check,
handle_unsniffable_binary_check,
UploadProblemException,
)
from galaxy.util import in_directory
from galaxy.util.checkers import (
check_binary,
check_html,
)
from galaxy.util.compression_utils import CompressedFile
DESCRIPTION = """Data Import Script"""
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
args = _arg_parser().parse_args(argv)
registry = Registry()
registry.load_datatypes(root_dir=args.galaxy_root, config=args.datatypes_registry)
request_path = args.request
assert os.path.exists(request_path)
with open(request_path) as f:
request = json.load(f)
upload_config = UploadConfig(request, registry)
galaxy_json = _request_to_galaxy_json(upload_config, request)
with open("galaxy.json", "w") as f:
json.dump(galaxy_json, f)
def _request_to_galaxy_json(upload_config, request):
targets = request.get("targets", [])
fetched_targets = []
for target in targets:
fetched_target = _fetch_target(upload_config, target)
fetched_targets.append(fetched_target)
return {"__unnamed_outputs": fetched_targets}
def _fetch_target(upload_config, target):
destination = target.get("destination", None)
assert destination, "No destination defined."
def expand_elements_from(target_or_item):
elements_from = target_or_item.get("elements_from", None)
items = None
if elements_from:
if elements_from == "archive":
decompressed_directory = _decompress_target(target_or_item)
items = _directory_to_items(decompressed_directory)
elif elements_from == "bagit":
_, elements_from_path = _has_src_to_path(target_or_item)
items = _bagit_to_items(elements_from_path)
elif elements_from == "bagit_archive":
decompressed_directory = _decompress_target(target_or_item)
items = _bagit_to_items(decompressed_directory)
elif elements_from == "directory":
_, elements_from_path = _has_src_to_path(target_or_item)
items = _directory_to_items(elements_from_path)
else:
raise Exception("Unknown elements from type encountered [%s]" % elements_from)
if items:
del target_or_item["elements_from"]
target_or_item["elements"] = items
_for_each_src(expand_elements_from, target)
items = target.get("elements", None)
assert items is not None, "No element definition found for destination [%s]" % destination
fetched_target = {}
fetched_target["destination"] = destination
if "collection_type" in target:
fetched_target["collection_type"] = target["collection_type"]
if "name" in target:
fetched_target["name"] = target["name"]
def _resolve_src(item):
converted_path = None
name, path = _has_src_to_path(item)
dbkey = item.get("dbkey", "?")
requested_ext = item.get("ext", "auto")
info = item.get("info", None)
object_id = item.get("object_id", None)
link_data_only = upload_config.link_data_only
if "link_data_only" in item:
# Allow overriding this on a per file basis.
link_data_only = _link_data_only(item)
to_posix_lines = upload_config.get_option(item, "to_posix_lines")
space_to_tab = upload_config.get_option(item, "space_to_tab")
in_place = item.get("in_place", False)
purge_source = item.get("purge_source", True)
# Follow upload.py logic but without the auto-decompress logic.
registry = upload_config.registry
check_content = upload_config.check_content
data_type, ext = None, requested_ext
is_binary = check_binary(path)
if is_binary:
data_type, ext = handle_sniffable_binary_check(data_type, ext, path, registry)
if data_type is None:
root_datatype = registry.get_datatype_by_extension(ext)
if getattr(root_datatype, 'compressed', False):
data_type = 'compressed archive'
ext = ext
elif is_binary:
data_type, ext = handle_unsniffable_binary_check(
data_type, ext, path, name, is_binary, requested_ext, check_content, registry
)
if not data_type and check_content and check_html(path):
raise UploadProblemException('The uploaded file contains inappropriate HTML content')
if data_type != 'binary':
if not link_data_only:
if to_posix_lines:
if space_to_tab:
line_count, converted_path = sniff.convert_newlines_sep2tabs(path, in_place=in_place, tmp_dir=".")
else:
line_count, converted_path = sniff.convert_newlines(path, in_place=in_place, tmp_dir=".")
else:
if space_to_tab:
line_count, converted_path = sniff.sep2tabs(path, in_place=in_place, tmp_dir=".")
if requested_ext == 'auto':
ext = sniff.guess_ext(converted_path or path, registry.sniff_order)
else:
ext = requested_ext
data_type = ext
if ext == 'auto' and data_type == 'binary':
ext = 'data'
if ext == 'auto' and requested_ext:
ext = requested_ext
if ext == 'auto':
ext = 'data'
datatype = registry.get_datatype_by_extension(ext)
if link_data_only:
# Never alter a file that will not be copied to Galaxy's local file store.
if datatype.dataset_content_needs_grooming(path):
err_msg = 'The uploaded files need grooming, so change your <b>Copy data into Galaxy?</b> selection to be ' + \
'<b>Copy files into Galaxy</b> instead of <b>Link to files without copying into Galaxy</b> so grooming can be performed.'
raise UploadProblemException(err_msg)
# If this file is not in the workdir make sure it gets there.
if not link_data_only and converted_path:
path = upload_config.ensure_in_working_directory(converted_path, purge_source, in_place)
elif not link_data_only:
path = upload_config.ensure_in_working_directory(path, purge_source, in_place)
if not link_data_only and datatype and datatype.dataset_content_needs_grooming(path):
# Groom the dataset content if necessary
datatype.groom_dataset_content(path)
rval = {"name": name, "filename": path, "dbkey": dbkey, "ext": ext, "link_data_only": link_data_only}
if info is not None:
rval["info"] = info
if object_id is not None:
rval["object_id"] = object_id
return rval
elements = elements_tree_map(_resolve_src, items)
fetched_target["elements"] = elements
return fetched_target
def _bagit_to_items(directory):
bdbag.bdbag_api.resolve_fetch(directory)
bdbag.bdbag_api.validate_bag(directory)
items = _directory_to_items(os.path.join(directory, "data"))
return items
def _decompress_target(target):
elements_from_name, elements_from_path = _has_src_to_path(target)
temp_directory = tempfile.mkdtemp(prefix=elements_from_name, dir=".")
decompressed_directory = CompressedFile(elements_from_path).extract(temp_directory)
return decompressed_directory
def elements_tree_map(f, items):
new_items = []
for item in items:
if "elements" in item:
new_item = item.copy()
new_item["elements"] = elements_tree_map(f, item["elements"])
new_items.append(new_item)
else:
new_items.append(f(item))
return new_items
def _directory_to_items(directory):
items = []
dir_elements = {}
for root, dirs, files in os.walk(directory):
if root in dir_elements:
target = dir_elements[root]
else:
target = items
for dir in dirs:
dir_dict = {"name": dir, "elements": []}
dir_elements[os.path.join(root, dir)] = dir_dict["elements"]
target.append(dir_dict)
for file in files:
target.append({"src": "path", "path": os.path.join(root, file)})
return items
def _has_src_to_path(item):
assert "src" in item, item
src = item.get("src")
name = item.get("name")
if src == "url":
url = item.get("url")
path = sniff.stream_url_to_file(url)
if name is None:
name = url.split("/")[-1]
else:
assert src == "path"
path = item["path"]
if name is None:
name = os.path.basename(path)
return name, path
def _arg_parser():
parser = argparse.ArgumentParser(description=DESCRIPTION)
parser.add_argument("--galaxy-root")
parser.add_argument("--datatypes-registry")
parser.add_argument("--request-version")
parser.add_argument("--request")
return parser
class UploadConfig(object):
def __init__(self, request, registry):
self.registry = registry
self.check_content = request.get("check_content" , True)
self.to_posix_lines = request.get("to_posix_lines", False)
self.space_to_tab = request.get("space_to_tab", False)
self.link_data_only = _link_data_only(request)
self.__workdir = os.path.abspath(".")
self.__upload_count = 0
def get_option(self, item, key):
"""Return item[key] if specified otherwise use default from UploadConfig.
This default represents the default for the whole request instead item which
is the option for individual files.
"""
if key in item:
return item[key]
else:
return getattr(self, key)
def __new_dataset_path(self):
path = "gxupload_%d" % self.__upload_count
self.__upload_count += 1
return path
def ensure_in_working_directory(self, path, purge_source, in_place):
if in_directory(path, self.__workdir):
return path
new_path = self.__new_dataset_path()
if purge_source:
try:
shutil.move(path, new_path)
except OSError as e:
# We may not have permission to remove converted_path
if e.errno != errno.EACCES:
raise
else:
shutil.copy(path, new_path)
return new_path
def _link_data_only(has_config_dict):
link_data_only = has_config_dict.get("link_data_only", False)
if not isinstance(link_data_only, bool):
# Allow the older string values of 'copy_files' and 'link_to_files'
link_data_only = link_data_only == "copy_files"
return link_data_only
def _for_each_src(f, obj):
if isinstance(obj, list):
for item in obj:
_for_each_src(f, item)
if isinstance(obj, dict):
if "src" in obj:
f(obj)
for key, value in obj.items():
_for_each_src(f, value)
if __name__ == "__main__":
main()
lib/galaxy/tools/data_fetch.xml
+33
View File
@@ -0,0 +1,33 @@
<tool id="__DATA_FETCH__"
name="Data Fetch"
version="0.1.0"
workflow_compatible="false"
profile="18.01">
<action module="galaxy.tools.actions.upload" class="FetchUploadToolAction"/>
<command><![CDATA[
python '$__tool_directory__/data_fetch.py'
--galaxy-root '$GALAXY_ROOT_DIR'
--datatypes-registry '$GALAXY_DATATYPES_CONF_FILE'
--request-version '$request_version'
--request '$request_path'
]]></command>
<inputs nginx_upload="true">
<param type="text" name="request_version" value="1">
</param>
<param type="text" name="request_json">
<sanitizer sanitize="False">
</sanitizer>
</param>
<upload_dataset name="files">
<param name="file_data" type="file" label="File" ajax-upload="true" file_type_name="">
</param>
</upload_dataset>
</inputs>
<configfiles>
<configfile name="request_path">$request_json</configfile>
</configfiles>
<outputs>
</outputs>
<help><![CDATA[
]]></help>
</tool>
lib/galaxy/tools/deps/mulled/mulled_build.py
+5 -1
View File
@@ -18,6 +18,7 @@ import subprocess
import sys
from sys import platform as _platform
from six.moves import shlex_quote
try:
import yaml
except ImportError:
@@ -47,6 +48,7 @@ DEFAULT_WORKING_DIR = '/source/'
IS_OS_X = _platform == "darwin"
INVOLUCRO_VERSION = "1.1.2"
DEST_BASE_IMAGE = os.environ.get('DEST_BASE_IMAGE', None)
CONDA_IMAGE = os.environ.get('CONDA_IMAGE', None)
SINGULARITY_TEMPLATE = """Bootstrap: docker
From: bgruening/busybox-bash:0.1
@@ -195,7 +197,7 @@ def mull_targets(
involucro_args = [
'-f', '%s/invfile.lua' % DIRNAME,
'-set', "CHANNELS='%s'" % channels,
'-set', "TEST='%s'" % test,
'-set', "TEST=%s" % shlex_quote(test),
'-set', "TARGETS='%s'" % target_str,
'-set', "REPO='%s'" % repo,
'-set', "BINDS='%s'" % bind_str,
@@ -203,6 +205,8 @@ def mull_targets(
if DEST_BASE_IMAGE:
involucro_args.extend(["-set", "DEST_BASE_IMAGE='%s'" % DEST_BASE_IMAGE])
if CONDA_IMAGE:
involucro_args.extend(["-set", "CONDA_IMAGE='%s'" % CONDA_IMAGE])
if verbose:
involucro_args.extend(["-set", "VERBOSE='1'"])
if singularity:
lib/galaxy/tools/execute.py
+3 -8
View File
@@ -31,7 +31,7 @@ class PartialJobExecution(Exception):
MappingParameters = collections.namedtuple("MappingParameters", ["param_template", "param_combinations"])
def execute(trans, tool, mapping_params, history, rerun_remap_job_id=None, collection_info=None, workflow_invocation_uuid=None, invocation_step=None, max_num_jobs=None, job_callback=None, completed_jobs=None, workflow_resource_parameters=None):
def execute(trans, tool, mapping_params, history, rerun_remap_job_id=None, collection_info=None, workflow_invocation_uuid=None, invocation_step=None, max_num_jobs=None, job_callback=None, completed_jobs=None):
"""
Execute a tool and return object containing summary (output data, number of
failures, etc...).
@@ -58,12 +58,7 @@ def execute(trans, tool, mapping_params, history, rerun_remap_job_id=None, colle
# Only workflow invocation code gets to set this, ignore user supplied
# values or rerun parameters.
del params['__workflow_invocation_uuid__']
if workflow_resource_parameters:
params['__workflow_resource_params__'] = workflow_resource_parameters
elif '__workflow_resource_params__' in params:
# Only workflow invocation code gets to set this, ignore user supplied
# values or rerun parameters.
del params['__workflow_resource_params__']
job, result = tool.handle_single_execution(trans, rerun_remap_job_id, execution_slice, history, execution_cache, completed_job)
if job:
message = EXECUTION_SUCCESS_MESSAGE % (tool.id, job.id, job_timer)
@@ -280,9 +275,9 @@ class ExecutionTracker(object):
trans=trans,
parent=history,
name=output_collection_name,
structure=effective_structure,
implicit_inputs=implicit_inputs,
implicit_output_name=output_name,
structure=effective_structure,
)
collection_instance.implicit_collection_jobs = implicit_collection_jobs
collection_instances[output_name] = collection_instance
lib/galaxy/tools/parameters/basic.py
+22 -9
View File
@@ -43,6 +43,8 @@ workflow_building_modes = Bunch(DISABLED=False, ENABLED=True, USE_HISTORY=1)
WORKFLOW_PARAMETER_REGULAR_EXPRESSION = re.compile('''\$\{.+?\}''')
MAX_DEFAULT_COLUMNS = 999
def contains_workflow_parameter(value, search=False):
if not isinstance(value, string_types):
@@ -519,11 +521,19 @@ class FileToolParameter(ToolParameter):
# Middleware or proxies may encode files in special ways (TODO: this
# should be pluggable)
if type(value) == dict:
upload_store = trans.app.config.nginx_upload_store
assert upload_store, "Request appears to have been processed by nginx_upload_module but Galaxy is not configured to recognize it."
# Check that the file is in the right location
local_filename = os.path.abspath(value['path'])
assert local_filename.startswith(upload_store), "Filename provided by nginx (%s) is not in correct directory (%s)." % (local_filename, upload_store)
if 'session_id' in value:
# handle api upload
session_id = value["session_id"]
upload_store = trans.app.config.new_file_path
if re.match('^[\w-]+$', session_id) is None:
raise ValueError("Invald session id format.")
local_filename = os.path.abspath(os.path.join(upload_store, session_id))
else:
# handle nginx upload
upload_store = trans.app.config.nginx_upload_store
assert upload_store, "Request appears to have been processed by nginx_upload_module but Galaxy is not configured to recognize it."
local_filename = os.path.abspath(value['path'])
assert local_filename.startswith(upload_store), "Filename provided by nginx (%s) is not in correct directory (%s)." % (local_filename, upload_store)
value = dict(filename=value["name"], local_filename=local_filename)
return value
@@ -1114,18 +1124,21 @@ class ColumnListParameter(SelectToolParameter):
if isinstance(dataset, trans.app.model.HistoryDatasetCollectionAssociation):
dataset = dataset.to_hda_representative()
# Columns can only be identified if metadata is available
if not hasattr(dataset, 'metadata') or not hasattr(dataset.metadata, 'columns') or not dataset.metadata.columns:
if not hasattr(dataset, 'metadata') or not hasattr(dataset.metadata, 'columns'):
return []
# Build up possible columns for this dataset
this_column_list = []
if self.numerical:
# Valid column-based datasets contain at least 1 column if that column has not been
# specified we prepopulate the selector assuming that the datasets is not ready yet.
if dataset.metadata.columns is None:
this_column_list = [str(i) for i in range(1, MAX_DEFAULT_COLUMNS + 1)]
elif self.numerical:
# If numerical was requested, filter columns based on metadata
for i, col in enumerate(dataset.metadata.column_types):
if col == 'int' or col == 'float':
this_column_list.append(str(i + 1))
else:
for i in range(0, dataset.metadata.columns):
this_column_list.append(str(i + 1))
this_column_list = [str(i) for i in range(1, dataset.metadata.columns + 1)]
# Take the intersection of these columns with the other columns.
if column_list is None:
column_list = this_column_list
lib/galaxy/tools/parameters/output_collect.py
+262 -15
View File
@@ -9,9 +9,11 @@ import re
from collections import namedtuple
from galaxy import util
from galaxy.dataset_collections.structure import UnitializedTree
from galaxy.tools.parser.output_collection_def import (
DEFAULT_DATASET_COLLECTOR_DESCRIPTION,
INPUT_DBKEY_TOKEN,
ToolProvidedMetadataDatasetCollection,
)
from galaxy.util import (
ExecutionTimer,
@@ -34,6 +36,9 @@ class NullToolProvidedMetadata(object):
def has_failed_outputs(self):
return False
def get_unnamed_outputs(self):
return []
class LegacyToolProvidedMetadata(object):
@@ -84,6 +89,9 @@ class LegacyToolProvidedMetadata(object):
return found_failed
def get_unnamed_outputs(self):
return []
class ToolProvidedMetadata(object):
@@ -124,14 +132,21 @@ class ToolProvidedMetadata(object):
def has_failed_outputs(self):
found_failed = False
for meta in self.tool_provided_job_metadata.values():
for output_name, meta in self.tool_provided_job_metadata.items():
if output_name == "__unnamed_outputs":
continue
if meta.get("failed", False):
found_failed = True
return found_failed
def get_unnamed_outputs(self):
log.debug("unnamed outputs [%s]" % self.tool_provided_job_metadata)
return self.tool_provided_job_metadata.get("__unnamed_outputs", [])
def collect_dynamic_collections(
def collect_dynamic_outputs(
tool,
output_collections,
tool_provided_metadata,
@@ -140,6 +155,7 @@ def collect_dynamic_collections(
job=None,
input_dbkey="?",
):
app = tool.app
collections_service = tool.app.dataset_collections_service
job_context = JobContext(
tool,
@@ -149,6 +165,145 @@ def collect_dynamic_collections(
inp_data,
input_dbkey,
)
# unmapped outputs do not correspond to explicit outputs of the tool, they were inferred entirely
# from the tool provided metadata (e.g. galaxy.json).
for unnamed_output_dict in tool_provided_metadata.get_unnamed_outputs():
assert "destination" in unnamed_output_dict
assert "elements" in unnamed_output_dict
destination = unnamed_output_dict["destination"]
elements = unnamed_output_dict["elements"]
assert "type" in destination
destination_type = destination["type"]
assert destination_type in ["library_folder", "hdca", "hdas"]
trans = job_context.work_context
# three destination types we need to handle here - "library_folder" (place discovered files in a library folder),
# "hdca" (place discovered files in a history dataset collection), and "hdas" (place discovered files in a history
# as stand-alone datasets).
if destination_type == "library_folder":
# populate a library folder (needs to be already have been created)
library_folder_manager = app.library_folder_manager
library_folder = library_folder_manager.get(trans, app.security.decode_id(destination.get("library_folder_id")))
def add_elements_to_folder(elements, library_folder):
for element in elements:
if "elements" in element:
assert "name" in element
name = element["name"]
description = element.get("description")
nested_folder = library_folder_manager.create(trans, library_folder.id, name, description)
add_elements_to_folder(element["elements"], nested_folder)
else:
discovered_file = discovered_file_for_unnamed_output(element, job_working_directory)
fields_match = discovered_file.match
designation = fields_match.designation
visible = fields_match.visible
ext = fields_match.ext
dbkey = fields_match.dbkey
info = element.get("info", None)
link_data = discovered_file.match.link_data
# Create new primary dataset
name = fields_match.name or designation
job_context.create_dataset(
ext=ext,
designation=designation,
visible=visible,
dbkey=dbkey,
name=name,
filename=discovered_file.path,
info=info,
library_folder=library_folder,
link_data=link_data
)
add_elements_to_folder(elements, library_folder)
elif destination_type == "hdca":
# create or populate a dataset collection in the history
history = job.history
assert "collection_type" in unnamed_output_dict
object_id = destination.get("object_id")
if object_id:
sa_session = tool.app.model.context
hdca = sa_session.query(app.model.HistoryDatasetCollectionAssociation).get(int(object_id))
else:
name = unnamed_output_dict.get("name", "unnamed collection")
collection_type = unnamed_output_dict["collection_type"]
collection_type_description = collections_service.collection_type_descriptions.for_collection_type(collection_type)
structure = UnitializedTree(collection_type_description)
hdca = collections_service.precreate_dataset_collection_instance(
trans, history, name, structure=structure
)
filenames = odict.odict()
def add_to_discovered_files(elements, parent_identifiers=[]):
for element in elements:
if "elements" in element:
add_to_discovered_files(element["elements"], parent_identifiers + [element["name"]])
else:
discovered_file = discovered_file_for_unnamed_output(element, job_working_directory, parent_identifiers)
filenames[discovered_file.path] = discovered_file
add_to_discovered_files(elements)
collection = hdca.collection
collection_builder = collections_service.collection_builder_for(
collection
)
job_context.populate_collection_elements(
collection,
collection_builder,
filenames,
)
collection_builder.populate()
elif destination_type == "hdas":
# discover files as individual datasets for the target history
history = job.history
datasets = []
def collect_elements_for_history(elements):
for element in elements:
if "elements" in element:
collect_elements_for_history(element["elements"])
else:
discovered_file = discovered_file_for_unnamed_output(element, job_working_directory)
fields_match = discovered_file.match
designation = fields_match.designation
ext = fields_match.ext
dbkey = fields_match.dbkey
info = element.get("info", None)
link_data = discovered_file.match.link_data
# Create new primary dataset
name = fields_match.name or designation
hda_id = discovered_file.match.object_id
primary_dataset = None
if hda_id:
sa_session = tool.app.model.context
primary_dataset = sa_session.query(app.model.HistoryDatasetAssociation).get(hda_id)
dataset = job_context.create_dataset(
ext=ext,
designation=designation,
visible=True,
dbkey=dbkey,
name=name,
filename=discovered_file.path,
info=info,
link_data=link_data,
primary_data=primary_dataset,
)
dataset.raw_set_dataset_state('ok')
if not hda_id:
datasets.append(dataset)
collect_elements_for_history(elements)
job.history.add_datasets(job_context.sa_session, datasets)
for name, has_collection in output_collections.items():
if name not in tool.output_collections:
@@ -168,13 +323,19 @@ def collect_dynamic_collections(
collection.populated_state = collection.populated_states.NEW
try:
collection_builder = collections_service.collection_builder_for(
collection
)
dataset_collectors = map(dataset_collector, output_collection_def.dataset_collector_descriptions)
output_name = output_collection_def.name
filenames = job_context.find_files(output_name, collection, dataset_collectors)
job_context.populate_collection_elements(
collection,
collection_builder,
output_collection_def,
filenames,
name=output_collection_def.name,
metadata_source_name=output_collection_def.metadata_source,
)
collection_builder.populate()
except Exception:
@@ -194,6 +355,11 @@ class JobContext(object):
self.tool_provided_metadata = tool_provided_metadata
self._permissions = None
@property
def work_context(self):
from galaxy.work.context import WorkRequestContext
return WorkRequestContext(self.app, user=self.job.user)
@property
def permissions(self):
if self._permissions is None:
@@ -214,15 +380,14 @@ class JobContext(object):
filenames[discovered_file.path] = discovered_file
return filenames
def populate_collection_elements(self, collection, root_collection_builder, output_collection_def):
def populate_collection_elements(self, collection, root_collection_builder, filenames, name=None, metadata_source_name=None):
# TODO: allow configurable sorting.
# <sort by="lexical" /> <!-- default -->
# <sort by="reverse_lexical" />
# <sort regex="example.(\d+).fastq" by="1:numerical" />
# <sort regex="part_(\d+)_sample_([^_]+).fastq" by="2:lexical,1:numerical" />
dataset_collectors = map(dataset_collector, output_collection_def.dataset_collector_descriptions)
output_name = output_collection_def.name
filenames = self.find_files(output_name, collection, dataset_collectors)
if name is None:
name = "unnamed output"
element_datasets = []
for filename, discovered_file in filenames.items():
@@ -241,6 +406,8 @@ class JobContext(object):
# Create new primary dataset
name = fields_match.name or designation
link_data = discovered_file.match.link_data
dataset = self.create_dataset(
ext=ext,
designation=designation,
@@ -248,14 +415,15 @@ class JobContext(object):
dbkey=dbkey,
name=name,
filename=filename,
metadata_source_name=output_collection_def.metadata_source,
metadata_source_name=metadata_source_name,
link_data=link_data,
)
log.debug(
"(%s) Created dynamic collection dataset for path [%s] with element identifier [%s] for output [%s] %s",
self.job.id,
filename,
designation,
output_collection_def.name,
name,
create_dataset_timer,
)
element_datasets.append((element_identifiers, dataset))
@@ -270,7 +438,7 @@ class JobContext(object):
log.debug(
"(%s) Add dynamic collection datsets to history for output [%s] %s",
self.job.id,
output_collection_def.name,
name,
add_datasets_timer,
)
@@ -300,12 +468,24 @@ class JobContext(object):
dbkey,
name,
filename,
metadata_source_name,
metadata_source_name=None,
info=None,
library_folder=None,
link_data=False,
primary_data=None,
):
app = self.app
sa_session = self.sa_session
primary_data = _new_hda(app, sa_session, ext, designation, visible, dbkey, self.permissions)
if primary_data is None:
if not library_folder:
primary_data = _new_hda(app, sa_session, ext, designation, visible, dbkey, self.permissions)
else:
primary_data = _new_ldda(self.work_context, name, ext, visible, dbkey, library_folder)
else:
primary_data.extension = ext
primary_data.visible = visible
primary_data.dbkey = dbkey
# Copy metadata from one of the inputs if requested.
metadata_source = None
@@ -314,7 +494,11 @@ class JobContext(object):
sa_session.flush()
# Move data from temp location to dataset location
app.object_store.update_from_file(primary_data.dataset, file_name=filename, create=True)
if not link_data:
app.object_store.update_from_file(primary_data.dataset, file_name=filename, create=True)
else:
primary_data.link_to(filename)
primary_data.set_size()
# If match specified a name use otherwise generate one from
# designation.
@@ -325,6 +509,9 @@ class JobContext(object):
else:
primary_data.init_meta()
if info is not None:
primary_data.info = info
primary_data.set_meta()
primary_data.set_peek()
@@ -491,6 +678,20 @@ def discover_files(output_name, tool_provided_metadata, extra_file_collectors, j
yield DiscoveredFile(match.path, collector, match)
def discovered_file_for_unnamed_output(dataset, job_working_directory, parent_identifiers=[]):
extra_file_collector = DEFAULT_TOOL_PROVIDED_DATASET_COLLECTOR
target_directory = discover_target_directory(extra_file_collector.directory, job_working_directory)
filename = dataset["filename"]
# handle link_data_only here, verify filename is in directory if not linking...
if not dataset.get("link_data_only"):
path = os.path.join(target_directory, filename)
if not util.in_directory(path, target_directory):
raise Exception("Problem with tool configuration, attempting to pull in datasets from outside working directory.")
else:
path = filename
return DiscoveredFile(path, extra_file_collector, JsonCollectedDatasetMatch(dataset, extra_file_collector, filename, path=path, parent_identifiers=parent_identifiers))
def discover_target_directory(dir_name, job_working_directory):
if dir_name:
directory = os.path.join(job_working_directory, dir_name)
@@ -605,11 +806,12 @@ def _compose(f, g):
class JsonCollectedDatasetMatch(object):
def __init__(self, as_dict, collector, filename, path=None):
def __init__(self, as_dict, collector, filename, path=None, parent_identifiers=[]):
self.as_dict = as_dict
self.collector = collector
self.filename = filename
self.path = path
self._parent_identifiers = parent_identifiers
@property
def designation(self):
@@ -627,7 +829,7 @@ class JsonCollectedDatasetMatch(object):
@property
def element_identifiers(self):
return self.raw_element_identifiers or [self.designation]
return self._parent_identifiers + (self.raw_element_identifiers or [self.designation])
@property
def raw_element_identifiers(self):
@@ -664,6 +866,14 @@ class JsonCollectedDatasetMatch(object):
except KeyError:
return self.collector.default_visible
@property
def link_data(self):
return bool(self.as_dict.get("link_data_only", False))
@property
def object_id(self):
return self.as_dict.get("object_id", None)
class RegexCollectedDatasetMatch(JsonCollectedDatasetMatch):
@@ -676,6 +886,42 @@ class RegexCollectedDatasetMatch(JsonCollectedDatasetMatch):
UNSET = object()
def _new_ldda(
trans,
name,
ext,
visible,
dbkey,
library_folder,
):
ld = trans.app.model.LibraryDataset(folder=library_folder, name=name)
trans.sa_session.add(ld)
trans.sa_session.flush()
trans.app.security_agent.copy_library_permissions(trans, library_folder, ld)
ldda = trans.app.model.LibraryDatasetDatasetAssociation(name=name,
extension=ext,
dbkey=dbkey,
library_dataset=ld,
user=trans.user,
create_dataset=True,
sa_session=trans.sa_session)
trans.sa_session.add(ldda)
ldda.state = ldda.states.OK
# Permissions must be the same on the LibraryDatasetDatasetAssociation and the associated LibraryDataset
trans.app.security_agent.copy_library_permissions(trans, ld, ldda)
# Copy the current user's DefaultUserPermissions to the new LibraryDatasetDatasetAssociation.dataset
trans.app.security_agent.set_all_dataset_permissions(ldda.dataset, trans.app.security_agent.user_get_default_permissions(trans.user))
library_folder.add_library_dataset(ld, genome_build=dbkey)
trans.sa_session.add(library_folder)
trans.sa_session.flush()
ld.library_dataset_dataset_association_id = ldda.id
trans.sa_session.add(ld)
trans.sa_session.flush()
return ldda
def _new_hda(
app,
sa_session,
@@ -702,3 +948,4 @@ def _new_hda(
DEFAULT_DATASET_COLLECTOR = DatasetCollector(DEFAULT_DATASET_COLLECTOR_DESCRIPTION)
DEFAULT_TOOL_PROVIDED_DATASET_COLLECTOR = ToolMetadataDatasetCollector(ToolProvidedMetadataDatasetCollection())
lib/galaxy/tools/parser/interface.py
+74 -8
View File
@@ -357,26 +357,77 @@ class ToolStdioExitCode(object):
class TestCollectionDef(object):
# TODO: do not require XML directly here.
def __init__(self, elem, parse_param_elem):
self.elements = []
def __init__(self, attrib, name, collection_type, elements):
self.attrib = attrib
self.collection_type = collection_type
self.elements = elements
self.name = name
@staticmethod
def from_xml(elem, parse_param_elem):
elements = []
attrib = dict(elem.attrib)
self.collection_type = attrib["type"]
self.name = attrib.get("name", "Unnamed Collection")
collection_type = attrib["type"]
name = attrib.get("name", "Unnamed Collection")
for element in elem.findall("element"):
element_attrib = dict(element.attrib)
element_identifier = element_attrib["name"]
nested_collection_elem = element.find("collection")
if nested_collection_elem is not None:
self.elements.append((element_identifier, TestCollectionDef(nested_collection_elem, parse_param_elem)))
element_definition = TestCollectionDef.from_xml(nested_collection_elem, parse_param_elem)
else:
self.elements.append((element_identifier, parse_param_elem(element)))
element_definition = parse_param_elem(element)
elements.append({"element_identifier": element_identifier, "element_definition": element_definition})
return TestCollectionDef(
attrib=attrib,
collection_type=collection_type,
elements=elements,
name=name,
)
def to_dict(self):
def element_to_dict(element_dict):
element_identifier, element_def = element_dict["element_identifier"], element_dict["element_definition"]
if isinstance(element_def, TestCollectionDef):
element_def = element_def.to_dict()
return {
"element_identifier": element_identifier,
"element_definition": element_def,
}
return {
"model_class": "TestCollectionDef",
"attributes": self.attrib,
"collection_type": self.collection_type,
"elements": map(element_to_dict, self.elements or []),
"name": self.name,
}
@staticmethod
def from_dict(as_dict):
assert as_dict["model_class"] == "TestCollectionDef"
def element_from_dict(element_dict):
if "element_definition" not in element_dict:
raise Exception("Invalid element_dict %s" % element_dict)
element_def = element_dict["element_definition"]
if element_def.get("model_class", None) == "TestCollectionDef":
element_def = TestCollectionDef.from_dict(element_def)
return {"element_identifier": element_dict["element_identifier"], "element_definition": element_def}
return TestCollectionDef(
attrib=as_dict["attributes"],
name=as_dict["name"],
elements=list(map(element_from_dict, as_dict["elements"] or [])),
collection_type=as_dict["collection_type"],
)
def collect_inputs(self):
inputs = []
for element in self.elements:
value = element[1]
value = element["element_definition"]
if isinstance(value, TestCollectionDef):
inputs.extend(value.collect_inputs())
else:
@@ -393,3 +444,18 @@ class TestCollectionOutputDef(object):
self.count = int(count) if count is not None else None
self.attrib = attrib
self.element_tests = element_tests
@staticmethod
def from_dict(as_dict):
return TestCollectionOutputDef(
name=as_dict["name"],
attrib=as_dict["attributes"],
element_tests=as_dict["element_tests"],
)
def to_dict(self):
return dict(
name=self.name,
attributes=self.attrib,
element_tests=self.element_tests
)
lib/galaxy/tools/parser/xml.py
+24 -18
View File
@@ -391,8 +391,6 @@ class XmlToolSource(ToolSource):
for i, test_elem in enumerate(tests_elem.findall("test")):
tests.append(_test_elem_to_dict(test_elem, i))
_copy_to_dict_if_present(tests_elem, rval, ["interactor"])
return rval
def parse_profile(self):
@@ -418,7 +416,7 @@ def _test_elem_to_dict(test_elem, i):
expect_failure=string_as_bool(test_elem.get("expect_failure", False)),
maxseconds=test_elem.get("maxseconds", None),
)
_copy_to_dict_if_present(test_elem, rval, ["interactor", "num_outputs"])
_copy_to_dict_if_present(test_elem, rval, ["num_outputs"])
return rval
@@ -431,7 +429,7 @@ def __parse_output_elems(test_elem):
outputs = []
for output_elem in test_elem.findall("output"):
name, file, attributes = __parse_output_elem(output_elem)
outputs.append((name, file, attributes))
outputs.append({"name": name, "value": file, "attributes": attributes})
return outputs
@@ -464,7 +462,7 @@ def __parse_output_collection_elem(output_collection_elem):
if name is None:
raise Exception("Test output collection does not have a 'name'")
element_tests = __parse_element_tests(output_collection_elem)
return TestCollectionOutputDef(name, attrib, element_tests)
return TestCollectionOutputDef(name, attrib, element_tests).to_dict()
def __parse_element_tests(parent_element):
@@ -565,7 +563,12 @@ def __parse_extra_files_elem(extra):
assert extra_type == 'directory' or extra_name is not None, \
'extra_files type (%s) requires a name attribute' % extra_type
extra_value, extra_attributes = __parse_test_attributes(extra, attrib)
return extra_type, extra_value, extra_name, extra_attributes
return {
"value": extra_value,
"name": extra_name,
"type": extra_type,
"attributes": extra_attributes
}
def __expand_input_elems(root_elem, prefix=""):
@@ -628,8 +631,8 @@ def _copy_to_dict_if_present(elem, rval, attributes):
def __parse_inputs_elems(test_elem, i):
raw_inputs = []
for param_elem in test_elem.findall("param"):
name, value, attrib = __parse_param_elem(param_elem, i)
raw_inputs.append((name, value, attrib))
raw_inputs.append(__parse_param_elem(param_elem, i))
return raw_inputs
@@ -641,40 +644,43 @@ def __parse_param_elem(param_elem, i=0):
value = attrib['value']
else:
value = None
attrib['children'] = param_elem
if attrib['children'] is not None:
children_elem = param_elem
if children_elem is not None:
# At this time, we can assume having children only
# occurs on DataToolParameter test items but this could
# change and would cause the below parsing to change
# based upon differences in children items
attrib['metadata'] = []
attrib['metadata'] = {}
attrib['composite_data'] = []
attrib['edit_attributes'] = []
# Composite datasets need to be renamed uniquely
composite_data_name = None
for child in attrib['children']:
for child in children_elem:
if child.tag == 'composite_data':
attrib['composite_data'].append(child)
file_name = child.get("value")
attrib['composite_data'].append(file_name)
if composite_data_name is None:
# Generate a unique name; each test uses a
# fresh history.
composite_data_name = '_COMPOSITE_RENAMED_t%d_%s' \
% (i, uuid.uuid1().hex)
elif child.tag == 'metadata':
attrib['metadata'].append(child)
elif child.tag == 'metadata':
attrib['metadata'].append(child)
attrib['metadata'][child.get("name")] = child.get("value")
elif child.tag == 'edit_attributes':
attrib['edit_attributes'].append(child)
elif child.tag == 'collection':
attrib['collection'] = TestCollectionDef(child, __parse_param_elem)
attrib['collection'] = TestCollectionDef.from_xml(child, __parse_param_elem)
if composite_data_name:
# Composite datasets need implicit renaming;
# inserted at front of list so explicit declarations
# take precedence
attrib['edit_attributes'].insert(0, {'type': 'name', 'value': composite_data_name})
name = attrib.pop('name')
return (name, value, attrib)
return {
"name": name,
"value": value,
"attributes": attrib
}
class StdioParser(object):
lib/galaxy/tools/parser/yaml.py
+7 -3
View File
@@ -188,7 +188,7 @@ def _parse_test(i, test_dict):
if _is_dict(inputs):
new_inputs = []
for key, value in inputs.items():
new_inputs.append((key, value, {}))
new_inputs.append({"name": key, "value": value, "attributes": {}})
test_dict["inputs"] = new_inputs
outputs = test_dict["outputs"]
@@ -202,7 +202,11 @@ def _parse_test(i, test_dict):
else:
file = value
attributes = {}
new_outputs.append((key, file, attributes))
new_outputs.append({
"name": key,
"value": file,
"attributes": attributes
})
else:
for output in outputs:
name = output["name"]
@@ -211,7 +215,7 @@ def _parse_test(i, test_dict):
new_outputs.append((name, value, attributes))
for output in new_outputs:
attributes = output[2]
attributes = output["attributes"]
defaults = {
'compare': 'diff',
'lines_diff': 0,
lib/galaxy/tools/special_tools.py
+1
View File
@@ -4,6 +4,7 @@ log = logging.getLogger(__name__)
SPECIAL_TOOLS = {
"history export": "galaxy/tools/imp_exp/exp_history_to_archive.xml",
"history import": "galaxy/tools/imp_exp/imp_history_from_archive.xml",
"data fetch": "galaxy/tools/data_fetch.xml",
}
lib/galaxy/tools/test.py
+180 -224
View File
@@ -6,6 +6,7 @@ from six import string_types
import galaxy.tools.parameters.basic
import galaxy.tools.parameters.grouping
from galaxy.tools.verify.interactor import ToolTestDescription
from galaxy.util import string_as_bool
try:
@@ -16,11 +17,6 @@ except ImportError:
log = logging.getLogger(__name__)
DEFAULT_FTYPE = 'auto'
DEFAULT_DBKEY = 'hg17'
DEFAULT_INTERACTOR = "api" # Default mechanism test code uses for interacting with Galaxy instance.
DEFAULT_MAX_SECS = None
@nottest
def parse_tests(tool, tests_source):
@@ -28,207 +24,128 @@ def parse_tests(tool, tests_source):
Build ToolTestBuilder objects for each "<test>" elements and
return default interactor (if any).
"""
default_interactor = os.environ.get('GALAXY_TEST_DEFAULT_INTERACTOR', DEFAULT_INTERACTOR)
tests_dict = tests_source.parse_tests_to_dict()
tests_default_interactor = tests_dict.get('interactor', default_interactor)
raw_tests_dict = tests_source.parse_tests_to_dict()
tests = []
for i, test_dict in enumerate(tests_dict.get('tests', [])):
test = ToolTestBuilder(tool, test_dict, i, default_interactor=tests_default_interactor)
for i, raw_test_dict in enumerate(raw_tests_dict.get('tests', [])):
test = description_from_tool_object(tool, i, raw_test_dict)
tests.append(test)
return tests
class ToolTestBuilder(object):
def description_from_tool_object(tool, test_index, raw_test_dict):
required_files = []
num_outputs = raw_test_dict.get('expect_num_outputs', None)
if num_outputs:
num_outputs = int(num_outputs)
try:
processed_inputs = _process_raw_inputs(tool, tool.inputs, raw_test_dict["inputs"], required_files)
processed_test_dict = {
"inputs": processed_inputs,
"outputs": raw_test_dict["outputs"],
"output_collections": raw_test_dict["output_collections"],
"num_outputs": num_outputs,
"command_line": raw_test_dict.get("command", None),
"stdout": raw_test_dict.get("stdout", None),
"stderr": raw_test_dict.get("stderr", None),
"expect_exit_code": raw_test_dict.get("expect_exit_code", None),
"expect_failure": raw_test_dict.get("expect_failure", False),
"required_files": required_files,
"tool_id": tool.id,
"test_index": test_index,
"error": False,
}
except Exception as e:
log.exception("Failed to load tool test number [%d] for %s" % (test_index, tool.id))
processed_test_dict = {
"tool_id": tool.id,
"test_index": test_index,
"inputs": {},
"error": True,
"exception": str(e),
}
return ToolTestDescription(processed_test_dict)
def _process_raw_inputs(tool, tool_inputs, raw_inputs, required_files, parent_context=None):
"""
Encapsulates information about a tool test, and allows creation of a
dynamic TestCase class (the unittest framework is very class oriented,
doing dynamic tests in this way allows better integration)
Recursively expand flat list of inputs into "tree" form of flat list
(| using to nest to new levels) structure and expand dataset
information as proceeding to populate self.required_files.
"""
def __init__(self, tool, test_dict, i, default_interactor):
name = test_dict.get('name', 'Test-%d' % (i + 1))
maxseconds = test_dict.get('maxseconds', DEFAULT_MAX_SECS)
if maxseconds is not None:
maxseconds = int(maxseconds)
self.tool = tool
self.name = name
self.maxseconds = maxseconds
self.required_files = []
self.inputs = {}
self.outputs = []
# By default do not making assertions on number of outputs - but to
# test filtering allow explicitly state number of outputs.
self.num_outputs = None
self.error = False
self.exception = None
self.__handle_test_dict(test_dict, i, default_interactor)
def test_data(self):
"""
Iterator over metadata representing the required files for upload.
"""
return test_data_iter(self.required_files)
def __matching_case_for_value(self, cond, declared_value):
test_param = cond.test_param
if isinstance(test_param, galaxy.tools.parameters.basic.BooleanToolParameter):
if declared_value is None:
# No explicit value for param in test case, determine from default
query_value = test_param.checked
else:
query_value = _process_bool_param_value(test_param, declared_value)
def matches_declared_value(case_value):
return _process_bool_param_value(test_param, case_value) == query_value
elif isinstance(test_param, galaxy.tools.parameters.basic.SelectToolParameter):
if declared_value is not None:
# Test case supplied explicit value to check against.
def matches_declared_value(case_value):
return case_value == declared_value
elif test_param.static_options:
# No explicit value in test case, not much to do if options are dynamic but
# if static options are available can find the one specified as default or
# fallback on top most option (like GUI).
for (name, value, selected) in test_param.static_options:
if selected:
default_option = name
else:
first_option = test_param.static_options[0]
first_option_value = first_option[1]
default_option = first_option_value
def matches_declared_value(case_value):
return case_value == default_option
else:
# No explicit value for this param and cannot determine a
# default - give up. Previously this would just result in a key
# error exception.
msg = "Failed to find test parameter value specification required for conditional %s" % cond.name
raise Exception(msg)
# Check the tool's defined cases against predicate to determine
# selected or default.
for i, case in enumerate(cond.cases):
if matches_declared_value(case.value):
return case
else:
msg_template = "%s - Failed to find case matching value (%s) for test parameter specification for conditional %s. Remainder of test behavior is unspecified."
msg = msg_template % (self.tool.id, declared_value, cond.name)
log.info(msg)
def __split_if_str(self, value):
split = isinstance(value, string_types)
if split:
value = value.split(",")
return value
def __handle_test_dict(self, test_dict, i, default_interactor):
try:
# Mechanism test code uses for interacting with Galaxy instance,
# until 'api' is the default switch this to API to use its new
# features. Once 'api' is the default set to 'twill' to use legacy
# features or workarounds.
self.interactor = test_dict.get('interactor', default_interactor)
self.inputs = self.__process_raw_inputs(self.tool.inputs, test_dict["inputs"])
self.outputs = test_dict["outputs"]
self.output_collections = test_dict["output_collections"]
num_outputs = test_dict.get('expect_num_outputs', None)
if num_outputs:
num_outputs = int(num_outputs)
self.num_outputs = num_outputs
self.command_line = test_dict.get("command", None)
self.stdout = test_dict.get("stdout", None)
self.stderr = test_dict.get("stderr", None)
self.expect_exit_code = test_dict.get("expect_exit_code", None)
self.expect_failure = test_dict.get("expect_failure", False)
self.md5 = test_dict.get("md5", None)
except Exception as e:
self.inputs = {}
self.error = True
self.exception = e
def __process_raw_inputs(self, tool_inputs, raw_inputs, parent_context=None):
"""
Recursively expand flat list of inputs into "tree" form of flat list
(| using to nest to new levels) structure and expand dataset
information as proceeding to populate self.required_files.
"""
parent_context = parent_context or RootParamContext()
expanded_inputs = {}
for key, value in tool_inputs.items():
if isinstance(value, galaxy.tools.parameters.grouping.Conditional):
cond_context = ParamContext(name=value.name, parent_context=parent_context)
case_context = ParamContext(name=value.test_param.name, parent_context=cond_context)
raw_input = case_context.extract_value(raw_inputs)
case_value = raw_input[1] if raw_input else None
case = self.__matching_case_for_value(value, case_value)
if case:
for input_name, input_value in case.inputs.items():
case_inputs = self.__process_raw_inputs({input_name: input_value}, raw_inputs, parent_context=cond_context)
expanded_inputs.update(case_inputs)
if not value.type == "text":
expanded_case_value = self.__split_if_str(case.value)
if case_value is not None:
# A bit tricky here - we are growing inputs with value
# that may be implicit (i.e. not defined by user just
# a default defined in tool). So we do not want to grow
# expanded_inputs and risk repeat block viewing this
# as a new instance with value defined and hence enter
# an infinite loop - hence the "case_value is not None"
# check.
processed_value = _process_simple_value(value.test_param, expanded_case_value)
expanded_inputs[case_context.for_state()] = processed_value
elif isinstance(value, galaxy.tools.parameters.grouping.Section):
context = ParamContext(name=value.name, parent_context=parent_context)
parent_context = parent_context or RootParamContext()
expanded_inputs = {}
for key, value in tool_inputs.items():
if isinstance(value, galaxy.tools.parameters.grouping.Conditional):
cond_context = ParamContext(name=value.name, parent_context=parent_context)
case_context = ParamContext(name=value.test_param.name, parent_context=cond_context)
raw_input_dict = case_context.extract_value(raw_inputs)
case_value = raw_input_dict["value"] if raw_input_dict else None
case = _matching_case_for_value(tool, value, case_value)
if case:
for input_name, input_value in case.inputs.items():
case_inputs = _process_raw_inputs(tool, {input_name: input_value}, raw_inputs, required_files, parent_context=cond_context)
expanded_inputs.update(case_inputs)
if not value.type == "text":
expanded_case_value = _split_if_str(case.value)
if case_value is not None:
# A bit tricky here - we are growing inputs with value
# that may be implicit (i.e. not defined by user just
# a default defined in tool). So we do not want to grow
# expanded_inputs and risk repeat block viewing this
# as a new instance with value defined and hence enter
# an infinite loop - hence the "case_value is not None"
# check.
processed_value = _process_simple_value(value.test_param, expanded_case_value)
expanded_inputs[case_context.for_state()] = processed_value
elif isinstance(value, galaxy.tools.parameters.grouping.Section):
context = ParamContext(name=value.name, parent_context=parent_context)
for r_name, r_value in value.inputs.items():
expanded_input = _process_raw_inputs(tool, {context.for_state(): r_value}, raw_inputs, required_files, parent_context=context)
if expanded_input:
expanded_inputs.update(expanded_input)
elif isinstance(value, galaxy.tools.parameters.grouping.Repeat):
repeat_index = 0
while True:
context = ParamContext(name=value.name, index=repeat_index, parent_context=parent_context)
updated = False
for r_name, r_value in value.inputs.items():
expanded_input = self.__process_raw_inputs({context.for_state(): r_value}, raw_inputs, parent_context=context)
expanded_input = _process_raw_inputs(tool, {context.for_state(): r_value}, raw_inputs, required_files, parent_context=context)
if expanded_input:
expanded_inputs.update(expanded_input)
elif isinstance(value, galaxy.tools.parameters.grouping.Repeat):
repeat_index = 0
while True:
context = ParamContext(name=value.name, index=repeat_index, parent_context=parent_context)
updated = False
for r_name, r_value in value.inputs.items():
expanded_input = self.__process_raw_inputs({context.for_state(): r_value}, raw_inputs, parent_context=context)
if expanded_input:
expanded_inputs.update(expanded_input)
updated = True
if not updated:
break
repeat_index += 1
else:
context = ParamContext(name=value.name, parent_context=parent_context)
raw_input = context.extract_value(raw_inputs)
if raw_input:
(name, param_value, param_extra) = raw_input
if not value.type == "text":
param_value = self.__split_if_str(param_value)
if isinstance(value, galaxy.tools.parameters.basic.DataToolParameter):
if not isinstance(param_value, list):
param_value = [param_value]
map(lambda v: self.__add_uploaded_dataset(context.for_state(), v, param_extra, value), param_value)
processed_value = param_value
elif isinstance(value, galaxy.tools.parameters.basic.DataCollectionToolParameter):
assert 'collection' in param_extra
collection_def = param_extra['collection']
for (name, value, extra) in collection_def.collect_inputs():
require_file(name, value, extra, self.required_files)
processed_value = collection_def
else:
processed_value = _process_simple_value(value, param_value)
expanded_inputs[context.for_state()] = processed_value
return expanded_inputs
def __add_uploaded_dataset(self, name, value, extra, input_parameter):
if value is None:
assert input_parameter.optional, '%s is not optional. You must provide a valid filename.' % name
return value
return require_file(name, value, extra, self.required_files)
updated = True
if not updated:
break
repeat_index += 1
else:
context = ParamContext(name=value.name, parent_context=parent_context)
raw_input_dict = context.extract_value(raw_inputs)
if raw_input_dict:
name = raw_input_dict["name"]
param_value = raw_input_dict["value"]
param_extra = raw_input_dict["attributes"]
if not value.type == "text":
param_value = _split_if_str(param_value)
if isinstance(value, galaxy.tools.parameters.basic.DataToolParameter):
if not isinstance(param_value, list):
param_value = [param_value]
map(lambda v: _add_uploaded_dataset(context.for_state(), v, param_extra, value, required_files), param_value)
processed_value = param_value
elif isinstance(value, galaxy.tools.parameters.basic.DataCollectionToolParameter):
assert 'collection' in param_extra
collection_def = param_extra['collection']
for input_dict in collection_def.collect_inputs():
name = input_dict["name"]
value = input_dict["value"]
attributes = input_dict["attributes"]
require_file(name, value, attributes, required_files)
processed_value = collection_def
else:
processed_value = _process_simple_value(value, param_value)
expanded_inputs[context.for_state()] = processed_value
return expanded_inputs
def _process_simple_value(param, param_value):
@@ -266,6 +183,69 @@ def _process_simple_value(param, param_value):
return processed_value
def _matching_case_for_value(tool, cond, declared_value):
test_param = cond.test_param
if isinstance(test_param, galaxy.tools.parameters.basic.BooleanToolParameter):
if declared_value is None:
# No explicit value for param in test case, determine from default
query_value = test_param.checked
else:
query_value = _process_bool_param_value(test_param, declared_value)
def matches_declared_value(case_value):
return _process_bool_param_value(test_param, case_value) == query_value
elif isinstance(test_param, galaxy.tools.parameters.basic.SelectToolParameter):
if declared_value is not None:
# Test case supplied explicit value to check against.
def matches_declared_value(case_value):
return case_value == declared_value
elif test_param.static_options:
# No explicit value in test case, not much to do if options are dynamic but
# if static options are available can find the one specified as default or
# fallback on top most option (like GUI).
for (name, value, selected) in test_param.static_options:
if selected:
default_option = name
else:
first_option = test_param.static_options[0]
first_option_value = first_option[1]
default_option = first_option_value
def matches_declared_value(case_value):
return case_value == default_option
else:
# No explicit value for this param and cannot determine a
# default - give up. Previously this would just result in a key
# error exception.
msg = "Failed to find test parameter value specification required for conditional %s" % cond.name
raise Exception(msg)
# Check the tool's defined cases against predicate to determine
# selected or default.
for i, case in enumerate(cond.cases):
if matches_declared_value(case.value):
return case
else:
msg_template = "%s - Failed to find case matching value (%s) for test parameter specification for conditional %s. Remainder of test behavior is unspecified."
msg = msg_template % (tool.id, declared_value, cond.name)
log.info(msg)
def _add_uploaded_dataset(name, value, extra, input_parameter, required_files):
if value is None:
assert input_parameter.optional, '%s is not optional. You must provide a valid filename.' % name
return value
return require_file(name, value, extra, required_files)
def _split_if_str(value):
split = isinstance(value, string_types)
if split:
value = value.split(",")
return value
def _process_bool_param_value(param, param_value):
assert isinstance(param, galaxy.tools.parameters.basic.BooleanToolParameter)
was_list = False
@@ -281,30 +261,6 @@ def _process_bool_param_value(param, param_value):
return [processed_value] if was_list else processed_value
@nottest
def test_data_iter(required_files):
for fname, extra in required_files:
data_dict = dict(
fname=fname,
metadata=extra.get('metadata', []),
composite_data=extra.get('composite_data', []),
ftype=extra.get('ftype', DEFAULT_FTYPE),
dbkey=extra.get('dbkey', DEFAULT_DBKEY),
)
edit_attributes = extra.get('edit_attributes', [])
# currently only renaming is supported
for edit_att in edit_attributes:
if edit_att.get('type', None) == 'name':
new_name = edit_att.get('value', None)
assert new_name, 'You must supply the new dataset name as the value tag of the edit_attributes tag'
data_dict['name'] = new_name
else:
raise Exception('edit_attributes type (%s) is unimplemented' % edit_att.get('type', None))
yield data_dict
def require_file(name, value, extra, required_files):
if (value, extra) not in required_files:
required_files.append((value, extra)) # these files will be uploaded
@@ -359,13 +315,13 @@ class ParamContext(object):
def __raw_param_found(self, param_name, raw_inputs):
index = None
for i, raw_input in enumerate(raw_inputs):
if raw_input[0] == param_name:
for i, raw_input_dict in enumerate(raw_inputs):
if raw_input_dict["name"] == param_name:
index = i
if index is not None:
raw_input = raw_inputs[index]
raw_input_dict = raw_inputs[index]
del raw_inputs[index]
return raw_input
return raw_input_dict
else:
return None
lib/galaxy/tools/toolbox/filters/__init__.py
+1 -1
View File
@@ -34,7 +34,7 @@ class FilterFactory(object):
filters = deepcopy(self.default_filters)
if trans.user:
for name, value in trans.user.preferences.items():
if value.strip():
if value and value.strip():
user_filters = listify(value, do_strip=True)
category = ''
if name == 'toolbox_tool_filters':
lib/galaxy/tools/verify/asserts/__init__.py
+7 -1
View File
@@ -2,6 +2,8 @@ import inspect
import logging
import sys
from galaxy.util import unicodify
log = logging.getLogger(__name__)
assertion_module_names = ['text', 'tabular', 'xml']
@@ -66,7 +68,11 @@ def verify_assertion(data, assertion_description):
# - <has_column_titles><with_name name="sequence"><with_name
# name="probability"></has_column_titles>.)
if "output" in assert_function_args:
args["output"] = data
# This was read in as bytes for checksum and such, but all current
# assertions expect text data. If binary assertions are added at
# some point, just checkout for "output_bytes" for instance and pass
# data in unchanged.
args["output"] = unicodify(data)
if "verify_assertions_function" in assert_function_args:
args["verify_assertions_function"] = verify_assertions
lib/galaxy/tools/verify/interactor.py
+982
View File
@@ -0,0 +1,982 @@
from __future__ import absolute_import
from __future__ import print_function
import os
import re
import sys
import time
from json import dumps
from logging import getLogger
try:
from nose.tools import nottest
except ImportError:
def nottest(x):
return x
try:
import requests
except ImportError:
requests = None
from six import StringIO, text_type
from galaxy import util
from galaxy.tools.parser.interface import TestCollectionDef, TestCollectionOutputDef
from galaxy.util.bunch import Bunch
from galaxy.util.odict import odict
from .asserts import verify_assertions
from ..verify import verify
log = getLogger(__name__)
# Off by default because it can pound the database pretty heavily
# and result in sqlite errors on larger tests or larger numbers of
# tests.
VERBOSE_ERRORS = util.asbool(os.environ.get("GALAXY_TEST_VERBOSE_ERRORS", False))
UPLOAD_ASYNC = util.asbool(os.environ.get("GALAXY_TEST_UPLOAD_ASYNC", True))
ERROR_MESSAGE_DATASET_SEP = "--------------------------------------"
DEFAULT_TOOL_TEST_WAIT = os.environ.get("GALAXY_TEST_DEFAULT_WAIT", 86400)
DEFAULT_FTYPE = 'auto'
# This following default dbkey was traditionally hg17 before Galaxy 18.05,
# restore this behavior by setting GALAXY_TEST_DEFAULT_DBKEY to hg17.
DEFAULT_DBKEY = os.environ.get("GALAXY_TEST_DEFAULT_DBKEY", "?")
DEFAULT_MAX_SECS = DEFAULT_TOOL_TEST_WAIT
def stage_data_in_history(galaxy_interactor, tool_id, all_test_data, history):
# Upload any needed files
upload_waits = []
assert tool_id
if UPLOAD_ASYNC:
for test_data in all_test_data:
upload_waits.append(galaxy_interactor.stage_data_async(test_data, history, tool_id))
for upload_wait in upload_waits:
upload_wait()
else:
for test_data in all_test_data:
upload_wait = galaxy_interactor.stage_data_async(test_data, history, tool_id)
upload_wait()
class GalaxyInteractorApi(object):
def __init__(self, **kwds):
self.api_url = "%s/api" % kwds["galaxy_url"].rstrip("/")
self.master_api_key = kwds["master_api_key"]
self.api_key = self.__get_user_key(kwds.get("api_key"), kwds.get("master_api_key"), test_user=kwds.get("test_user"))
self.keep_outputs_dir = kwds["keep_outputs_dir"]
self.uploads = {}
def __get_user_key(self, user_key, admin_key, test_user=None):
if not test_user:
test_user = "test@bx.psu.edu"
if user_key:
return user_key
test_user = self.ensure_user_with_email(test_user)
return self._post("users/%s/api_key" % test_user['id'], key=admin_key).json()
# def get_tools(self):
# response = self._get("tools?in_panel=false")
# assert response.status_code == 200, "Non 200 response from tool index API. [%s]" % response.content
# return response.json()
def get_tests_summary(self):
response = self._get("tools/tests_summary")
assert response.status_code == 200, "Non 200 response from tool tests available API. [%s]" % response.content
return response.json()
def get_tool_tests(self, tool_id, tool_version=None):
url = "tools/%s/test_data" % tool_id
if tool_version is not None:
url += "?tool_version=%s" % tool_version
response = self._get(url)
assert response.status_code == 200, "Non 200 response from tool test API. [%s]" % response.content
return response.json()
def verify_output(self, history_id, jobs, output_data, output_testdef, tool_id, maxseconds):
outfile = output_testdef.outfile
attributes = output_testdef.attributes
name = output_testdef.name
self.wait_for_jobs(history_id, jobs, maxseconds)
hid = self.__output_id(output_data)
# TODO: Twill version verifys dataset is 'ok' in here.
self.verify_output_dataset(history_id=history_id, hda_id=hid, outfile=outfile, attributes=attributes, tool_id=tool_id)
primary_datasets = attributes.get('primary_datasets', {})
if primary_datasets:
job_id = self._dataset_provenance(history_id, hid)["job_id"]
outputs = self._get("jobs/%s/outputs" % (job_id)).json()
for designation, (primary_outfile, primary_attributes) in primary_datasets.items():
primary_output = None
for output in outputs:
if output["name"] == '__new_primary_file_%s|%s__' % (name, designation):
primary_output = output
break
if not primary_output:
msg_template = "Failed to find primary dataset with designation [%s] for output with name [%s]"
msg_args = (designation, name)
raise Exception(msg_template % msg_args)
primary_hda_id = primary_output["dataset"]["id"]
self.verify_output_dataset(history_id, primary_hda_id, primary_outfile, primary_attributes, tool_id=tool_id)
def wait_for_jobs(self, history_id, jobs, maxseconds):
for job in jobs:
self.wait_for_job(job['id'], history_id, maxseconds)
def verify_output_dataset(self, history_id, hda_id, outfile, attributes, tool_id):
fetcher = self.__dataset_fetcher(history_id)
test_data_path_builder = self.__test_data_path_builder(tool_id)
verify_hid(
outfile,
hda_id=hda_id,
attributes=attributes,
dataset_fetcher=fetcher,
test_data_path_builder=test_data_path_builder,
keep_outputs_dir=self.keep_outputs_dir
)
self._verify_metadata(history_id, hda_id, attributes)
def _verify_metadata(self, history_id, hid, attributes):
"""Check dataset metadata.
ftype on output maps to `file_ext` on the hda's API description, `name`, `info`,
`dbkey` and `tags` all map to the API description directly. Other metadata attributes
are assumed to be datatype-specific and mapped with a prefix of `metadata_`.
"""
metadata = attributes.get('metadata', {}).copy()
for key, value in metadata.copy().items():
if key not in ['name', 'info', 'tags']:
new_key = "metadata_%s" % key
metadata[new_key] = metadata[key]
del metadata[key]
elif key == "info":
metadata["misc_info"] = metadata["info"]
del metadata["info"]
expected_file_type = attributes.get('ftype', None)
if expected_file_type:
metadata["file_ext"] = expected_file_type
if metadata:
time.sleep(5)
dataset = self._get("histories/%s/contents/%s" % (history_id, hid)).json()
for key, value in metadata.items():
try:
dataset_value = dataset.get(key, None)
def compare(val, expected):
if text_type(val) != text_type(expected):
msg = "Dataset metadata verification for [%s] failed, expected [%s] but found [%s]. Dataset API value was [%s]."
msg_params = (key, value, dataset_value, dataset)
msg = msg % msg_params
raise Exception(msg)
if isinstance(dataset_value, list):
value = text_type(value).split(",")
if len(value) != len(dataset_value):
msg = "Dataset metadata verification for [%s] failed, expected [%s] but found [%s], lists differ in length. Dataset API value was [%s]."
msg_params = (key, value, dataset_value, dataset)
msg = msg % msg_params
raise Exception(msg)
for val, expected in zip(dataset_value, value):
compare(val, expected)
else:
compare(dataset_value, value)
except KeyError:
msg = "Failed to verify dataset metadata, metadata key [%s] was not found." % key
raise Exception(msg)
def wait_for_job(self, job_id, history_id, maxseconds):
self.wait_for(lambda: not self.__job_ready(job_id, history_id), maxseconds=maxseconds)
def wait_for(self, func, **kwd):
sleep_amount = 0.2
slept = 0
walltime_exceeded = kwd.get("maxseconds", DEFAULT_TOOL_TEST_WAIT)
while slept <= walltime_exceeded:
result = func()
if result:
time.sleep(sleep_amount)
slept += sleep_amount
sleep_amount *= 2
else:
return
message = 'Tool test run exceeded walltime [total %s, max %s], terminating.' % (slept, walltime_exceeded)
log.info(message)
raise AssertionError(message)
def get_job_stdio(self, job_id):
job_stdio = self.__get_job_stdio(job_id).json()
return job_stdio
def __get_job(self, job_id):
return self._get('jobs/%s' % job_id)
def __get_job_stdio(self, job_id):
return self._get('jobs/%s?full=true' % job_id)
def new_history(self):
history_json = self._post("histories", {"name": "test_history"}).json()
return history_json['id']
@nottest
def test_data_path(self, tool_id, filename):
return self._get("tools/%s/test_data_path?filename=%s" % (tool_id, filename)).json()
def __output_id(self, output_data):
# Allow data structure coming out of tools API - {id: <id>, output_name: <name>, etc...}
# or simple id as comes out of workflow API.
try:
output_id = output_data.get('id')
except AttributeError:
output_id = output_data
return output_id
def stage_data_async(self, test_data, history_id, tool_id, async=True):
fname = test_data['fname']
tool_input = {
"file_type": test_data['ftype'],
"dbkey": test_data['dbkey'],
}
metadata = test_data.get("metadata", {})
if not hasattr(metadata, "items"):
raise Exception("Invalid metadata description found for input [%s] - [%s]" % (fname, metadata))
for name, value in test_data.get('metadata', {}).items():
tool_input["files_metadata|%s" % name] = value
composite_data = test_data['composite_data']
if composite_data:
files = {}
for i, file_name in enumerate(composite_data):
file_name = self.test_data_path(tool_id, file_name)
files["files_%s|file_data" % i] = open(file_name, 'rb')
tool_input.update({
"files_%d|type" % i: "upload_dataset",
})
name = test_data['name']
else:
file_name = self.test_data_path(tool_id, fname)
name = test_data.get('name', None)
if not name:
name = os.path.basename(file_name)
tool_input.update({
"files_0|NAME": name,
"files_0|type": "upload_dataset",
})
# TODO: Option to upload by path since we are getting the paths from Galaxy now it makes more
# sense to move this there.
files = {
"files_0|file_data": open(file_name, 'rb')
}
submit_response_object = self.__submit_tool(history_id, "upload1", tool_input, extra_data={"type": "upload_dataset"}, files=files)
if submit_response_object.status_code != 200:
raise Exception("Request to upload dataset failed [%s]" % submit_response_object.content)
submit_response = submit_response_object.json()
assert "outputs" in submit_response, "Invalid response from server [%s], expecteding outputs in response." % submit_response
outputs = submit_response["outputs"]
assert len(outputs) > 0, "Invalid response from server [%s], expecting an output dataset." % submit_response
dataset = outputs[0]
hid = dataset['id']
self.uploads[os.path.basename(fname)] = self.uploads[fname] = self.uploads[name] = {"src": "hda", "id": hid}
assert "jobs" in submit_response, "Invalid response from server [%s], expecting jobs in response." % submit_response
jobs = submit_response["jobs"]
assert len(jobs) > 0, "Invalid response from server [%s], expecting a job." % submit_response
return lambda: self.wait_for_job(jobs[0]["id"], history_id, DEFAULT_TOOL_TEST_WAIT)
def run_tool(self, testdef, history_id, resource_parameters={}):
# We need to handle the case where we've uploaded a valid compressed file since the upload
# tool will have uncompressed it on the fly.
inputs_tree = testdef.inputs.copy()
for key, value in inputs_tree.items():
values = [value] if not isinstance(value, list) else value
new_values = []
for value in values:
if isinstance(value, TestCollectionDef):
hdca_id = self._create_collection(history_id, value)
new_values = [dict(src="hdca", id=hdca_id)]
elif value in self.uploads:
new_values.append(self.uploads[value])
else:
new_values.append(value)
inputs_tree[key] = new_values
if resource_parameters:
inputs_tree["__job_resource|__job_resource__select"] = "yes"
for key, value in resource_parameters.items():
inputs_tree["__job_resource|%s" % key] = value
# HACK: Flatten single-value lists. Required when using expand_grouping
for key, value in inputs_tree.items():
if isinstance(value, list) and len(value) == 1:
inputs_tree[key] = value[0]
submit_response = self.__submit_tool(history_id, tool_id=testdef.tool_id, tool_input=inputs_tree)
submit_response_object = submit_response.json()
try:
return Bunch(
inputs=inputs_tree,
outputs=self.__dictify_outputs(submit_response_object),
output_collections=self.__dictify_output_collections(submit_response_object),
jobs=submit_response_object['jobs'],
)
except KeyError:
message = "Error creating a job for these tool inputs - %s" % submit_response_object['err_msg']
raise RunToolException(message, inputs_tree)
def _create_collection(self, history_id, collection_def):
create_payload = dict(
name=collection_def.name,
element_identifiers=dumps(self._element_identifiers(collection_def)),
collection_type=collection_def.collection_type,
history_id=history_id,
)
return self._post("dataset_collections", data=create_payload).json()["id"]
def _element_identifiers(self, collection_def):
element_identifiers = []
for element_dict in collection_def.elements:
element_identifier = element_dict["element_identifier"]
element_def = element_dict["element_definition"]
if isinstance(element_def, TestCollectionDef):
subelement_identifiers = self._element_identifiers(element_def)
element = dict(
name=element_identifier,
src="new_collection",
collection_type=element_def.collection_type,
element_identifiers=subelement_identifiers
)
else:
element = self.uploads[element_def["value"]].copy()
element["name"] = element_identifier
element_identifiers.append(element)
return element_identifiers
def __dictify_output_collections(self, submit_response):
output_collections_dict = odict()
for output_collection in submit_response['output_collections']:
output_collections_dict[output_collection.get("output_name")] = output_collection
return output_collections_dict
def __dictify_outputs(self, datasets_object):
# Convert outputs list to a dictionary that can be accessed by
# output_name so can be more flexiable about ordering of outputs
# but also allows fallback to legacy access as list mode.
outputs_dict = odict()
index = 0
for output in datasets_object['outputs']:
outputs_dict[index] = outputs_dict[output.get("output_name")] = output
index += 1
# Adding each item twice (once with index for backward compat),
# overiding length to reflect the real number of outputs.
outputs_dict.__len__ = lambda: index
return outputs_dict
def output_hid(self, output_data):
return output_data['id']
def delete_history(self, history):
return None
def __job_ready(self, job_id, history_id):
if job_id is None:
raise ValueError("__job_ready passed empty job_id")
job_json = self._get("jobs/%s" % job_id).json()
state = job_json['state']
try:
return self._state_ready(state, error_msg="Job in error state.")
except Exception:
if VERBOSE_ERRORS:
self._summarize_history(history_id)
raise
def _summarize_history(self, history_id):
if history_id is None:
raise ValueError("_summarize_history passed empty history_id")
print("Problem in history with id %s - summary of datasets below." % history_id)
try:
history_contents = self.__contents(history_id)
except Exception:
print("*TEST FRAMEWORK FAILED TO FETCH HISTORY DETAILS*")
for history_content in history_contents:
dataset = history_content
print(ERROR_MESSAGE_DATASET_SEP)
dataset_id = dataset.get('id', None)
print("| %d - %s (HID - NAME) " % (int(dataset['hid']), dataset['name']))
if history_content['history_content_type'] == 'dataset_collection':
history_contents_json = self._get("histories/%s/contents/dataset_collections/%s" % (history_id, history_content["id"])).json()
print("| Dataset Collection: %s" % history_contents_json)
continue
try:
dataset_info = self._dataset_info(history_id, dataset_id)
print("| Dataset State:")
print(self.format_for_summary(dataset_info.get("state"), "Dataset state is unknown."))
print("| Dataset Blurb:")
print(self.format_for_summary(dataset_info.get("misc_blurb", ""), "Dataset blurb was empty."))
print("| Dataset Info:")
print(self.format_for_summary(dataset_info.get("misc_info", ""), "Dataset info is empty."))
print("| Peek:")
print(self.format_for_summary(dataset_info.get("peek", ""), "Peek unavilable."))
except Exception:
print("| *TEST FRAMEWORK ERROR FETCHING DATASET DETAILS*")
try:
provenance_info = self._dataset_provenance(history_id, dataset_id)
print("| Dataset Job Standard Output:")
print(self.format_for_summary(provenance_info.get("stdout", ""), "Standard output was empty."))
print("| Dataset Job Standard Error:")
print(self.format_for_summary(provenance_info.get("stderr", ""), "Standard error was empty."))
except Exception:
print("| *TEST FRAMEWORK ERROR FETCHING JOB DETAILS*")
print("|")
print(ERROR_MESSAGE_DATASET_SEP)
def format_for_summary(self, blob, empty_message, prefix="| "):
contents = "\n".join(["%s%s" % (prefix, line.strip()) for line in StringIO(blob).readlines() if line.rstrip("\n\r")])
return contents or "%s*%s*" % (prefix, empty_message)
def _dataset_provenance(self, history_id, id):
provenance = self._get("histories/%s/contents/%s/provenance" % (history_id, id)).json()
return provenance
def _dataset_info(self, history_id, id):
dataset_json = self._get("histories/%s/contents/%s" % (history_id, id)).json()
return dataset_json
def __contents(self, history_id):
history_contents_json = self._get("histories/%s/contents" % history_id).json()
return history_contents_json
def _state_ready(self, state_str, error_msg):
if state_str == 'ok':
return True
elif state_str == 'error':
raise Exception(error_msg)
return False
def __submit_tool(self, history_id, tool_id, tool_input, extra_data={}, files=None):
data = dict(
history_id=history_id,
tool_id=tool_id,
inputs=dumps(tool_input),
**extra_data
)
return self._post("tools", files=files, data=data)
def ensure_user_with_email(self, email, password=None):
admin_key = self.master_api_key
all_users = self._get('users', key=admin_key).json()
try:
test_user = [user for user in all_users if user["email"] == email][0]
except IndexError:
username = re.sub('[^a-z-]', '--', email.lower())
password = password or 'testpass'
# If remote user middleware is enabled - this endpoint consumes
# ``remote_user_email`` otherwise it requires ``email``, ``password``
# and ``username``.
data = dict(
remote_user_email=email,
email=email,
password=password,
username=username,
)
test_user = self._post('users', data, key=admin_key).json()
return test_user
def __test_data_path_builder(self, tool_id):
return lambda filename: self.test_data_path(tool_id, filename)
def __dataset_fetcher(self, history_id):
def fetcher(hda_id, base_name=None):
url = "histories/%s/contents/%s/display?raw=true" % (history_id, hda_id)
if base_name:
url += "&filename=%s" % base_name
return self._get(url).content
return fetcher
def _post(self, path, data={}, files=None, key=None, admin=False, anon=False):
if not anon:
if not key:
key = self.api_key if not admin else self.master_api_key
data = data.copy()
data['key'] = key
return requests.post("%s/%s" % (self.api_url, path), data=data, files=files)
def _delete(self, path, data={}, key=None, admin=False, anon=False):
if not anon:
if not key:
key = self.api_key if not admin else self.master_api_key
data = data.copy()
data['key'] = key
return requests.delete("%s/%s" % (self.api_url, path), params=data)
def _patch(self, path, data={}, key=None, admin=False, anon=False):
if not anon:
if not key:
key = self.api_key if not admin else self.master_api_key
params = dict(key=key)
data = data.copy()
data['key'] = key
else:
params = {}
return requests.patch("%s/%s" % (self.api_url, path), params=params, data=data)
def _put(self, path, data={}, key=None, admin=False, anon=False):
if not anon:
if not key:
key = self.api_key if not admin else self.master_api_key
params = dict(key=key)
data = data.copy()
data['key'] = key
else:
params = {}
return requests.put("%s/%s" % (self.api_url, path), params=params, data=data)
def _get(self, path, data={}, key=None, admin=False, anon=False):
if not anon:
if not key:
key = self.api_key if not admin else self.master_api_key
data = data.copy()
data['key'] = key
if path.startswith("/api"):
path = path[len("/api"):]
url = "%s/%s" % (self.api_url, path)
return requests.get(url, params=data)
class RunToolException(Exception):
def __init__(self, message, inputs=None):
super(RunToolException, self).__init__(message)
self.inputs = inputs
# Galaxy specific methods - rest of this can be used with arbitrary files and such.
def verify_hid(filename, hda_id, attributes, test_data_path_builder, hid="", dataset_fetcher=None, keep_outputs_dir=False):
assert dataset_fetcher is not None
def verify_extra_files(extra_files):
_verify_extra_files_content(extra_files, hda_id, dataset_fetcher=dataset_fetcher, test_data_path_builder=test_data_path_builder, keep_outputs_dir=keep_outputs_dir)
data = dataset_fetcher(hda_id)
item_label = "History item %s" % hid
verify(
item_label,
data,
attributes=attributes,
filename=filename,
get_filename=test_data_path_builder,
keep_outputs_dir=keep_outputs_dir,
verify_extra_files=verify_extra_files,
)
def _verify_composite_datatype_file_content(file_name, hda_id, base_name=None, attributes=None, dataset_fetcher=None, test_data_path_builder=None, keep_outputs_dir=False):
assert dataset_fetcher is not None
data = dataset_fetcher(hda_id, base_name)
item_label = "History item %s" % hda_id
try:
verify(
item_label,
data,
attributes=attributes,
filename=file_name,
get_filename=test_data_path_builder,
keep_outputs_dir=keep_outputs_dir,
)
except AssertionError as err:
errmsg = 'Composite file (%s) of %s different than expected, difference:\n' % (base_name, item_label)
errmsg += str(err)
raise AssertionError(errmsg)
def _verify_extra_files_content(extra_files, hda_id, dataset_fetcher, test_data_path_builder, keep_outputs_dir):
files_list = []
for extra_file_dict in extra_files:
extra_file_type = extra_file_dict["type"]
extra_file_name = extra_file_dict["name"]
extra_file_attributes = extra_file_dict["attributes"]
extra_file_value = extra_file_dict["value"]
if extra_file_type == 'file':
files_list.append((extra_file_name, extra_file_value, extra_file_attributes))
elif extra_file_type == 'directory':
for filename in os.listdir(test_data_path_builder(extra_file_value)):
files_list.append((filename, os.path.join(extra_file_value, filename), extra_file_attributes))
else:
raise ValueError('unknown extra_files type: %s' % extra_file_type)
for filename, filepath, attributes in files_list:
_verify_composite_datatype_file_content(filepath, hda_id, base_name=filename, attributes=attributes, dataset_fetcher=dataset_fetcher, test_data_path_builder=test_data_path_builder, keep_outputs_dir=keep_outputs_dir)
def verify_tool(tool_id, galaxy_interactor, resource_parameters={}, register_job_data=None, test_index=0, tool_version=None, quiet=False):
tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version)
tool_test_dict = tool_test_dicts[test_index]
testdef = ToolTestDescription(tool_test_dict)
_handle_def_errors(testdef)
test_history = galaxy_interactor.new_history()
stage_data_in_history(galaxy_interactor, tool_id, testdef.test_data(), test_history)
# Once data is ready, run the tool and check the outputs - record API
# input, job info, tool run exception, as well as exceptions related to
# job output checking and register they with the test plugin so it can
# record structured information.
tool_inputs = None
job_stdio = None
job_output_exceptions = None
tool_execution_exception = None
expected_failure_occurred = False
begin_time = time.time()
try:
try:
tool_response = galaxy_interactor.run_tool(testdef, test_history, resource_parameters=resource_parameters)
data_list, jobs, tool_inputs = tool_response.outputs, tool_response.jobs, tool_response.inputs
data_collection_list = tool_response.output_collections
except RunToolException as e:
tool_inputs = e.inputs
tool_execution_exception = e
if not testdef.expect_failure:
raise e
else:
expected_failure_occurred = True
except Exception as e:
tool_execution_exception = e
raise e
if not expected_failure_occurred:
assert data_list or data_collection_list
try:
job_stdio = _verify_outputs(testdef, test_history, jobs, tool_id, data_list, data_collection_list, galaxy_interactor, quiet=quiet)
except JobOutputsError as e:
job_stdio = e.job_stdio
job_output_exceptions = e.output_exceptions
raise e
except Exception as e:
job_output_exceptions = [e]
raise e
finally:
if register_job_data is not None:
end_time = time.time()
job_data = {
"tool_id": tool_id,
"tool_version": tool_version,
"test_index": test_index,
"time_seconds": end_time - begin_time,
}
if tool_inputs is not None:
job_data["inputs"] = tool_inputs
if job_stdio is not None:
job_data["job"] = job_stdio
status = "success"
if job_output_exceptions:
job_data["output_problems"] = [str(_) for _ in job_output_exceptions]
status = "failure"
if tool_execution_exception:
job_data["execution_problem"] = str(tool_execution_exception)
status = "error"
job_data["status"] = status
register_job_data(job_data)
galaxy_interactor.delete_history(test_history)
def _handle_def_errors(testdef):
# If the test generation had an error, raise
if testdef.error:
if testdef.exception:
if isinstance(testdef.exception, Exception):
raise testdef.exception
else:
raise Exception(testdef.exception)
else:
raise Exception("Test parse failure")
def _verify_outputs(testdef, history, jobs, tool_id, data_list, data_collection_list, galaxy_interactor, quiet=False):
assert len(jobs) == 1, "Test framework logic error, somehow tool test resulted in more than one job."
job = jobs[0]
maxseconds = testdef.maxseconds
if testdef.num_outputs is not None:
expected = testdef.num_outputs
actual = len(data_list)
if expected != actual:
messaage_template = "Incorrect number of outputs - expected %d, found %s."
message = messaage_template % (expected, actual)
raise Exception(message)
found_exceptions = []
def register_exception(e):
if not found_exceptions and not quiet:
# Only print this stuff out once.
for stream in ['stdout', 'stderr']:
if stream in job_stdio:
print(_format_stream(job_stdio[stream], stream=stream, format=True), file=sys.stderr)
found_exceptions.append(e)
if testdef.expect_failure:
if testdef.outputs:
raise Exception("Cannot specify outputs in a test expecting failure.")
# Wait for the job to complete and register expections if the final
# status was not what test was expecting.
job_failed = False
try:
galaxy_interactor.wait_for_job(job['id'], history, maxseconds)
except Exception as e:
job_failed = True
if not testdef.expect_failure:
found_exceptions.append(e)
job_stdio = galaxy_interactor.get_job_stdio(job['id'])
if not job_failed and testdef.expect_failure:
error = AssertionError("Expected job to fail but Galaxy indicated the job successfully completed.")
register_exception(error)
expect_exit_code = testdef.expect_exit_code
if expect_exit_code is not None:
exit_code = job_stdio["exit_code"]
if str(expect_exit_code) != str(exit_code):
error = AssertionError("Expected job to complete with exit code %s, found %s" % (expect_exit_code, exit_code))
register_exception(error)
for output_index, output_dict in enumerate(testdef.outputs):
# Get the correct hid
name = output_dict["name"]
outfile = output_dict["value"]
attributes = output_dict["attributes"]
output_testdef = Bunch(name=name, outfile=outfile, attributes=attributes)
try:
output_data = data_list[name]
except (TypeError, KeyError):
# Legacy - fall back on ordered data list access if data_list is
# just a list (case with twill variant or if output changes its
# name).
if hasattr(data_list, "values"):
output_data = list(data_list.values())[output_index]
else:
output_data = data_list[len(data_list) - len(testdef.outputs) + output_index]
assert output_data is not None
try:
galaxy_interactor.verify_output(history, jobs, output_data, output_testdef=output_testdef, tool_id=tool_id, maxseconds=maxseconds)
except Exception as e:
register_exception(e)
other_checks = {
"command_line": "Command produced by the job",
"stdout": "Standard output of the job",
"stderr": "Standard error of the job",
}
for what, description in other_checks.items():
if getattr(testdef, what, None) is not None:
try:
data = job_stdio[what]
verify_assertions(data, getattr(testdef, what))
except AssertionError as err:
errmsg = '%s different than expected\n' % description
errmsg += str(err)
register_exception(AssertionError(errmsg))
for output_collection_def in testdef.output_collections:
try:
name = output_collection_def.name
# TODO: data_collection_list is clearly a bad name for dictionary.
if name not in data_collection_list:
template = "Failed to find output [%s], tool outputs include [%s]"
message = template % (name, ",".join(data_collection_list.keys()))
raise AssertionError(message)
# Data collection returned from submission, elements may have been populated after
# the job completed so re-hit the API for more information.
data_collection_returned = data_collection_list[name]
data_collection = galaxy_interactor._get("dataset_collections/%s" % data_collection_returned["id"], data={"instance_type": "history"}).json()
def get_element(elements, id):
for element in elements:
if element["element_identifier"] == id:
return element
return False
expected_collection_type = output_collection_def.collection_type
if expected_collection_type:
collection_type = data_collection["collection_type"]
if expected_collection_type != collection_type:
template = "Expected output collection [%s] to be of type [%s], was of type [%s]."
message = template % (name, expected_collection_type, collection_type)
raise AssertionError(message)
expected_element_count = output_collection_def.count
if expected_element_count:
actual_element_count = len(data_collection["elements"])
if expected_element_count != actual_element_count:
template = "Expected output collection [%s] to have %s elements, but it had %s."
message = template % (name, expected_element_count, actual_element_count)
raise AssertionError(message)
def verify_elements(element_objects, element_tests):
for element_identifier, (element_outfile, element_attrib) in element_tests.items():
element = get_element(element_objects, element_identifier)
if not element:
template = "Failed to find identifier [%s] for testing, tool generated collection elements [%s]"
message = template % (element_identifier, element_objects)
raise AssertionError(message)
element_type = element["element_type"]
if element_type != "dataset_collection":
hda = element["object"]
galaxy_interactor.verify_output_dataset(
history,
hda_id=hda["id"],
outfile=element_outfile,
attributes=element_attrib,
tool_id=tool_id
)
if element_type == "dataset_collection":
elements = element["object"]["elements"]
verify_elements(elements, element_attrib.get("elements", {}))
verify_elements(data_collection["elements"], output_collection_def.element_tests)
except Exception as e:
register_exception(e)
if found_exceptions:
raise JobOutputsError(found_exceptions, job_stdio)
else:
return job_stdio
def _format_stream(output, stream, format):
output = output or ''
if format:
msg = "---------------------- >> begin tool %s << -----------------------\n" % stream
msg += output + "\n"
msg += "----------------------- >> end tool %s << ------------------------\n" % stream
else:
msg = output
return msg
class JobOutputsError(AssertionError):
def __init__(self, output_exceptions, job_stdio):
big_message = "\n".join(map(str, output_exceptions))
super(JobOutputsError, self).__init__(big_message)
self.job_stdio = job_stdio
self.output_exceptions = output_exceptions
class ToolTestDescription(object):
"""
Encapsulates information about a tool test, and allows creation of a
dynamic TestCase class (the unittest framework is very class oriented,
doing dynamic tests in this way allows better integration)
"""
def __init__(self, processed_test_dict):
test_index = processed_test_dict["test_index"]
name = processed_test_dict.get('name', 'Test-%d' % (test_index + 1))
maxseconds = processed_test_dict.get('maxseconds', DEFAULT_MAX_SECS)
if maxseconds is not None:
maxseconds = int(maxseconds)
self.test_index = test_index
self.tool_id = processed_test_dict["tool_id"]
self.name = name
self.maxseconds = maxseconds
self.required_files = processed_test_dict.get("required_files", [])
inputs = processed_test_dict.get("inputs", {})
loaded_inputs = {}
for key, value in inputs.items():
if isinstance(value, dict) and value.get("model_class"):
loaded_inputs[key] = TestCollectionDef.from_dict(value)
else:
loaded_inputs[key] = value
self.inputs = loaded_inputs
self.outputs = processed_test_dict.get("outputs", [])
self.num_outputs = processed_test_dict.get("num_outputs", 0)
self.error = processed_test_dict.get("error", False)
self.exception = processed_test_dict.get("exception", None)
self.output_collections = map(TestCollectionOutputDef.from_dict, processed_test_dict.get("output_collections", []))
self.command_line = processed_test_dict.get("command", None)
self.stdout = processed_test_dict.get("stdout", None)
self.stderr = processed_test_dict.get("stderr", None)
self.expect_exit_code = processed_test_dict.get("expect_exit_code", None)
self.expect_failure = processed_test_dict.get("expect_failure", False)
def test_data(self):
"""
Iterator over metadata representing the required files for upload.
"""
return test_data_iter(self.required_files)
def to_dict(self):
inputs_dict = {}
for key, value in self.inputs.items():
if hasattr(value, "to_dict"):
inputs_dict[key] = value.to_dict()
else:
inputs_dict[key] = value
return {
"inputs": inputs_dict,
"outputs": self.outputs,
"output_collections": map(lambda o: o.to_dict(), self.output_collections),
"num_outputs": self.num_outputs,
"command_line": self.command_line,
"stdout": self.stdout,
"stderr": self.stderr,
"expect_exit_code": self.expect_exit_code,
"expect_failure": self.expect_failure,
"name": self.name,
"test_index": self.test_index,
"tool_id": self.tool_id,
"required_files": self.required_files,
"error": self.error,
"exception": self.exception,
}
@nottest
def test_data_iter(required_files):
for fname, extra in required_files:
data_dict = dict(
fname=fname,
metadata=extra.get('metadata', {}),
composite_data=extra.get('composite_data', []),
ftype=extra.get('ftype', DEFAULT_FTYPE),
dbkey=extra.get('dbkey', DEFAULT_DBKEY),
)
edit_attributes = extra.get('edit_attributes', [])
# currently only renaming is supported
for edit_att in edit_attributes:
if edit_att.get('type', None) == 'name':
new_name = edit_att.get('value', None)
assert new_name, 'You must supply the new dataset name as the value tag of the edit_attributes tag'
data_dict['name'] = new_name
else:
raise Exception('edit_attributes type (%s) is unimplemented' % edit_att.get('type', None))
yield data_dict
lib/galaxy/tools/verify/script.py
+107
View File
@@ -0,0 +1,107 @@
#!/usr/bin/env python
from __future__ import print_function
import argparse
import json
import sys
from galaxy.tools.verify.interactor import GalaxyInteractorApi, verify_tool
DESCRIPTION = """Script to quickly run a tool test against a running Galaxy instance."""
ALL_TESTS = "*all_tests*"
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
args = _arg_parser().parse_args(argv)
galaxy_interactor_kwds = {
"galaxy_url": args.galaxy_url,
"master_api_key": args.admin_key,
"api_key": args.key,
"keep_outputs_dir": args.output,
}
tool_id = args.tool_id
tool_version = args.tool_version
galaxy_interactor = GalaxyInteractorApi(**galaxy_interactor_kwds)
raw_test_index = args.test_index
if raw_test_index == ALL_TESTS:
tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version)
test_indices = list(range(len(tool_test_dicts)))
else:
test_indices = [int(raw_test_index)]
test_results = []
if args.append:
with open(args.output_json, "r") as f:
previous_results = json.load(f)
test_results = previous_results["tests"]
exceptions = []
verbose = args.verbose
for test_index in test_indices:
if tool_version:
tool_id_and_version = "%s/%s" % (tool_id, tool_version)
else:
tool_id_and_version = tool_id
test_identifier = "tool %s test # %d" % (tool_id_and_version, test_index)
def register(job_data):
test_results.append({
'id': tool_id + "-" + str(test_index),
'has_data': True,
'data': job_data,
})
try:
verify_tool(
tool_id, galaxy_interactor, test_index=test_index, tool_version=tool_version,
register_job_data=register, quiet=not verbose
)
if verbose:
print("%s passed" % test_identifier)
except Exception as e:
if verbose:
print("%s failed, %s" % (test_identifier, e))
exceptions.append(e)
report_obj = {
'version': '0.1',
'tests': test_results,
}
output_json = args.output_json
if output_json:
if args.output_json == "-":
assert not args.append
print(json.dumps(report_obj))
else:
with open(args.output_json, "w") as f:
json.dump(report_obj, f)
if exceptions:
raise exceptions[0]
def _arg_parser():
parser = argparse.ArgumentParser(description=DESCRIPTION)
parser.add_argument('-u', '--galaxy-url', default="http://localhost:8080", help='Galaxy URL')
parser.add_argument('-k', '--key', default=None, help='Galaxy User API Key')
parser.add_argument('-a', '--admin-key', default=None, help='Galaxy Admin API Key')
parser.add_argument('-t', '--tool-id', default=None, help='Tool ID')
parser.add_argument('--tool-version', default=None, help='Tool Version')
parser.add_argument('-i', '--test-index', default=ALL_TESTS, help='Tool Test Index (starting at 0) - by default all tests will run.')
parser.add_argument('-o', '--output', default=None, help='directory to dump outputs to')
parser.add_argument('--append', default=False, action="store_true", help="Extend a test record json (created with --output-json) with additional tests.")
parser.add_argument('-j', '--output-json', default=None, help='output metadata json')
parser.add_argument('--verbose', default=False, action="store_true", help="Verbose logging.")
return parser
if __name__ == "__main__":
main()
lib/galaxy/tools/verify/test_data.py
+15 -20
View File
@@ -8,6 +8,7 @@ from string import Template
from galaxy.util import (
asbool,
in_directory,
smart_str
)
@@ -27,32 +28,25 @@ LIST_SEP = re.compile("\s*,\s*")
class TestDataResolver(object):
def __init__(self, env_var='GALAXY_TEST_FILE_DIR', environ=os.environ):
file_dirs = environ.get(env_var, None)
def __init__(self, file_dirs=None, env_var='GALAXY_TEST_FILE_DIR', environ=os.environ):
if file_dirs is None:
file_dirs = environ.get(env_var, None)
if file_dirs is None:
file_dirs = "test-data,https://github.com/galaxyproject/galaxy-test-data.git"
if file_dirs:
self.resolvers = [build_resolver(u, environ) for u in LIST_SEP.split(file_dirs)]
else:
self.resolvers = []
def get_filename(self, name):
if not self.resolvers:
filename = None
else:
resolver = self.resolvers[0]
filename = None
for resolver in self.resolvers or []:
if not resolver.exists(name):
continue
filename = resolver.path(name)
if not resolver.exists(filename):
for resolver in self.resolvers[1:]:
if resolver.exists(name):
filename = resolver.path(name)
else:
# For backward compat. returning first path if none
# exist - though I don't know if this function is ever
# actually used in a context where one should return
# a file even if it doesn't exist (e.g. a prefix or
# or something) - I am pretty sure it is not used in
# such a fashion in the context of tool tests.
filename = resolver.path(name)
return os.path.abspath(filename)
if filename:
return os.path.abspath(filename)
def build_resolver(uri, environ):
@@ -68,7 +62,8 @@ class FileDataResolver(object):
self.file_dir = file_dir
def exists(self, filename):
return os.path.exists(self.path(filename))
path = os.path.abspath(self.path(filename))
return os.path.exists(path) and in_directory(path, self.file_dir)
def path(self, filename):
return os.path.join(self.file_dir, filename)
lib/galaxy/util/compression_utils.py
+125
View File
@@ -1,5 +1,10 @@
from __future__ import absolute_import
import gzip
import io
import logging
import os
import tarfile
import zipfile
from .checkers import (
@@ -8,6 +13,8 @@ from .checkers import (
is_gzip
)
log = logging.getLogger(__name__)
def get_fileobj(filename, mode="r", compressed_formats=None):
"""
@@ -45,3 +52,121 @@ def get_fileobj(filename, mode="r", compressed_formats=None):
return io.TextIOWrapper(fh, encoding='utf-8')
else:
return fh
class CompressedFile(object):
def __init__(self, file_path, mode='r'):
if tarfile.is_tarfile(file_path):
self.file_type = 'tar'
elif zipfile.is_zipfile(file_path) and not file_path.endswith('.jar'):
self.file_type = 'zip'
self.file_name = os.path.splitext(os.path.basename(file_path))[0]
if self.file_name.endswith('.tar'):
self.file_name = os.path.splitext(self.file_name)[0]
self.type = self.file_type
method = 'open_%s' % self.file_type
if hasattr(self, method):
self.archive = getattr(self, method)(file_path, mode)
else:
raise NameError('File type %s specified, no open method found.' % self.file_type)
def extract(self, path):
'''Determine the path to which the archive should be extracted.'''
contents = self.getmembers()
extraction_path = path
common_prefix = ''
if len(contents) == 1:
# The archive contains a single file, return the extraction path.
if self.isfile(contents[0]):
extraction_path = os.path.join(path, self.file_name)
if not os.path.exists(extraction_path):
os.makedirs(extraction_path)
self.archive.extractall(extraction_path)
else:
# Get the common prefix for all the files in the archive. If the common prefix ends with a slash,
# or self.isdir() returns True, the archive contains a single directory with the desired contents.
# Otherwise, it contains multiple files and/or directories at the root of the archive.
common_prefix = os.path.commonprefix([self.getname(item) for item in contents])
if len(common_prefix) >= 1 and not common_prefix.endswith(os.sep) and self.isdir(self.getmember(common_prefix)):
common_prefix += os.sep
if not common_prefix.endswith(os.sep):
common_prefix = ''
extraction_path = os.path.join(path, self.file_name)
if not os.path.exists(extraction_path):
os.makedirs(extraction_path)
self.archive.extractall(extraction_path)
# Since .zip files store unix permissions separately, we need to iterate through the zip file
# and set permissions on extracted members.
if self.file_type == 'zip':
for zipped_file in contents:
filename = self.getname(zipped_file)
absolute_filepath = os.path.join(extraction_path, filename)
external_attributes = self.archive.getinfo(filename).external_attr
# The 2 least significant bytes are irrelevant, the next two contain unix permissions.
unix_permissions = external_attributes >> 16
if unix_permissions != 0:
if os.path.exists(absolute_filepath):
os.chmod(absolute_filepath, unix_permissions)
else:
log.warning("Unable to change permission on extracted file '%s' as it does not exist" % absolute_filepath)
return os.path.abspath(os.path.join(extraction_path, common_prefix))
def getmembers_tar(self):
return self.archive.getmembers()
def getmembers_zip(self):
return self.archive.infolist()
def getname_tar(self, item):
return item.name
def getname_zip(self, item):
return item.filename
def getmember(self, name):
for member in self.getmembers():
if self.getname(member) == name:
return member
def getmembers(self):
return getattr(self, 'getmembers_%s' % self.type)()
def getname(self, member):
return getattr(self, 'getname_%s' % self.type)(member)
def isdir(self, member):
return getattr(self, 'isdir_%s' % self.type)(member)
def isdir_tar(self, member):
return member.isdir()
def isdir_zip(self, member):
if member.filename.endswith(os.sep):
return True
return False
def isfile(self, member):
if not self.isdir(member):
return True
return False
def open_tar(self, filepath, mode):
return tarfile.open(filepath, mode, errorlevel=0)
def open_zip(self, filepath, mode):
return zipfile.ZipFile(filepath, mode)
def zipfile_ok(self, path_to_archive):
"""
This function is a bit pedantic and not functionally necessary. It checks whether there is
no file pointing outside of the extraction, because ZipFile.extractall() has some potential
security holes. See python zipfile documentation for more details.
"""
basename = os.path.realpath(os.path.dirname(path_to_archive))
zip_archive = zipfile.ZipFile(path_to_archive)
for member in zip_archive.namelist():
member_path = os.path.realpath(os.path.join(basename, member))
if not member_path.startswith(basename):
return False
return True
lib/galaxy/util/path/__init__.py
+11 -2
View File
@@ -6,7 +6,10 @@ import errno
import imp
import logging
from functools import partial
from grp import getgrgid
try:
from grp import getgrgid
except ImportError:
getgrgid = None
from itertools import starmap
from operator import getitem
from os import (
@@ -28,7 +31,10 @@ from os.path import (
relpath,
sep as separator,
)
from pwd import getpwuid
try:
from pwd import getpwuid
except ImportError:
getpwuid = None
from six import iteritems, string_types
from six.moves import filter, map, zip
@@ -165,6 +171,9 @@ def __path_permission_for_user(path, username):
:type username: string
:param username: a username matching the systems username
"""
if getpwuid is None:
raise NotImplementedError("This functionality is not implemented for Windows.")
group_id_of_file = stat(path).st_gid
file_owner = getpwuid(stat(path).st_uid)
group_members = getgrgid(group_id_of_file).gr_mem
lib/galaxy/util/sanitize_html.py
+39 -421
View File
@@ -1,427 +1,45 @@
"""
HTML Sanitizer (ripped from feedparser)
HTML Sanitizer (lists of acceptable_* ripped from feedparser)
"""
import bleach
import re
import sgmllib
_acceptable_elements = ['a', 'abbr', 'acronym', 'address', 'area', 'article',
'aside', 'audio', 'b', 'big', 'blockquote', 'br', 'button', 'canvas',
'caption', 'center', 'cite', 'code', 'col', 'colgroup', 'command',
'datagrid', 'datalist', 'dd', 'del', 'details', 'dfn', 'dialog', 'dir',
'div', 'dl', 'dt', 'em', 'event-source', 'fieldset', 'figure',
'footer', 'font', 'form', 'header', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
'hr', 'i', 'img', 'input', 'ins', 'keygen', 'kbd', 'label', 'legend',
'li', 'm', 'map', 'menu', 'meter', 'multicol', 'nav', 'nextid', 'ol',
'output', 'optgroup', 'option', 'p', 'pre', 'progress', 'q', 's',
'samp', 'section', 'select', 'small', 'sound', 'source', 'spacer',
'span', 'strike', 'strong', 'sub', 'sup', 'table', 'tbody', 'td',
'textarea', 'time', 'tfoot', 'th', 'thead', 'tr', 'tt', 'u', 'ul',
'var', 'video', 'noscript']
from six import unichr
from six.moves.html_entities import name2codepoint
from galaxy.util import unicodify
_cp1252 = {
unichr(128): unichr(8364), # euro sign
unichr(130): unichr(8218), # single low-9 quotation mark
unichr(131): unichr(402), # latin small letter f with hook
unichr(132): unichr(8222), # double low-9 quotation mark
unichr(133): unichr(8230), # horizontal ellipsis
unichr(134): unichr(8224), # dagger
unichr(135): unichr(8225), # double dagger
unichr(136): unichr(710), # modifier letter circumflex accent
unichr(137): unichr(8240), # per mille sign
unichr(138): unichr(352), # latin capital letter s with caron
unichr(139): unichr(8249), # single left-pointing angle quotation mark
unichr(140): unichr(338), # latin capital ligature oe
unichr(142): unichr(381), # latin capital letter z with caron
unichr(145): unichr(8216), # left single quotation mark
unichr(146): unichr(8217), # right single quotation mark
unichr(147): unichr(8220), # left double quotation mark
unichr(148): unichr(8221), # right double quotation mark
unichr(149): unichr(8226), # bullet
unichr(150): unichr(8211), # en dash
unichr(151): unichr(8212), # em dash
unichr(152): unichr(732), # small tilde
unichr(153): unichr(8482), # trade mark sign
unichr(154): unichr(353), # latin small letter s with caron
unichr(155): unichr(8250), # single right-pointing angle quotation mark
unichr(156): unichr(339), # latin small ligature oe
unichr(158): unichr(382), # latin small letter z with caron
unichr(159): unichr(376)} # latin capital letter y with diaeresis
_acceptable_attributes = ['abbr', 'accept', 'accept-charset', 'accesskey',
'action', 'align', 'alt', 'autocomplete', 'autofocus', 'axis',
'background', 'balance', 'bgcolor', 'bgproperties', 'border',
'bordercolor', 'bordercolordark', 'bordercolorlight', 'bottompadding',
'cellpadding', 'cellspacing', 'ch', 'challenge', 'char', 'charoff',
'choff', 'charset', 'checked', 'cite', 'class', 'clear', 'color',
'cols', 'colspan', 'compact', 'contenteditable', 'controls', 'coords',
'data', 'datafld', 'datapagesize', 'datasrc', 'datetime', 'default',
'delay', 'dir', 'disabled', 'draggable', 'dynsrc', 'enctype', 'end',
'face', 'for', 'form', 'frame', 'galleryimg', 'gutter', 'headers',
'height', 'hidefocus', 'hidden', 'high', 'href', 'hreflang', 'hspace',
'icon', 'id', 'inputmode', 'ismap', 'keytype', 'label', 'leftspacing',
'lang', 'list', 'longdesc', 'loop', 'loopcount', 'loopend',
'loopstart', 'low', 'lowsrc', 'max', 'maxlength', 'media', 'method',
'min', 'multiple', 'name', 'nohref', 'noshade', 'nowrap', 'open',
'optimum', 'pattern', 'ping', 'point-size', 'prompt', 'pqg',
'radiogroup', 'readonly', 'rel', 'repeat-max', 'repeat-min', 'replace',
'required', 'rev', 'rightspacing', 'rows', 'rowspan', 'rules', 'scope',
'selected', 'shape', 'size', 'span', 'src', 'start', 'step', 'summary',
'suppress', 'tabindex', 'target', 'template', 'title', 'toppadding',
'type', 'unselectable', 'usemap', 'urn', 'valign', 'value', 'variable',
'volume', 'vspace', 'vrml', 'width', 'wrap', 'xml:lang']
class _BaseHTMLProcessor(sgmllib.SGMLParser):
special = re.compile('''[<>'"]''')
bare_ampersand = re.compile("&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)")
elements_no_end_tag = ['area', 'base', 'basefont', 'br', 'col', 'frame', 'hr',
'img', 'input', 'isindex', 'link', 'meta', 'param']
def __init__(self, encoding, type):
self.encoding = encoding
self.type = type
# if _debug: sys.stderr.write('entering BaseHTMLProcessor, encoding=%s\n' % self.encoding)
sgmllib.SGMLParser.__init__(self)
def reset(self):
self.pieces = []
sgmllib.SGMLParser.reset(self)
def _shorttag_replace(self, match):
tag = match.group(1)
if tag in self.elements_no_end_tag:
return '<' + tag + ' />'
else:
return '<' + tag + '></' + tag + '>'
def parse_starttag(self, i):
j = sgmllib.SGMLParser.parse_starttag(self, i)
if self.type == 'application/xhtml+xml':
if j > 2 and self.rawdata[j - 2:j] == '/>':
self.unknown_endtag(self.lasttag)
return j
def feed(self, data):
data = re.compile(r'<!((?!DOCTYPE|--|\[))', re.IGNORECASE).sub(r'&lt;!\1', data)
data = re.sub(r'<([^<>\s]+?)\s*/>', self._shorttag_replace, data)
data = data.replace('&#39;', "'")
data = data.replace('&#34;', '"')
sgmllib.SGMLParser.feed(self, data)
sgmllib.SGMLParser.close(self)
def normalize_attrs(self, attrs):
if not attrs:
return attrs
# utility method to be called by descendants
attrs = dict([(k.lower(), v) for k, v in attrs]).items()
attrs = [(k, k in ('rel', 'type') and v.lower() or v) for k, v in attrs]
attrs.sort()
return attrs
def unknown_starttag(self, tag, attrs):
# called for each start tag
# attrs is a list of (attr, value) tuples
# e.g. for <pre class='screen'>, tag='pre', attrs=[('class', 'screen')]
uattrs = []
strattrs = ''
if attrs:
for key, value in attrs:
value = value.replace('>', '&gt;').replace('<', '&lt;').replace('"', '&quot;')
value = self.bare_ampersand.sub("&amp;", value)
uattrs.append((key, value))
strattrs = ''.join([' %s="%s"' % (k, v) for k, v in uattrs])
if tag in self.elements_no_end_tag:
self.pieces.append('<%s%s />' % (tag, strattrs))
else:
self.pieces.append('<%s%s>' % (tag, strattrs))
def unknown_endtag(self, tag):
# called for each end tag, e.g. for </pre>, tag will be 'pre'
# Reconstruct the original end tag.
if tag not in self.elements_no_end_tag:
self.pieces.append("</%(tag)s>" % locals())
def handle_charref(self, ref):
# called for each character reference, e.g. for '&#160;', ref will be '160'
# Reconstruct the original character reference.
if ref.startswith('x'):
value = unichr(int(ref[1:], 16))
else:
value = unichr(int(ref))
if value in _cp1252.keys():
self.pieces.append('&#%s;' % hex(ord(_cp1252[value]))[1:])
else:
self.pieces.append('&#%(ref)s;' % locals())
def handle_entityref(self, ref):
# called for each entity reference, e.g. for '&copy;', ref will be 'copy'
# Reconstruct the original entity reference.
if ref in name2codepoint:
self.pieces.append('&%(ref)s;' % locals())
else:
self.pieces.append('&amp;%(ref)s' % locals())
def handle_data(self, text):
# called for each block of plain text, i.e. outside of any tag and
# not containing any character or entity references
# Store the original text verbatim.
self.pieces.append(text)
def handle_comment(self, text):
# called for each HTML comment, e.g. <!-- insert Javascript code here -->
# Reconstruct the original comment.
self.pieces.append('<!--%(text)s-->' % locals())
def handle_pi(self, text):
# called for each processing instruction, e.g. <?instruction>
# Reconstruct original processing instruction.
self.pieces.append('<?%(text)s>' % locals())
def handle_decl(self, text):
# called for the DOCTYPE, if present, e.g.
# <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
# "http://www.w3.org/TR/html4/loose.dtd">
# Reconstruct original DOCTYPE
self.pieces.append('<!%(text)s>' % locals())
_new_declname_match = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9:]*\s*').match
def _scan_name(self, i, declstartpos):
rawdata = self.rawdata
n = len(rawdata)
if i == n:
return None, -1
m = self._new_declname_match(rawdata, i)
if m:
s = m.group()
name = s.strip()
if (i + len(s)) == n:
return None, -1 # end of buffer
return name.lower(), m.end()
else:
self.handle_data(rawdata)
# self.updatepos(declstartpos, i)
return None, -1
def convert_charref(self, name):
return '&#%s;' % name
def convert_entityref(self, name):
return '&%s;' % name
def output(self):
'''Return processed HTML as a single string'''
return ''.join(self.pieces)
class _HTMLSanitizer(_BaseHTMLProcessor):
acceptable_elements = ['a', 'abbr', 'acronym', 'address', 'area', 'article',
'aside', 'audio', 'b', 'big', 'blockquote', 'br', 'button', 'canvas',
'caption', 'center', 'cite', 'code', 'col', 'colgroup', 'command',
'datagrid', 'datalist', 'dd', 'del', 'details', 'dfn', 'dialog', 'dir',
'div', 'dl', 'dt', 'em', 'event-source', 'fieldset', 'figure', 'footer',
'font', 'form', 'header', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i',
'img', 'input', 'ins', 'keygen', 'kbd', 'label', 'legend', 'li', 'm', 'map',
'menu', 'meter', 'multicol', 'nav', 'nextid', 'ol', 'output', 'optgroup',
'option', 'p', 'pre', 'progress', 'q', 's', 'samp', 'section', 'select',
'small', 'sound', 'source', 'spacer', 'span', 'strike', 'strong', 'sub',
'sup', 'table', 'tbody', 'td', 'textarea', 'time', 'tfoot', 'th', 'thead',
'tr', 'tt', 'u', 'ul', 'var', 'video', 'noscript']
acceptable_attributes = ['abbr', 'accept', 'accept-charset', 'accesskey',
'action', 'align', 'alt', 'autocomplete', 'autofocus', 'axis',
'background', 'balance', 'bgcolor', 'bgproperties', 'border',
'bordercolor', 'bordercolordark', 'bordercolorlight', 'bottompadding',
'cellpadding', 'cellspacing', 'ch', 'challenge', 'char', 'charoff',
'choff', 'charset', 'checked', 'cite', 'class', 'clear', 'color', 'cols',
'colspan', 'compact', 'contenteditable', 'controls', 'coords', 'data',
'datafld', 'datapagesize', 'datasrc', 'datetime', 'default', 'delay',
'dir', 'disabled', 'draggable', 'dynsrc', 'enctype', 'end', 'face', 'for',
'form', 'frame', 'galleryimg', 'gutter', 'headers', 'height', 'hidefocus',
'hidden', 'high', 'href', 'hreflang', 'hspace', 'icon', 'id', 'inputmode',
'ismap', 'keytype', 'label', 'leftspacing', 'lang', 'list', 'longdesc',
'loop', 'loopcount', 'loopend', 'loopstart', 'low', 'lowsrc', 'max',
'maxlength', 'media', 'method', 'min', 'multiple', 'name', 'nohref',
'noshade', 'nowrap', 'open', 'optimum', 'pattern', 'ping', 'point-size',
'prompt', 'pqg', 'radiogroup', 'readonly', 'rel', 'repeat-max',
'repeat-min', 'replace', 'required', 'rev', 'rightspacing', 'rows',
'rowspan', 'rules', 'scope', 'selected', 'shape', 'size', 'span', 'src',
'start', 'step', 'summary', 'suppress', 'tabindex', 'target', 'template',
'title', 'toppadding', 'type', 'unselectable', 'usemap', 'urn', 'valign',
'value', 'variable', 'volume', 'vspace', 'vrml', 'width', 'wrap',
'xml:lang']
unacceptable_elements_with_end_tag = ['script', 'applet', 'style']
acceptable_css_properties = ['azimuth', 'background-color',
'border-bottom-color', 'border-collapse', 'border-color',
'border-left-color', 'border-right-color', 'border-top-color', 'clear',
'color', 'cursor', 'direction', 'display', 'elevation', 'float', 'font',
'font-family', 'font-size', 'font-style', 'font-variant', 'font-weight',
'height', 'letter-spacing', 'line-height', 'overflow', 'pause',
'pause-after', 'pause-before', 'pitch', 'pitch-range', 'richness',
'speak', 'speak-header', 'speak-numeral', 'speak-punctuation',
'speech-rate', 'stress', 'text-align', 'text-decoration', 'text-indent',
'unicode-bidi', 'vertical-align', 'voice-family', 'volume',
'white-space', 'width']
# survey of common keywords found in feeds
acceptable_css_keywords = ['auto', 'aqua', 'black', 'block', 'blue',
'bold', 'both', 'bottom', 'brown', 'center', 'collapse', 'dashed',
'dotted', 'fuchsia', 'gray', 'green', '!important', 'italic', 'left',
'lime', 'maroon', 'medium', 'none', 'navy', 'normal', 'nowrap', 'olive',
'pointer', 'purple', 'red', 'right', 'solid', 'silver', 'teal', 'top',
'transparent', 'underline', 'white', 'yellow']
valid_css_values = re.compile('^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|' +
'\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$')
mathml_elements = ['annotation', 'annotation-xml', 'maction', 'math',
'merror', 'mfenced', 'mfrac', 'mi', 'mmultiscripts', 'mn', 'mo', 'mover', 'mpadded',
'mphantom', 'mprescripts', 'mroot', 'mrow', 'mspace', 'msqrt', 'mstyle',
'msub', 'msubsup', 'msup', 'mtable', 'mtd', 'mtext', 'mtr', 'munder',
'munderover', 'none', 'semantics']
mathml_attributes = ['actiontype', 'align', 'columnalign', 'columnalign',
'columnalign', 'close', 'columnlines', 'columnspacing', 'columnspan', 'depth',
'display', 'displaystyle', 'encoding', 'equalcolumns', 'equalrows',
'fence', 'fontstyle', 'fontweight', 'frame', 'height', 'linethickness',
'lspace', 'mathbackground', 'mathcolor', 'mathvariant', 'mathvariant',
'maxsize', 'minsize', 'open', 'other', 'rowalign', 'rowalign', 'rowalign',
'rowlines', 'rowspacing', 'rowspan', 'rspace', 'scriptlevel', 'selection',
'separator', 'separators', 'stretchy', 'width', 'width', 'xlink:href',
'xlink:show', 'xlink:type', 'xmlns', 'xmlns:xlink']
# svgtiny - foreignObject + linearGradient + radialGradient + stop
svg_elements = ['a', 'animate', 'animateColor', 'animateMotion',
'animateTransform', 'circle', 'defs', 'desc', 'ellipse', 'foreignObject',
'font-face', 'font-face-name', 'font-face-src', 'g', 'glyph', 'hkern',
'linearGradient', 'line', 'marker', 'metadata', 'missing-glyph', 'mpath',
'path', 'polygon', 'polyline', 'radialGradient', 'rect', 'set', 'stop',
'svg', 'switch', 'text', 'title', 'tspan', 'use']
# svgtiny + class + opacity + offset + xmlns + xmlns:xlink
svg_attributes = ['accent-height', 'accumulate', 'additive', 'alphabetic',
'arabic-form', 'ascent', 'attributeName', 'attributeType',
'baseProfile', 'bbox', 'begin', 'by', 'calcMode', 'cap-height',
'class', 'color', 'color-rendering', 'content', 'cx', 'cy', 'd', 'dx',
'dy', 'descent', 'display', 'dur', 'end', 'fill', 'fill-opacity',
'fill-rule', 'font-family', 'font-size', 'font-stretch', 'font-style',
'font-variant', 'font-weight', 'from', 'fx', 'fy', 'g1', 'g2',
'glyph-name', 'gradientUnits', 'hanging', 'height', 'horiz-adv-x',
'horiz-origin-x', 'id', 'ideographic', 'k', 'keyPoints', 'keySplines',
'keyTimes', 'lang', 'mathematical', 'marker-end', 'marker-mid',
'marker-start', 'markerHeight', 'markerUnits', 'markerWidth', 'max',
'min', 'name', 'offset', 'opacity', 'orient', 'origin',
'overline-position', 'overline-thickness', 'panose-1', 'path',
'pathLength', 'points', 'preserveAspectRatio', 'r', 'refX', 'refY',
'repeatCount', 'repeatDur', 'requiredExtensions', 'requiredFeatures',
'restart', 'rotate', 'rx', 'ry', 'slope', 'stemh', 'stemv',
'stop-color', 'stop-opacity', 'strikethrough-position',
'strikethrough-thickness', 'stroke', 'stroke-dasharray',
'stroke-dashoffset', 'stroke-linecap', 'stroke-linejoin',
'stroke-miterlimit', 'stroke-opacity', 'stroke-width', 'systemLanguage',
'target', 'text-anchor', 'to', 'transform', 'type', 'u1', 'u2',
'underline-position', 'underline-thickness', 'unicode', 'unicode-range',
'units-per-em', 'values', 'version', 'viewBox', 'visibility', 'width',
'widths', 'x', 'x-height', 'x1', 'x2', 'xlink:actuate', 'xlink:arcrole',
'xlink:href', 'xlink:role', 'xlink:show', 'xlink:title', 'xlink:type',
'xml:base', 'xml:lang', 'xml:space', 'xmlns', 'xmlns:xlink', 'y', 'y1',
'y2', 'zoomAndPan']
svg_attr_map = None
svg_elem_map = None
acceptable_svg_properties = ['fill', 'fill-opacity', 'fill-rule',
'stroke', 'stroke-width', 'stroke-linecap', 'stroke-linejoin',
'stroke-opacity']
def reset(self):
_BaseHTMLProcessor.reset(self)
self.unacceptablestack = 0
self.mathmlOK = 0
self.svgOK = 0
def unknown_starttag(self, tag, attrs):
acceptable_attributes = self.acceptable_attributes
keymap = {}
if tag not in self.acceptable_elements or self.svgOK:
if tag in self.unacceptable_elements_with_end_tag:
self.unacceptablestack += 1
# not otherwise acceptable, perhaps it is MathML or SVG?
if tag == 'math' and ('xmlns', 'http://www.w3.org/1998/Math/MathML') in attrs:
self.mathmlOK += 1
if tag == 'svg' and ('xmlns', 'http://www.w3.org/2000/svg') in attrs:
self.svgOK += 1
# chose acceptable attributes based on tag class, else bail
if self.mathmlOK and tag in self.mathml_elements:
acceptable_attributes = self.mathml_attributes
elif self.svgOK and tag in self.svg_elements:
# for most vocabularies, lowercasing is a good idea. Many
# svg elements, however, are camel case
if not self.svg_attr_map:
lower = [attr.lower() for attr in self.svg_attributes]
mix = [a for a in self.svg_attributes if a not in lower]
self.svg_attributes = lower
self.svg_attr_map = dict([(a.lower(), a) for a in mix])
lower = [attr.lower() for attr in self.svg_elements]
mix = [a for a in self.svg_elements if a not in lower]
self.svg_elements = lower
self.svg_elem_map = dict([(a.lower(), a) for a in mix])
acceptable_attributes = self.svg_attributes
tag = self.svg_elem_map.get(tag, tag)
keymap = self.svg_attr_map
elif tag not in self.acceptable_elements:
return
# declare xlink namespace, if needed
if self.mathmlOK or self.svgOK:
if any(map(lambda n, v: n.startswith('xlink:'), attrs)):
if not ('xmlns:xlink', 'http://www.w3.org/1999/xlink') in attrs:
attrs.append(('xmlns:xlink', 'http://www.w3.org/1999/xlink'))
clean_attrs = []
for key, value in self.normalize_attrs(attrs):
if key == "href" and value.strip().startswith("javascript"):
pass
elif key in acceptable_attributes:
key = keymap.get(key, key)
clean_attrs.append((key, value))
elif key == 'style':
pass
# clean_value = self.sanitize_style(value)
# if clean_value: clean_attrs.append((key,clean_value))
_BaseHTMLProcessor.unknown_starttag(self, tag, clean_attrs)
def unknown_endtag(self, tag):
if tag not in self.acceptable_elements:
if tag in self.unacceptable_elements_with_end_tag:
self.unacceptablestack -= 1
if self.mathmlOK and tag in self.mathml_elements:
if tag == 'math' and self.mathmlOK:
self.mathmlOK -= 1
elif self.svgOK and tag in self.svg_elements:
tag = self.svg_elem_map.get(tag, tag)
if tag == 'svg' and self.svgOK:
self.svgOK -= 1
else:
return
_BaseHTMLProcessor.unknown_endtag(self, tag)
def handle_pi(self, text):
pass
def handle_decl(self, text):
pass
def handle_data(self, text):
if not self.unacceptablestack:
_BaseHTMLProcessor.handle_data(self, text)
def sanitize_style(self, style):
# disallow urls
style = re.compile('url\s*\(\s*[^\s)]+?\s*\)\s*').sub(' ', style)
# gauntlet
if not re.match("""^([:,;#%.\sa-zA-Z0-9!]|\w-\w|'[\s\w]+'|"[\s\w]+"|\([\d,\s]+\))*$""", style):
return ''
if not re.match("^(\s*[-\w]+\s*:\s*[^:;]*(;|$))*$", style):
return ''
clean = []
for prop, value in re.findall("([-\w]+)\s*:\s*([^:;]*)", style):
if not value:
continue
if prop.lower() in self.acceptable_css_properties:
clean.append(prop + ': ' + value + ';')
elif prop.split('-')[0].lower() in ['background', 'border', 'margin', 'padding']:
for keyword in value.split():
if keyword not in self.acceptable_css_keywords and \
not self.valid_css_values.match(keyword):
break
else:
clean.append(prop + ': ' + value + ';')
elif self.svgOK and prop.lower() in self.acceptable_svg_properties:
clean.append(prop + ': ' + value + ';')
return ' '.join(clean)
def sanitize_html(htmlSource, encoding="utf-8", type="text/html"):
p = _HTMLSanitizer(encoding, type)
p.feed(unicodify(htmlSource, encoding))
data = p.output()
data = data.strip().replace('\r\n', '\n')
return data
def sanitize_html(htmlSource):
return bleach.clean(htmlSource, tags=_acceptable_elements, attributes=_acceptable_attributes, strip=True)
lib/galaxy/util/validation.py
+6 -6
View File
@@ -12,13 +12,13 @@ def validate_and_sanitize_basestring(key, val):
if not isinstance(val, string_types):
raise exceptions.RequestParameterInvalidException('%s must be a string or unicode: %s'
% (key, str(type(val))))
return sanitize_html(val, 'utf-8', 'text/html')
return sanitize_html(val)
def validate_and_sanitize_basestring_list(key, val):
try:
assert isinstance(val, list)
return [sanitize_html(t, 'utf-8', 'text/html') for t in val]
return [sanitize_html(t) for t in val]
except (AssertionError, TypeError):
raise exceptions.RequestParameterInvalidException('%s must be a list of strings: %s'
% (key, str(type(val))))
@@ -32,7 +32,7 @@ def validate_boolean(key, val):
# TODO:
# def validate_integer( self, key, val, min, max ):
# def validate_float( self, key, val, min, max ):
# def validate_number( self, key, val, min, max ):
# def validate_genome_build( self, key, val ):
# def validate_integer(self, key, val, min, max):
# def validate_float(self, key, val, min, max):
# def validate_number(self, key, val, min, max):
# def validate_genome_build(self, key, val):
lib/galaxy/visualization/plugins/interactive_environments.py
+15 -1
View File
@@ -6,6 +6,7 @@ import shlex
import stat
import string
import tempfile
import time
import uuid
from itertools import product
from subprocess import PIPE, Popen
@@ -442,7 +443,20 @@ class InteractiveEnvironmentRequest(object):
"""
run_args = self.container_run_args(image, env_override, volumes)
container = self.attr.container_interface.run_in_container(None, **run_args)
container_port = self._find_port_mapping(container.ports)
attempt = 0
container_ports = container.ports
while container_ports is None and attempt < 30:
# TODO: it would be better to do this in /interactive_environments/ready so the client doesn't block here,
# but _find_port_mapping needs certain non-persisted data (the port configured to be published) and the
# proxy manager doesn't have an update method, so that'd require bigger changes than I have the time for
# right now
attempt += 1
log.warning("Sleeping for 2 seconds while waiting for container %s ports", container.id)
time.sleep(2)
container_ports = container.ports
if container_ports is None:
raise Exception("Failed to determine ports for container '%s' after 30 attempts" % container.id)
container_port = self._find_port_mapping(container_ports)
log.debug("Container '%s' accessible at: %s:%s", container.id, container_port.hostaddr, container_port.hostport)
self.attr.proxy_request = self.trans.app.proxy_manager.setup_proxy(
self.trans,
lib/galaxy/visualization/plugins/plugin.py
+14 -7
View File
@@ -269,6 +269,14 @@ class InteractiveEnvironmentPlugin(VisualizationPlugin):
context['base_url'] = 'interactive_environments'
super(InteractiveEnvironmentPlugin, self).__init__(app, path, name, config, context=context, **kwargs)
def _error_template(self, trans):
return trans.fill_template('message.mako',
message='Loading the interactive environment failed, please contact the {admin_tag} for assistance'.format(
admin_tag='<a href="mailto:{admin_mail}">Galaxy administrator</a>'.format(
admin_mail=trans.app.config.error_email_to)
if trans.app.config.error_email_to else 'Galaxy administrator'),
status='error')
def _render(self, render_vars, trans=None, embedded=None, **kwargs):
"""
Override to add interactive environment specific template vars.
@@ -294,16 +302,15 @@ class InteractiveEnvironmentPlugin(VisualizationPlugin):
request = self.INTENV_REQUEST_FACTORY(trans, self)
except Exception:
log.exception("IE plugin request handling failed")
return trans.fill_template('message.mako',
message='Loading the interactive environment failed, please contact the {admin_tag} for assistance'.format(
admin_tag='<a href="mailto:{admin_mail}">Galaxy administrator</a>'.format(
admin_mail=trans.app.config.error_email_to)
if trans.app.config.error_email_to else 'Galaxy administrator'),
status='error')
return self._error_template(trans)
render_vars["ie_request"] = request
template_filename = self.config['entry_point']['file']
return trans.fill_template(template_filename, template_lookup=self.template_lookup, **render_vars)
try:
return trans.fill_template(template_filename, template_lookup=self.template_lookup, **render_vars)
except Exception:
log.exception("IE plugin template fill failed")
return self._error_template(trans)
class ScriptVisualizationPlugin(VisualizationPlugin):
lib/galaxy/visualization/plugins/resource_parser.py
+2 -2
View File
@@ -29,7 +29,7 @@ class ResourceParser(object):
new keys (e.g. dataset_id="NNN" -> hda=<HistoryDatasetAssociation>).
"""
primitive_parsers = {
'str' : lambda param: galaxy.util.sanitize_html.sanitize_html(param, 'utf-8'),
'str' : lambda param: galaxy.util.sanitize_html.sanitize_html(param),
'bool' : lambda param: galaxy.util.string_as_bool(param),
'int' : int,
'float' : float,
@@ -217,7 +217,7 @@ class ResourceParser(object):
# TODO: ideally this would check v. a list of valid dbkeys
elif param_type == 'dbkey':
dbkey = query_param
parsed_param = galaxy.util.sanitize_html.sanitize_html(dbkey, 'utf-8')
parsed_param = galaxy.util.sanitize_html.sanitize_html(dbkey)
return parsed_param
lib/galaxy/web/base/controller.py
+2 -2
View File
@@ -517,7 +517,7 @@ class UsesLibraryMixinItems(SharableItemSecurityMixin):
# PRECONDITION: folder_id has already been altered to remove the folder prefix ('F')
# TODO: allow name and other, editable ldda attrs?
if ldda_message:
ldda_message = util.sanitize_html.sanitize_html(ldda_message, 'utf-8')
ldda_message = sanitize_html(ldda_message)
# check permissions on (all three?) resources: hda, library, folder
# TODO: do we really need the library??
@@ -1121,7 +1121,7 @@ class UsesVisualizationMixin(UsesLibraryMixinItems):
else:
self.create_item_slug(trans.sa_session, visualization)
if annotation:
annotation = sanitize_html(annotation, 'utf-8', 'text/html')
annotation = sanitize_html(annotation)
# TODO: if this is to stay in the mixin, UsesAnnotations should be added to the superclasses
# right now this is depending on the classes that include this mixin to have UsesAnnotations
self.add_item_annotation(trans.sa_session, trans.user, visualization, annotation)
lib/galaxy/web/framework/decorators.py
+1 -1
View File
@@ -134,7 +134,7 @@ def expose_api(func, to_json=True, user_required=True):
trans.response.headers['Cache-Control'] = "max-age=0,no-cache,no-store"
# Perform api_run_as processing, possibly changing identity
if 'payload' in kwargs and 'run_as' in kwargs['payload']:
if 'payload' in kwargs and isinstance(kwargs['payload'], dict) and 'run_as' in kwargs['payload']:
if not trans.user_can_do_run_as():
error_message = 'User does not have permissions to run jobs as another user'
return error
lib/galaxy/webapps/galaxy/api/_fetch_util.py
+217
View File
@@ -0,0 +1,217 @@
import logging
import os
from galaxy.actions.library import (
validate_path_upload,
validate_server_directory_upload,
)
from galaxy.exceptions import (
RequestParameterInvalidException
)
from galaxy.tools.actions.upload_common import validate_url
from galaxy.util import (
relpath,
)
log = logging.getLogger(__name__)
VALID_DESTINATION_TYPES = ["library", "library_folder", "hdca", "hdas"]
ELEMENTS_FROM_TYPE = ["archive", "bagit", "bagit_archive", "directory"]
# These elements_from cannot be sym linked to because they only exist during upload.
ELEMENTS_FROM_TRANSIENT_TYPES = ["archive", "bagit_archive"]
def validate_and_normalize_targets(trans, payload):
"""Validate and normalize all src references in fetch targets.
- Normalize ftp_import and server_dir src entries into simple path entires
with the relevant paths resolved and permissions / configuration checked.
- Check for file:// URLs in items src of "url" and convert them into path
src items - after verifying path pastes are allowed and user is admin.
- Check for valid URLs to be fetched for http and https entries.
- Based on Galaxy configuration and upload types set purge_source and in_place
as needed for each upload.
"""
targets = payload.get("targets", [])
for target in targets:
destination = _get_required_item(target, "destination", "Each target must specify a 'destination'")
destination_type = _get_required_item(destination, "type", "Each target destination must specify a 'type'")
if "object_id" in destination:
raise RequestParameterInvalidException("object_id not allowed to appear in the request.")
if destination_type not in VALID_DESTINATION_TYPES:
template = "Invalid target destination type [%s] encountered, must be one of %s"
msg = template % (destination_type, VALID_DESTINATION_TYPES)
raise RequestParameterInvalidException(msg)
if destination_type == "library":
library_name = _get_required_item(destination, "name", "Must specify a library name")
description = destination.get("description", "")
synopsis = destination.get("synopsis", "")
library = trans.app.library_manager.create(
trans, library_name, description=description, synopsis=synopsis
)
destination["type"] = "library_folder"
for key in ["name", "description", "synopsis"]:
if key in destination:
del destination[key]
destination["library_folder_id"] = trans.app.security.encode_id(library.root_folder.id)
# Unlike upload.py we don't transmit or use run_as_real_user in the job - we just make sure
# in_place and purge_source are set on the individual upload fetch sources as needed based
# on this.
run_as_real_user = trans.app.config.external_chown_script is not None # See comment in upload.py
purge_ftp_source = getattr(trans.app.config, 'ftp_upload_purge', True) and not run_as_real_user
payload["check_content"] = trans.app.config.check_upload_content
def check_src(item):
if "object_id" in item:
raise RequestParameterInvalidException("object_id not allowed to appear in the request.")
# Normalize file:// URLs into paths.
if item["src"] == "url" and item["url"].startswith("file://"):
item["src"] = "path"
item["path"] = item["url"][len("file://"):]
del item["path"]
if "in_place" in item:
raise RequestParameterInvalidException("in_place cannot be set in the upload request")
src = item["src"]
# Check link_data_only can only be set for certain src types and certain elements_from types.
_handle_invalid_link_data_only_elements_type(item)
if src not in ["path", "server_dir"]:
_handle_invalid_link_data_only_type(item)
elements_from = item.get("elements_from", None)
if elements_from and elements_from not in ELEMENTS_FROM_TYPE:
raise RequestParameterInvalidException("Invalid elements_from/items_from found in request")
if src == "path" or (src == "url" and item["url"].startswith("file:")):
# Validate is admin, leave alone.
validate_path_upload(trans)
elif src == "server_dir":
# Validate and replace with path definition.
server_dir = item["server_dir"]
full_path, _ = validate_server_directory_upload(trans, server_dir)
item["src"] = "path"
item["path"] = full_path
elif src == "ftp_import":
ftp_path = item["ftp_path"]
full_path = None
# It'd be nice if this can be de-duplicated with what is in parameters/grouping.py.
user_ftp_dir = trans.user_ftp_dir
is_directory = False
assert not os.path.islink(user_ftp_dir), "User FTP directory cannot be a symbolic link"
for (dirpath, dirnames, filenames) in os.walk(user_ftp_dir):
for filename in filenames:
if ftp_path == filename:
path = relpath(os.path.join(dirpath, filename), user_ftp_dir)
if not os.path.islink(os.path.join(dirpath, filename)):
full_path = os.path.abspath(os.path.join(user_ftp_dir, path))
break
for dirname in dirnames:
if ftp_path == dirname:
path = relpath(os.path.join(dirpath, dirname), user_ftp_dir)
if not os.path.islink(os.path.join(dirpath, dirname)):
full_path = os.path.abspath(os.path.join(user_ftp_dir, path))
is_directory = True
break
if is_directory:
# If the target is a directory - make sure no files under it are symbolic links
for (dirpath, dirnames, filenames) in os.walk(full_path):
for filename in filenames:
if ftp_path == filename:
path = relpath(os.path.join(dirpath, filename), full_path)
if not os.path.islink(os.path.join(dirpath, filename)):
full_path = False
break
for dirname in dirnames:
if ftp_path == dirname:
path = relpath(os.path.join(dirpath, filename), full_path)
if not os.path.islink(os.path.join(dirpath, filename)):
full_path = False
break
if not full_path:
raise RequestParameterInvalidException("Failed to find referenced ftp_path or symbolic link was enountered")
item["src"] = "path"
item["path"] = full_path
item["purge_source"] = purge_ftp_source
elif src == "url":
url = item["url"]
looks_like_url = False
for url_prefix in ["http://", "https://", "ftp://", "ftps://"]:
if url.startswith(url_prefix):
looks_like_url = True
break
if not looks_like_url:
raise RequestParameterInvalidException("Invalid URL [%s] found in src definition." % url)
validate_url(url, trans.app.config.fetch_url_whitelist_ips)
item["in_place"] = run_as_real_user
elif src == "files":
item["in_place"] = run_as_real_user
# Small disagreement with traditional uploads - we purge less by default since whether purging
# happens varies based on upload options in non-obvious ways.
# https://github.com/galaxyproject/galaxy/issues/5361
if "purge_source" not in item:
item["purge_source"] = False
_replace_request_syntax_sugar(targets)
_for_each_src(check_src, targets)
def _replace_request_syntax_sugar(obj):
# For data libraries and hdas to make sense - allow items and items_from in place of elements
# and elements_from. This is destructive and modifies the supplied request.
if isinstance(obj, list):
for el in obj:
_replace_request_syntax_sugar(el)
elif isinstance(obj, dict):
if "items" in obj:
obj["elements"] = obj["items"]
del obj["items"]
if "items_from" in obj:
obj["elements_from"] = obj["items_from"]
del obj["items_from"]
for value in obj.values():
_replace_request_syntax_sugar(value)
def _handle_invalid_link_data_only_type(item):
link_data_only = item.get("link_data_only", False)
if link_data_only:
raise RequestParameterInvalidException("link_data_only is invalid for src type [%s]" % item.get("src"))
def _handle_invalid_link_data_only_elements_type(item):
link_data_only = item.get("link_data_only", False)
if link_data_only and item.get("elements_from", False) in ELEMENTS_FROM_TRANSIENT_TYPES:
raise RequestParameterInvalidException("link_data_only is invalid for derived elements from [%s]" % item.get("elements_from"))
def _get_required_item(from_dict, key, message):
if key not in from_dict:
raise RequestParameterInvalidException(message)
return from_dict[key]
def _for_each_src(f, obj):
if isinstance(obj, list):
for item in obj:
_for_each_src(f, item)
if isinstance(obj, dict):
if "src" in obj:
f(obj)
for key, value in obj.items():
_for_each_src(f, value)
lib/galaxy/webapps/galaxy/api/annotations.py
+2 -2
View File
@@ -8,7 +8,7 @@ from galaxy import (
managers
)
from galaxy.model.item_attrs import UsesAnnotations
from galaxy.util import sanitize_html
from galaxy.util.sanitize_html import sanitize_html
from galaxy.web import _future_expose_api as expose_api
from galaxy.web.base.controller import (
BaseAPIController,
@@ -36,7 +36,7 @@ class BaseAnnotationsController(BaseAPIController, UsesStoredWorkflowMixin, Uses
if item is not None:
new_annotation = payload.get("text")
# TODO: sanitize on display not entry
new_annotation = sanitize_html.sanitize_html(new_annotation, 'utf-8', 'text/html')
new_annotation = sanitize_html(new_annotation)
self.add_item_annotation(trans.sa_session, trans.get_user(), item, new_annotation)
trans.sa_session.flush()
lib/galaxy/webapps/galaxy/api/folder_contents.py
+1 -1
View File
@@ -275,7 +275,7 @@ class FolderContentsController(BaseAPIController, UsesLibraryMixin, UsesLibraryM
from_hdca_id = payload.pop('from_hdca_id', None)
ldda_message = payload.pop('ldda_message', '')
if ldda_message:
ldda_message = util.sanitize_html.sanitize_html(ldda_message, 'utf-8')
ldda_message = util.sanitize_html.sanitize_html(ldda_message)
try:
if from_hda_id:
decoded_hda_id = self.decode_id(from_hda_id)
lib/galaxy/webapps/galaxy/api/library_contents.py
+2 -1
View File
@@ -185,7 +185,8 @@ class LibraryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibrary
* upload_option: (optional) one of 'upload_file' (default), 'upload_directory' or 'upload_paths'
* server_dir: (optional, only if upload_option is
'upload_directory') relative path of the subdirectory of Galaxy
``library_import_dir`` to upload. All and only the files (i.e.
``library_import_dir`` (if admin) or ``user_library_import_dir``
(if non-admin) to upload. All and only the files (i.e.
no subdirectories) contained in the specified directory will be
uploaded.
* filesystem_paths: (optional, only if upload_option is
lib/galaxy/webapps/galaxy/api/page_revisions.py
+1 -1
View File
@@ -66,7 +66,7 @@ class PageRevisionsController(BaseAPIController, SharableItemSecurityMixin, Uses
else:
title = page.title
content = sanitize_html(content, 'utf-8', 'text/html')
content = sanitize_html(content)
page_revision = trans.app.model.PageRevision()
page_revision.title = title
lib/galaxy/webapps/galaxy/api/pages.py
+2 -2
View File
@@ -81,13 +81,13 @@ class PagesController(BaseAPIController, SharableItemSecurityMixin, UsesAnnotati
raise exceptions.DuplicatedSlugException("Page slug must be unique")
content = payload.get("content", "")
content = sanitize_html(content, 'utf-8', 'text/html')
content = sanitize_html(content)
# Create the new stored page
page = trans.app.model.Page()
page.title = payload['title']
page.slug = payload['slug']
page_annotation = sanitize_html(payload.get("annotation", ""), 'utf-8', 'text/html')
page_annotation = sanitize_html(payload.get("annotation", ""))
self.add_item_annotation(trans.sa_session, trans.get_user(), page, page_annotation)
page.user = user
# And the first (empty) page revision
lib/galaxy/webapps/galaxy/api/tools.py
+116
View File
@@ -6,15 +6,23 @@ from six.moves.urllib.parse import unquote_plus
import galaxy.queue_worker
from galaxy import exceptions, managers, util, web
from galaxy.managers.collections_util import dictify_dataset_collection_instance
from galaxy.util.json import safe_dumps
from galaxy.util.odict import odict
from galaxy.visualization.genomes import GenomeRegion
from galaxy.web import _future_expose_api as expose_api
from galaxy.web import _future_expose_api_anonymous as expose_api_anonymous
from galaxy.web import _future_expose_api_anonymous_and_sessionless as expose_api_anonymous_and_sessionless
from galaxy.web import _future_expose_api_raw_anonymous_and_sessionless as expose_api_raw_anonymous_and_sessionless
from galaxy.web.base.controller import BaseAPIController
from galaxy.web.base.controller import UsesVisualizationMixin
from ._fetch_util import validate_and_normalize_targets
log = logging.getLogger(__name__)
# Do not allow these tools to be called directly - they (it) enforces extra security and
# provides access via a different API endpoint.
PROTECTED_TOOLS = ["__DATA_FETCH__"]
class ToolsController(BaseAPIController, UsesVisualizationMixin):
"""
@@ -98,6 +106,74 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin):
tool = self._get_tool(id, tool_version=tool_version, user=trans.user)
return tool.to_json(trans, kwd.get('inputs', kwd))
@expose_api
@web.require_admin
def test_data_path(self, trans, id, **kwd):
"""
GET /api/tools/{tool_id}/test_data_path?tool_version={tool_version}
"""
# TODO: eliminate copy and paste with above code.
if 'payload' in kwd:
kwd = kwd.get('payload')
tool_version = kwd.get('tool_version', None)
tool = self._get_tool(id, tool_version=tool_version, user=trans.user)
path = tool.test_data_path(kwd.get("filename"))
if path:
return path
else:
raise exceptions.ObjectNotFound("Specified test data path not found.")
@expose_api_anonymous_and_sessionless
def tests_summary(self, trans, **kwd):
"""
GET /api/tools/tests_summary
Fetch summary information for each tool and version combination with tool tests
defined. This summary information currently includes tool name and a count of
the tests.
Fetch complete test data for each tool with /api/tools/{tool_id}/test_data?tool_version=<tool_version>
"""
test_counts_by_tool = {}
for id, tool in self.app.toolbox.tools():
tests = tool.tests
if tests:
if tool.id not in test_counts_by_tool:
test_counts_by_tool[tool.id] = {}
available_versions = test_counts_by_tool[tool.id]
available_versions[tool.version] = {
"tool_name": tool.name,
"count": len(tests),
}
return test_counts_by_tool
@expose_api_raw_anonymous_and_sessionless
def test_data(self, trans, id, **kwd):
"""
GET /api/tools/{tool_id}/test_data?tool_version={tool_version}
This API endpoint is unstable and experimental. In particular the format of the
response has not been entirely nailed down (it exposes too many Galaxy
internals/Pythonisms in a rough way). If this endpoint is being used from outside
of scripts shipped with Galaxy let us know and please be prepared for the response
from this API to change its format in some ways.
"""
# TODO: eliminate copy and paste with above code.
if 'payload' in kwd:
kwd = kwd.get('payload')
tool_version = kwd.get('tool_version', None)
tool = self._get_tool(id, tool_version=tool_version, user=trans.user)
# Encode in this method to handle odict objects in tool representation.
def json_encodeify(obj):
if isinstance(obj, odict):
return dict(obj)
else:
return obj
result = [t.to_dict() for t in tool.tests]
return safe_dumps(result, default=json_encodeify)
@expose_api
@web.require_admin
def reload(self, trans, id, **kwd):
@@ -290,12 +366,52 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin):
trans.response.headers["Content-Disposition"] = 'attachment; filename="%s.tgz"' % (id)
return download_file
@expose_api_anonymous
def fetch(self, trans, payload, **kwd):
"""Adapt clean API to tool-constrained API.
"""
log.info("Keywords are %s" % payload)
request_version = '1'
history_id = payload.pop("history_id")
clean_payload = {}
files_payload = {}
for key, value in payload.items():
if key == "key":
continue
if key.startswith('files_') or key.startswith('__files_'):
files_payload[key] = value
continue
clean_payload[key] = value
log.info("payload %s" % clean_payload)
validate_and_normalize_targets(trans, clean_payload)
clean_payload["check_content"] = trans.app.config.check_upload_content
request = dumps(clean_payload)
log.info(request)
create_payload = {
'tool_id': "__DATA_FETCH__",
'history_id': history_id,
'inputs': {
'request_version': request_version,
'request_json': request,
},
}
create_payload.update(files_payload)
return self._create(trans, create_payload, **kwd)
@expose_api_anonymous
def create(self, trans, payload, **kwd):
"""
POST /api/tools
Executes tool using specified inputs and returns tool's outputs.
"""
tool_id = payload.get("tool_id")
if tool_id in PROTECTED_TOOLS:
raise exceptions.RequestParameterInvalidException("Cannot execute tool [%s] directly, must use alternative endpoint." % tool_id)
if tool_id is None:
raise exceptions.RequestParameterInvalidException("Must specify a valid tool_id to use this endpoint.")
return self._create(trans, payload, **kwd)
def _create(self, trans, payload, **kwd):
# HACK: for now, if action is rerun, rerun tool.
action = payload.get('action', None)
if action == 'rerun':
lib/galaxy/webapps/galaxy/api/uploads.py
+53
View File
@@ -0,0 +1,53 @@
"""
API operations for uploaded files in storage.
"""
import logging
import os
import re
from galaxy.exceptions import MessageException, NotImplemented
from galaxy.web import expose_api_anonymous
from galaxy.web.base.controller import BaseAPIController
log = logging.getLogger(__name__)
class UploadsAPIController(BaseAPIController):
READ_CHUNK_SIZE = 2 ** 16
@expose_api_anonymous
def index(self, trans, **kwd):
raise NotImplemented("Listing uploads is not implemented.")
@expose_api_anonymous
def create(self, trans, payload, **kwd):
"""
POST /api/uploads/
"""
session_id = payload.get("session_id")
session_start = payload.get("session_start")
session_chunk = payload.get("session_chunk")
if re.match('^[\w-]+$', session_id) is None:
raise MessageException("Requires a session id.")
if session_start is None:
raise MessageException("Requires a session start.")
if not hasattr(session_chunk, "file"):
raise MessageException("Requires a session chunk.")
target_file = os.path.join(trans.app.config.new_file_path, session_id)
target_size = 0
if os.path.exists(target_file):
target_size = os.path.getsize(target_file)
if session_start != target_size:
raise MessageException("Incorrect session start.")
chunk_size = os.fstat(session_chunk.file.fileno()).st_size
if chunk_size > trans.app.config.chunk_upload_size:
raise MessageException("Invalid chunk size.")
with open(target_file, "a") as f:
while True:
read_chunk = session_chunk.file.read(self.READ_CHUNK_SIZE)
if not read_chunk:
break
f.write(read_chunk)
session_chunk.file.close()
return {"message": "Successful."}
lib/galaxy/webapps/galaxy/api/visualizations.py
+5 -5
View File
@@ -176,7 +176,7 @@ class VisualizationsController(BaseAPIController, UsesVisualizationMixin, Sharab
if key == 'type':
if not isinstance(val, string_types):
raise ValidationError('%s must be a string or unicode: %s' % (key, str(type(val))))
val = util.sanitize_html.sanitize_html(val, 'utf-8')
val = util.sanitize_html.sanitize_html(val)
elif key == 'config':
if not isinstance(val, dict):
raise ValidationError('%s must be a dictionary: %s' % (key, str(type(val))))
@@ -184,22 +184,22 @@ class VisualizationsController(BaseAPIController, UsesVisualizationMixin, Sharab
elif key == 'annotation':
if not isinstance(val, string_types):
raise ValidationError('%s must be a string or unicode: %s' % (key, str(type(val))))
val = util.sanitize_html.sanitize_html(val, 'utf-8')
val = util.sanitize_html.sanitize_html(val)
# these are keys that actually only be *updated* at the revision level and not here
# (they are still valid for create, tho)
elif key == 'title':
if not isinstance(val, string_types):
raise ValidationError('%s must be a string or unicode: %s' % (key, str(type(val))))
val = util.sanitize_html.sanitize_html(val, 'utf-8')
val = util.sanitize_html.sanitize_html(val)
elif key == 'slug':
if not isinstance(val, string_types):
raise ValidationError('%s must be a string: %s' % (key, str(type(val))))
val = util.sanitize_html.sanitize_html(val, 'utf-8')
val = util.sanitize_html.sanitize_html(val)
elif key == 'dbkey':
if not isinstance(val, string_types):
raise ValidationError('%s must be a string or unicode: %s' % (key, str(type(val))))
val = util.sanitize_html.sanitize_html(val, 'utf-8')
val = util.sanitize_html.sanitize_html(val)
elif key not in valid_but_uneditable_keys:
continue
lib/galaxy/webapps/galaxy/buildapp.py
+5
View File
@@ -256,6 +256,7 @@ def populate_api_routes(webapp, app):
webapp.mapper.resource('dataset_collection', 'dataset_collections', path_prefix='/api/')
webapp.mapper.resource('form', 'forms', path_prefix='/api')
webapp.mapper.resource('role', 'roles', path_prefix='/api')
webapp.mapper.resource('upload', 'uploads', path_prefix='/api')
webapp.mapper.connect('/api/ftp_files', controller='remote_files')
webapp.mapper.resource('remote_file', 'remote_files', path_prefix='/api')
webapp.mapper.resource('group', 'groups', path_prefix='/api')
@@ -271,9 +272,13 @@ def populate_api_routes(webapp, app):
# ====== TOOLS API ======
# =======================
webapp.mapper.connect('/api/tools/fetch', action='fetch', controller='tools', conditions=dict(method=["POST"]))
webapp.mapper.connect('/api/tools/all_requirements', action='all_requirements', controller="tools")
webapp.mapper.connect('/api/tools/{id:.+?}/build', action='build', controller="tools")
webapp.mapper.connect('/api/tools/{id:.+?}/reload', action='reload', controller="tools")
webapp.mapper.connect('/api/tools/tests_summary', action='tests_summary', controller="tools")
webapp.mapper.connect('/api/tools/{id:.+?}/test_data_path', action='test_data_path', controller="tools")
webapp.mapper.connect('/api/tools/{id:.+?}/test_data', action='test_data', controller="tools")
webapp.mapper.connect('/api/tools/{id:.+?}/diagnostics', action='diagnostics', controller="tools")
webapp.mapper.connect('/api/tools/{id:.+?}/citations', action='citations', controller="tools")
webapp.mapper.connect('/api/tools/{id:.+?}/download', action='download', controller="tools")
lib/galaxy/webapps/galaxy/config_schema.yml
+21 -23
View File
@@ -1239,6 +1239,15 @@ mapping:
operations on the remote end. See the Galaxy nginx documentation for the
corresponding nginx configuration.
chunk_upload_size:
type: int
default: 104857600
required: false
desc: |
Galaxy can upload user files in chunks without using nginx. Enable the chunk
uploader by specifying a chunk size larger than 0. The chunk size is specified
in bytes (default: 100MB).
dynamic_proxy_manage:
type: bool
default: true
@@ -1817,6 +1826,18 @@ mapping:
desc: |
Set maximum size of ngrams
tool_test_data_directories:
type: str
default: 'test-data'
required: false
desc: |
Set tool test data directory. The test framework sets this value to
'test-data,https://github.com/galaxyproject/galaxy-test-data.git' which will
cause Galaxy to clone down extra test data on the fly for certain tools
distributed with Galaxy but this is likely not appropriate for production systems.
Instead one can simply clone that repository directly and specify a path here
instead of a Git HTTP repository.
id_secret:
type: str
default: USING THE DEFAULT IS NOT SECURE!
@@ -2518,29 +2539,6 @@ mapping:
overwrite default job resources such as number of processors, memory and
walltime.
workflow_resource_params_file:
type: str
default: config/workflow_resource_params_conf.xml
required: false
desc: |
Similar to the above parameter, workflows can describe parameters used to
influence scheduling of jobs within the workflow. This requires both a description
of the fields available (which defaults to the definitions in
job_resource_params_file if not set).
workflow_resource_params_mapper:
type: str
default: config/workflow_resource_mapper_conf.yml
required: false
desc: |
This parameter describes how to map users and workflows to a set of workflow
resource parameter to present (typically input IDs from workflow_resource_params_file).
If this this is a function reference it will be passed various inputs (workflow model
object and user) and it should produce a list of input IDs. If it is a path
it is expected to an XML or YAML file describing how to map group names to parameter
descriptions (additional types of mappings via these files could be implemented but
haven't yet - for instance using workflow tags to do the mapping).
cache_user_job_count:
type: bool
default: false
lib/galaxy/webapps/galaxy/controllers/dataset.py
+2 -2
View File
@@ -435,7 +435,7 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE
data.datatype.after_setting_metadata(data)
# Sanitize annotation before adding it.
if payload.get('annotation'):
annotation = sanitize_html(payload.get('annotation'), 'utf-8', 'text/html')
annotation = sanitize_html(payload.get('annotation'))
self.add_item_annotation(trans.sa_session, trans.get_user(), data, annotation)
# if setting metadata previously failed and all required elements have now been set, clear the failed state.
if data._state == trans.model.Dataset.states.FAILED_METADATA and not data.missing_meta():
@@ -700,7 +700,7 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE
web.httpexceptions.HTTPNotFound()
if dataset and new_annotation:
# Sanitize annotation before adding it.
new_annotation = sanitize_html(new_annotation, 'utf-8', 'text/html')
new_annotation = sanitize_html(new_annotation)
self.add_item_annotation(trans.sa_session, trans.get_user(), dataset, new_annotation)
trans.sa_session.flush()
return new_annotation
lib/galaxy/webapps/galaxy/controllers/history.py
+7 -7
View File
@@ -494,11 +494,10 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt
history = trans.sa_session.query(model.History).options(
eagerload_all('active_datasets.creating_job_associations.job.workflow_invocation_step.workflow_invocation.workflow'),
).get(id)
assert history
# TODO: formalize to trans.show_error
assert (history.user and (history.user.id == trans.user.id) or
(history.id == trans.history.id) or
(trans.user_is_admin()))
if not (history and ((history.user and trans.user and history.user.id == trans.user.id) or
(trans.history and history.id == trans.history.id) or
trans.user_is_admin())):
return trans.show_error_message("Cannot display history structure.")
# Resolve jobs and workflow invocations for the datasets in the history
# items is filled with items (hdas, jobs, or workflows) that go at the
# top level
@@ -1127,7 +1126,7 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt
@web.expose
def purge_deleted_datasets(self, trans):
count = 0
if trans.app.config.allow_user_dataset_purge:
if trans.app.config.allow_user_dataset_purge and trans.history:
for hda in trans.history.datasets:
if not hda.deleted or hda.purged:
continue
@@ -1145,7 +1144,8 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt
except Exception:
log.exception('Unable to purge dataset (%s) on purge of hda (%s):' % (hda.dataset.id, hda.id))
count += 1
return trans.show_ok_message("%d datasets have been deleted permanently" % count, refresh_frames=['history'])
return trans.show_ok_message("%d datasets have been deleted permanently" % count, refresh_frames=['history'])
return trans.show_error_message("Cannot purge deleted datasets from this session.")
@web.expose
def delete(self, trans, id, purge=False):
lib/galaxy/webapps/galaxy/controllers/page.py
+187 -31
View File
@@ -1,16 +1,75 @@
import re
from json import loads
from markupsafe import escape
from sqlalchemy import and_, desc, false, true
from sqlalchemy.orm import eagerload, undefer
from six.moves.html_entities import name2codepoint
from six.moves.html_parser import HTMLParser
from sqlalchemy import (
and_,
desc,
false,
true
)
from sqlalchemy.orm import (
eagerload,
undefer
)
from galaxy import managers, model, util, web
from galaxy import (
managers,
model,
util,
web
)
from galaxy.model.item_attrs import UsesItemRatings
from galaxy.util import unicodify
from galaxy.util.sanitize_html import _BaseHTMLProcessor, sanitize_html
from galaxy.web import error, url_for
from galaxy.web.base.controller import BaseUIController, SharableMixin, UsesStoredWorkflowMixin, UsesVisualizationMixin
from galaxy.web.framework.helpers import grids, time_ago
from galaxy.util.sanitize_html import sanitize_html
from galaxy.web import (
error,
url_for
)
from galaxy.web.base.controller import (
BaseUIController,
SharableMixin,
UsesStoredWorkflowMixin,
UsesVisualizationMixin
)
from galaxy.web.framework.helpers import (
grids,
time_ago
)
# Copied from https://github.com/kurtmckee/feedparser
_cp1252 = {
128: u'\u20ac', # euro sign
130: u'\u201a', # single low-9 quotation mark
131: u'\u0192', # latin small letter f with hook
132: u'\u201e', # double low-9 quotation mark
133: u'\u2026', # horizontal ellipsis
134: u'\u2020', # dagger
135: u'\u2021', # double dagger
136: u'\u02c6', # modifier letter circumflex accent
137: u'\u2030', # per mille sign
138: u'\u0160', # latin capital letter s with caron
139: u'\u2039', # single left-pointing angle quotation mark
140: u'\u0152', # latin capital ligature oe
142: u'\u017d', # latin capital letter z with caron
145: u'\u2018', # left single quotation mark
146: u'\u2019', # right single quotation mark
147: u'\u201c', # left double quotation mark
148: u'\u201d', # right double quotation mark
149: u'\u2022', # bullet
150: u'\u2013', # en dash
151: u'\u2014', # em dash
152: u'\u02dc', # small tilde
153: u'\u2122', # trade mark sign
154: u'\u0161', # latin small letter s with caron
155: u'\u203a', # single right-pointing angle quotation mark
156: u'\u0153', # latin small ligature oe
158: u'\u017e', # latin small letter z with caron
159: u'\u0178', # latin capital letter y with diaeresis
}
def format_bool(b):
@@ -223,18 +282,52 @@ class VisualizationSelectionGrid(ItemSelectionGrid):
)
class _PageContentProcessor(_BaseHTMLProcessor):
""" Processes page content to produce HTML that is suitable for display. For now, processor renders embedded objects. """
# Adapted from the _BaseHTMLProcessor class of https://github.com/kurtmckee/feedparser
class _PageContentProcessor(HTMLParser, object):
"""
Processes page content to produce HTML that is suitable for display.
For now, processor renders embedded objects.
"""
bare_ampersand = re.compile("&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)")
elements_no_end_tag = set([
'area', 'base', 'basefont', 'br', 'col', 'command', 'embed', 'frame',
'hr', 'img', 'input', 'isindex', 'keygen', 'link', 'meta', 'param',
'source', 'track', 'wbr'
])
def __init__(self, trans, encoding, type, render_embed_html_fn):
_BaseHTMLProcessor.__init__(self, encoding, type)
def __init__(self, trans, render_embed_html_fn):
HTMLParser.__init__(self)
self.trans = trans
self.ignore_content = False
self.num_open_tags_for_ignore = 0
self.render_embed_html_fn = render_embed_html_fn
def unknown_starttag(self, tag, attrs):
""" Called for each start tag; attrs is a list of (attr, value) tuples. """
def reset(self):
self.pieces = []
HTMLParser.reset(self)
def _shorttag_replace(self, match):
tag = match.group(1)
if tag in self.elements_no_end_tag:
return '<' + tag + ' />'
else:
return '<' + tag + '></' + tag + '>'
def feed(self, data):
data = re.compile(r'<!((?!DOCTYPE|--|\[))', re.IGNORECASE).sub(r'&lt;!\1', data)
data = re.sub(r'<([^<>\s]+?)\s*/>', self._shorttag_replace, data)
data = data.replace('&#39;', "'")
data = data.replace('&#34;', '"')
HTMLParser.feed(self, data)
HTMLParser.close(self)
def handle_starttag(self, tag, attrs):
"""
Called for each start tag
attrs is a list of (attr, value) tuples, e.g. for <pre class='screen'>,
tag='pre', attrs=[('class', 'screen')]
"""
# If ignoring content, just increment tag count and ignore.
if self.ignore_content:
@@ -264,17 +357,25 @@ class _PageContentProcessor(_BaseHTMLProcessor):
return
# Default behavior: not ignoring and no embedded content.
_BaseHTMLProcessor.unknown_starttag(self, tag, attrs)
uattrs = []
strattrs = ''
if attrs:
for key, value in attrs:
value = value.replace('>', '&gt;').replace('<', '&lt;').replace('"', '&quot;')
value = self.bare_ampersand.sub("&amp;", value)
uattrs.append((key, value))
strattrs = ''.join(' %s="%s"' % (k, v) for k, v in uattrs)
if tag in self.elements_no_end_tag:
self.pieces.append('<%s%s />' % (tag, strattrs))
else:
self.pieces.append('<%s%s>' % (tag, strattrs))
def handle_data(self, text):
""" Called for each block of plain text. """
if self.ignore_content:
return
_BaseHTMLProcessor.handle_data(self, text)
def unknown_endtag(self, tag):
""" Called for each end tag. """
def handle_endtag(self, tag):
"""
Called for each end tag
E.g. for </pre>, tag will be 'pre'
"""
# If ignoring content, see if current tag is the end of content to ignore.
if self.ignore_content:
self.num_open_tags_for_ignore -= 1
@@ -283,8 +384,63 @@ class _PageContentProcessor(_BaseHTMLProcessor):
self.ignore_content = False
return
# Default behavior:
_BaseHTMLProcessor.unknown_endtag(self, tag)
# Default behavior: reconstruct the original end tag.
if tag not in self.elements_no_end_tag:
self.pieces.append("</%s>" % tag)
def handle_charref(self, ref):
# called for each character reference, e.g. for '&#160;', ref will be '160'
# Reconstruct the original character reference.
ref = ref.lower()
if ref.startswith('x'):
value = int(ref[1:], 16)
else:
value = int(ref)
if value in _cp1252:
self.pieces.append('&#%s;' % hex(ord(_cp1252[value]))[1:])
else:
self.pieces.append('&#%s;' % ref)
def handle_entityref(self, ref):
# called for each entity reference, e.g. for '&copy;', ref will be 'copy'
# Reconstruct the original entity reference.
if ref in name2codepoint or ref == 'apos':
self.pieces.append('&%s;' % ref)
else:
self.pieces.append('&amp;%s' % ref)
def handle_data(self, text):
"""
Called for each block of plain text
Called outside of any tag and not containing any character or entity
references. Store the original text verbatim.
"""
if self.ignore_content:
return
self.pieces.append(text)
def handle_comment(self, text):
# called for each HTML comment, e.g. <!-- insert Javascript code here -->
# Reconstruct the original comment.
self.pieces.append('<!--%s-->' % text)
def handle_decl(self, text):
# called for the DOCTYPE, if present, e.g.
# <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
# "http://www.w3.org/TR/html4/loose.dtd">
# Reconstruct original DOCTYPE
self.pieces.append('<!%s>' % text)
def handle_pi(self, text):
# called for each processing instruction, e.g. <?instruction>
# Reconstruct original processing instruction.
self.pieces.append('<?%s>' % text)
def output(self):
'''Return processed HTML as a single string'''
return ''.join(self.pieces)
class PageController(BaseUIController, SharableMixin,
@@ -387,7 +543,7 @@ class PageController(BaseUIController, SharableMixin,
p.slug = p_slug
p.user = user
if p_annotation:
p_annotation = sanitize_html(p_annotation, 'utf-8', 'text/html')
p_annotation = sanitize_html(p_annotation)
self.add_item_annotation(trans.sa_session, user, p, p_annotation)
# And the first (empty) page revision
p_revision = model.PageRevision()
@@ -449,7 +605,7 @@ class PageController(BaseUIController, SharableMixin,
p.title = p_title
p.slug = p_slug
if p_annotation:
p_annotation = sanitize_html(p_annotation, 'utf-8', 'text/html')
p_annotation = sanitize_html(p_annotation)
self.add_item_annotation(trans.sa_session, user, p, p_annotation)
trans.sa_session.add(p)
trans.sa_session.flush()
@@ -550,7 +706,7 @@ class PageController(BaseUIController, SharableMixin,
assert page.user == trans.user
# Sanitize content
content = sanitize_html(content, 'utf-8', 'text/html')
content = sanitize_html(content)
# Add a new revision to the page with the provided content.
page_revision = model.PageRevision()
@@ -567,7 +723,7 @@ class PageController(BaseUIController, SharableMixin,
item = trans.sa_session.query(item_class).filter_by(id=item_id).first()
if not item:
raise RuntimeError("cannot find annotated item")
text = sanitize_html(annotation_dict['text'], 'utf-8', 'text/html')
text = sanitize_html(annotation_dict['text'])
# Add/update annotation.
if item_id and item_class and text:
@@ -615,8 +771,10 @@ class PageController(BaseUIController, SharableMixin,
self.security_check(trans, page, False, True)
# Process page content.
processor = _PageContentProcessor(trans, 'utf-8', 'text/html', self._get_embed_html)
processor = _PageContentProcessor(trans, self._get_embed_html)
processor.feed(page.latest_revision.content)
# Output is string, so convert to unicode for display.
page_content = unicodify(processor.output(), 'utf-8')
# Get rating data.
user_item_rating = 0
@@ -628,8 +786,6 @@ class PageController(BaseUIController, SharableMixin,
user_item_rating = 0
ave_item_rating, num_ratings = self.get_ave_item_rating_data(trans.sa_session, page)
# Output is string, so convert to unicode for display.
page_content = unicodify(processor.output(), 'utf-8')
return trans.fill_template_mako("page/display.mako", item=page,
item_data=page_content,
user_item_rating=user_item_rating,
lib/galaxy/webapps/galaxy/controllers/root.py
-1
View File
@@ -108,7 +108,6 @@ class RootController(controller.JSAppLauncher, UsesAnnotations):
js_options = self._get_js_options(trans)
config = js_options['config']
config.update(self._get_extended_config(trans))
return self.template(trans, 'analysis', options=js_options)
@web.expose
lib/galaxy/webapps/galaxy/controllers/user.py
+6 -1
View File
@@ -925,7 +925,9 @@ class User(BaseUIController, UsesFormDefinitionsMixin, CreatesUsersMixin, Create
Check whether token fits the user and then activate the user's account.
"""
params = util.Params(kwd, sanitize=False)
email = unquote(params.get('email', None))
email = params.get('email', None)
if email is not None:
email = unquote(email)
activation_token = params.get('activation_token', None)
if email is None or activation_token is None:
@@ -934,6 +936,9 @@ class User(BaseUIController, UsesFormDefinitionsMixin, CreatesUsersMixin, Create
else:
# Find the user
user = trans.sa_session.query(trans.app.model.User).filter(trans.app.model.User.table.c.email == email).first()
if not user:
# Probably wrong email address
return trans.show_error_message("You are using an invalid activation link. Try to log in and we will send you a new activation email. <br><a href='%s'>Go to login page.</a>") % web.url_for(controller="root", action="index")
# If the user is active already don't try to activate
if user.active is True:
return trans.show_ok_message("Your account is already active. Nothing has changed. <br><a href='%s'>Go to login page.</a>") % web.url_for(controller='root', action='index')
lib/galaxy/webapps/galaxy/controllers/visualization.py
+1 -1
View File
@@ -620,7 +620,7 @@ class VisualizationController(BaseUIController, SharableMixin, UsesVisualization
v.slug = v_slug
v.dbkey = v_dbkey
if v_annotation:
v_annotation = sanitize_html(v_annotation, 'utf-8', 'text/html')
v_annotation = sanitize_html(v_annotation)
self.add_item_annotation(trans.sa_session, trans.get_user(), v, v_annotation)
trans.sa_session.add(v)
trans.sa_session.flush()
lib/galaxy/webapps/galaxy/controllers/workflow.py
+9 -8
View File
@@ -4,10 +4,10 @@ import base64
import json
import logging
import os
import sgmllib
import requests
from markupsafe import escape
from six.moves.html_parser import HTMLParser
from six.moves.http_client import HTTPConnection
from sqlalchemy import and_
from sqlalchemy.orm import eagerload, joinedload, lazyload, undefer
@@ -153,16 +153,17 @@ class StoredWorkflowAllPublishedGrid(grids.Grid):
self.model_class.deleted == expression.false())
# Simple SGML parser to get all content in a single tag.
class SingleTagContentsParser(sgmllib.SGMLParser):
# Simple HTML parser to get all content in a single tag.
class SingleTagContentsParser(HTMLParser):
def __init__(self, target_tag):
sgmllib.SGMLParser.__init__(self)
# Cannot use super() because HTMLParser is an old-style class in Python2
HTMLParser.__init__(self)
self.target_tag = target_tag
self.cur_tag = None
self.tag_content = ""
def unknown_starttag(self, tag, attrs):
def handle_starttag(self, tag, attrs):
""" Called for each start tag. """
self.cur_tag = tag
@@ -404,7 +405,7 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi
stored = self.get_stored_workflow(trans, id)
if new_annotation:
# Sanitize annotation before adding it.
new_annotation = sanitize_html(new_annotation, 'utf-8', 'text/html')
new_annotation = sanitize_html(new_annotation)
self.add_item_annotation(trans.sa_session, trans.get_user(), stored, new_annotation)
trans.sa_session.flush()
return new_annotation
@@ -547,7 +548,7 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi
workflow.stored_workflow = stored_workflow
stored_workflow.latest_workflow = workflow
# Add annotation.
workflow_annotation = sanitize_html(workflow_annotation, 'utf-8', 'text/html')
workflow_annotation = sanitize_html(workflow_annotation)
self.add_item_annotation(trans.sa_session, trans.get_user(), stored_workflow, workflow_annotation)
# Persist
session = trans.sa_session
@@ -573,7 +574,7 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi
workflow.stored_workflow = stored_workflow
stored_workflow.latest_workflow = workflow
# Add annotation.
workflow_annotation = sanitize_html(workflow_annotation, 'utf-8', 'text/html')
workflow_annotation = sanitize_html(workflow_annotation)
self.add_item_annotation(trans.sa_session, trans.get_user(), stored_workflow, workflow_annotation)
# Persist
lib/galaxy/workflow/modules.py
+4 -3
View File
@@ -825,6 +825,9 @@ class ToolModule(WorkflowModule):
invocation = invocation_step.workflow_invocation
step = invocation_step.workflow_step
tool = trans.app.toolbox.get_tool(step.tool_id, tool_version=step.tool_version)
if not tool.is_workflow_compatible:
message = "Specified tool [%s] in workflow is not workflow-compatible." % tool.id
raise Exception(message)
tool_state = step.state
# Not strictly needed - but keep Tool state clean by stripping runtime
# metadata parameters from it.
@@ -843,7 +846,6 @@ class ToolModule(WorkflowModule):
else:
iteration_elements_iter = [None]
resource_parameters = invocation.resource_parameters
for iteration_elements in iteration_elements_iter:
execution_state = tool_state.copy()
# TODO: Move next step into copy()
@@ -916,8 +918,7 @@ class ToolModule(WorkflowModule):
invocation_step=invocation_step,
max_num_jobs=max_num_jobs,
job_callback=lambda job: self._handle_post_job_actions(step, job, invocation.replacement_dict),
completed_jobs=completed_jobs,
workflow_resource_parameters=resource_parameters
completed_jobs=completed_jobs
)
complete = True
except PartialJobExecution as pje:
lib/galaxy/workflow/run_request.py
+8 -48
View File
@@ -7,7 +7,6 @@ from galaxy import (
)
from galaxy.managers import histories
from galaxy.tools.parameters.meta import expand_workflow_inputs
from galaxy.workflow.resources import get_resource_mapper_function
INPUT_STEP_TYPES = ['data_input', 'data_collection_input', 'parameter_input']
@@ -45,17 +44,15 @@ class WorkflowRunConfig(object):
def __init__(self, target_history,
replacement_dict,
copy_inputs_to_history=False,
inputs={},
param_map={},
inputs=None,
param_map=None,
allow_tool_state_corrections=False,
use_cached_job=False,
resource_params={}):
use_cached_job=False):
self.target_history = target_history
self.replacement_dict = replacement_dict
self.copy_inputs_to_history = copy_inputs_to_history
self.inputs = inputs
self.param_map = param_map
self.resource_params = resource_params
self.inputs = inputs or {}
self.param_map = param_map or {}
self.allow_tool_state_corrections = allow_tool_state_corrections
self.use_cached_job = use_cached_job
@@ -171,7 +168,8 @@ def _flatten_step_params(param_dict, prefix=""):
return new_params
def _get_target_history(trans, workflow, payload, param_keys=[], index=0):
def _get_target_history(trans, workflow, payload, param_keys=None, index=0):
param_keys = param_keys or []
history_name = payload.get('new_history_name', None)
history_id = payload.get('history_id', None)
history_param = payload.get('history', None)
@@ -307,35 +305,6 @@ def build_workflow_run_configs(trans, workflow, payload):
normalized_inputs[key] = value['content']
else:
normalized_inputs[key] = value
resource_params = payload.get('resource_params', {})
if resource_params:
# quick attempt to validate parameters, just handle select options now since is what
# is needed for DTD - arbitrary plugins can define arbitrary logic at runtime in the
# destination function. In the future this should be extended to allow arbitrary
# pluggable validation.
resource_mapper_function = get_resource_mapper_function(trans.app)
# TODO: Do we need to do anything with the stored_workflow or can this be removed.
resource_parameters = resource_mapper_function(trans=trans, stored_workflow=None, workflow=workflow)
for resource_parameter in resource_parameters:
if resource_parameter.get("type") == "select":
name = resource_parameter.get("name")
if name in resource_params:
value = resource_params[name]
valid_option = False
# TODO: How should be handle the case where no selection is made by the user
# This can happen when there is a select on the page but the user has no options to select
# Here I have the validation pass it through. An alternative may be to remove the parameter if
# it is None.
if value is None:
valid_option = True
else:
for option_elem in resource_parameter.get('data'):
option_value = option_elem.get("value")
if value == option_value:
valid_option = True
if not valid_option:
raise exceptions.RequestParameterInvalidException("Invalid value for parameter '%s' found." % name)
run_configs.append(WorkflowRunConfig(
target_history=history,
replacement_dict=payload.get('replacement_params', {}),
@@ -343,7 +312,6 @@ def build_workflow_run_configs(trans, workflow, payload):
param_map=param_map,
allow_tool_state_corrections=allow_tool_state_corrections,
use_cached_job=use_cached_job,
resource_params=resource_params,
))
return run_configs
@@ -383,8 +351,7 @@ def workflow_run_config_to_request(trans, run_config, workflow):
use_cached_job=run_config.use_cached_job,
inputs={},
param_map={},
allow_tool_state_corrections=run_config.allow_tool_state_corrections,
resource_params=run_config.resource_params
allow_tool_state_corrections=run_config.allow_tool_state_corrections
)
subworkflow_invocation = workflow_run_config_to_request(
trans,
@@ -406,9 +373,6 @@ def workflow_run_config_to_request(trans, run_config, workflow):
for step_id, content in run_config.inputs.items():
workflow_invocation.add_input(content, step_id)
resource_parameters = run_config.resource_params
for key, value in resource_parameters.items():
add_parameter(key, value, param_types.RESOURCE_PARAMETERS)
add_parameter("copy_inputs_to_history", "true" if run_config.copy_inputs_to_history else "false", param_types.META_PARAMETERS)
add_parameter("use_cached_job", "true" if run_config.use_cached_job else "false", param_types.META_PARAMETERS)
return workflow_invocation
@@ -420,7 +384,6 @@ def workflow_request_to_run_config(work_request_context, workflow_invocation):
replacement_dict = {}
inputs = {}
param_map = {}
resource_params = {}
copy_inputs_to_history = None
use_cached_job = False
for parameter in workflow_invocation.input_parameters:
@@ -433,8 +396,6 @@ def workflow_request_to_run_config(work_request_context, workflow_invocation):
copy_inputs_to_history = (parameter.value == "true")
if parameter.name == 'use_cached_job':
use_cached_job = (parameter.value == 'true')
elif parameter_type == param_types.RESOURCE_PARAMETERS:
resource_params[parameter.name] = parameter.value
for input_association in workflow_invocation.input_datasets:
inputs[input_association.workflow_step_id] = input_association.dataset
for input_association in workflow_invocation.input_dataset_collections:
@@ -450,7 +411,6 @@ def workflow_request_to_run_config(work_request_context, workflow_invocation):
param_map=param_map,
copy_inputs_to_history=copy_inputs_to_history,
use_cached_job=use_cached_job,
resource_params=resource_params,
)
return workflow_run_config
lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py
+1 -118
View File
@@ -16,6 +16,7 @@ from galaxy.util import (
asbool,
download_to_file
)
from galaxy.util.compression_utils import CompressedFile
from galaxy.util.template import fill_template
from tool_shed.galaxy_install.tool_dependencies.env_manager import EnvManager
from tool_shed.util import basic_util, tool_dependency_util
@@ -25,124 +26,6 @@ log = logging.getLogger(__name__)
VIRTUALENV_URL = 'https://pypi.python.org/packages/d4/0c/9840c08189e030873387a73b90ada981885010dd9aea134d6de30cd24cb8/virtualenv-15.1.0.tar.gz'
class CompressedFile(object):
def __init__(self, file_path, mode='r'):
if tarfile.is_tarfile(file_path):
self.file_type = 'tar'
elif zipfile.is_zipfile(file_path) and not file_path.endswith('.jar'):
self.file_type = 'zip'
self.file_name = os.path.splitext(os.path.basename(file_path))[0]
if self.file_name.endswith('.tar'):
self.file_name = os.path.splitext(self.file_name)[0]
self.type = self.file_type
method = 'open_%s' % self.file_type
if hasattr(self, method):
self.archive = getattr(self, method)(file_path, mode)
else:
raise NameError('File type %s specified, no open method found.' % self.file_type)
def extract(self, path):
'''Determine the path to which the archive should be extracted.'''
contents = self.getmembers()
extraction_path = path
common_prefix = ''
if len(contents) == 1:
# The archive contains a single file, return the extraction path.
if self.isfile(contents[0]):
extraction_path = os.path.join(path, self.file_name)
if not os.path.exists(extraction_path):
os.makedirs(extraction_path)
self.archive.extractall(extraction_path)
else:
# Get the common prefix for all the files in the archive. If the common prefix ends with a slash,
# or self.isdir() returns True, the archive contains a single directory with the desired contents.
# Otherwise, it contains multiple files and/or directories at the root of the archive.
common_prefix = os.path.commonprefix([self.getname(item) for item in contents])
if len(common_prefix) >= 1 and not common_prefix.endswith(os.sep) and self.isdir(self.getmember(common_prefix)):
common_prefix += os.sep
if not common_prefix.endswith(os.sep):
common_prefix = ''
extraction_path = os.path.join(path, self.file_name)
if not os.path.exists(extraction_path):
os.makedirs(extraction_path)
self.archive.extractall(extraction_path)
# Since .zip files store unix permissions separately, we need to iterate through the zip file
# and set permissions on extracted members.
if self.file_type == 'zip':
for zipped_file in contents:
filename = self.getname(zipped_file)
absolute_filepath = os.path.join(extraction_path, filename)
external_attributes = self.archive.getinfo(filename).external_attr
# The 2 least significant bytes are irrelevant, the next two contain unix permissions.
unix_permissions = external_attributes >> 16
if unix_permissions != 0:
if os.path.exists(absolute_filepath):
os.chmod(absolute_filepath, unix_permissions)
else:
log.warning("Unable to change permission on extracted file '%s' as it does not exist" % absolute_filepath)
return os.path.abspath(os.path.join(extraction_path, common_prefix))
def getmembers_tar(self):
return self.archive.getmembers()
def getmembers_zip(self):
return self.archive.infolist()
def getname_tar(self, item):
return item.name
def getname_zip(self, item):
return item.filename
def getmember(self, name):
for member in self.getmembers():
if self.getname(member) == name:
return member
def getmembers(self):
return getattr(self, 'getmembers_%s' % self.type)()
def getname(self, member):
return getattr(self, 'getname_%s' % self.type)(member)
def isdir(self, member):
return getattr(self, 'isdir_%s' % self.type)(member)
def isdir_tar(self, member):
return member.isdir()
def isdir_zip(self, member):
if member.filename.endswith(os.sep):
return True
return False
def isfile(self, member):
if not self.isdir(member):
return True
return False
def open_tar(self, filepath, mode):
return tarfile.open(filepath, mode, errorlevel=0)
def open_zip(self, filepath, mode):
return zipfile.ZipFile(filepath, mode)
def zipfile_ok(self, path_to_archive):
"""
This function is a bit pedantic and not functionally necessary. It checks whether there is
no file pointing outside of the extraction, because ZipFile.extractall() has some potential
security holes. See python zipfile documentation for more details.
"""
basename = os.path.realpath(os.path.dirname(path_to_archive))
zip_archive = zipfile.ZipFile(path_to_archive)
for member in zip_archive.namelist():
member_path = os.path.realpath(os.path.join(basename, member))
if not member_path.startswith(basename):
return False
return True
class Download(object):
def url_download(self, install_dir, downloaded_file_name, download_url, extract=True, checksums={}):

Some files were not shown because too many files have changed in this diff Show More