")
.appendTo(parent_div)
.text(tool.get("name"));
tool.get("inputs").each(param => {
@@ -1989,44 +1993,42 @@ var TracksterToolView = Backbone.View.extend({
regions: [region.toJSON()]
};
- var current_track = track;
-
- var // Set name of track to include tool name, parameters, and region used.
- track_name = tool.get("name") + current_track.tool_region_and_parameters_str(region);
+ // Set name of track to include tool name, parameters, and region used.
+ var track_name = tool.get("name") + track.tool_region_and_parameters_str(region);
var container;
// If track not in a group, create a group for it and add new track to group. If track
// already in group, add track to group.
- if (current_track.container === view) {
+ if (track.container === track.view) {
// Create new group.
- var group = new DrawableGroup(view, view, {
+ var group = new DrawableGroup(track.view, track.view, {
name: track.config.get_value("name")
});
// Replace track with group.
- var index = current_track.container.replace_drawable(current_track, group, false);
+ var index = track.container.replace_drawable(track, group, false);
// Update HTML.
// FIXME: this is ugly way to replace a track with a group -- make this easier via
// a Drawable or DrawableCollection function.
- group.container_div.insertBefore(current_track.view.content_div.children()[index]);
- group.add_drawable(current_track);
- current_track.container_div.appendTo(group.content_div);
+ group.container_div.insertBefore(track.view.content_div.children()[index]);
+ group.add_drawable(track);
+ track.container_div.appendTo(group.content_div);
container = group;
} else {
// Use current group.
- container = current_track.container;
+ container = track.container;
}
// Create and init new track.
- var new_track = new current_track.constructor(view, container, {
+ var new_track = new track.constructor(track.view, container, {
name: track_name,
hda_ldda: "hda"
});
new_track.init_for_tool_data();
- new_track.change_mode(current_track.mode);
- new_track.set_filters_manager(current_track.filters_manager.copy(new_track));
+ new_track.change_mode(track.mode);
+ new_track.set_filters_manager(track.filters_manager.copy(new_track));
new_track.update_icons();
container.add_drawable(new_track);
new_track.tiles_div.text("Starting job.");
@@ -2441,12 +2443,17 @@ extend(Track.prototype, Drawable.prototype, {
name: "param_space_viz_icon",
title: _l("Tool parameter space visualization"),
css_class: "arrow-split",
- on_click_fn: function(track) {
- var html = `
Tool:${track.tool.get(
- "name"
- )}
Dataset:${track.config.get_value(
- "name"
- )}
Region(s):
`;
+ on_click_fn: track => {
+ var html = `
+
Tool:${track.tool.get("name")}
+
Dataset:${track.config.get_value("name")}
+
Region(s):
+
+ `;
var cancel_fn = () => {
Galaxy.modal.hide();
@@ -2454,22 +2461,22 @@ extend(Track.prototype, Drawable.prototype, {
};
var ok_fn = () => {
- var regions_to_use = $('select[name="regions"] option:selected').val(),
- regions,
- view_region = new visualization.GenomeRegion({
- chrom: view.chrom,
- start: view.low,
- end: view.high
- }),
- bookmarked_regions = _.map(
- $(".bookmark"),
- elt =>
- new visualization.GenomeRegion({
- from_str: $(elt)
- .children(".position")
- .text()
- })
- );
+ var regions_to_use = $('select[name="regions"] option:selected').val();
+ var regions;
+ var view_region = new visualization.GenomeRegion({
+ chrom: this.view.chrom,
+ start: this.view.low,
+ end: this.view.high
+ });
+ var bookmarked_regions = _.map(
+ $(".bookmark"),
+ elt =>
+ new visualization.GenomeRegion({
+ from_str: $(elt)
+ .children(".position")
+ .text()
+ })
+ );
// Get regions for visualization.
if (regions_to_use === "cur") {
@@ -2493,6 +2500,8 @@ extend(Track.prototype, Drawable.prototype, {
})}`;
};
+ /*
+ * TODO: Re-enable this when functional.
var check_enter_esc = e => {
if ((e.keyCode || e.which) === 27) {
// Escape key
@@ -2502,6 +2511,7 @@ extend(Track.prototype, Drawable.prototype, {
ok_fn();
}
};
+ */
// show dialog
Galaxy.modal.show({
@@ -2732,7 +2742,7 @@ extend(Track.prototype, Drawable.prototype, {
track.container_div.addClass("pending");
track.show_message(DATA_PENDING);
//$("
![]()
").attr("src", image_path + "/yui/rel_interstitial_loading.gif").appendTo(track.tiles_div);
- setTimeout(() => {
+ window.setTimeout(() => {
track.init();
}, track.data_query_wait);
} else if (result === "data" || result.status === "data") {
@@ -3040,7 +3050,7 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
var clear_after = options && options.clear_after;
var low = this.view.low;
var high = this.view.high;
- var range = high - low;
+ //var range = high - low;
var width = this.view.container.width();
var w_scale = this.view.resolution_px_b;
var resolution = 1 / w_scale;
@@ -3049,7 +3059,7 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
if (this.is_overview) {
low = this.view.max_low;
high = this.view.max_high;
- w_scale = width / (view.max_high - view.max_low);
+ w_scale = width / (this.view.max_high - this.view.max_low);
resolution = 1 / w_scale;
}
@@ -3105,17 +3115,16 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
}
// When all tiles are drawn, call post-draw actions.
- var track = this;
$.when.apply($, tile_promises).then(() => {
// Step (c) for (re)moving tiles when clear_after is true:
- track.tiles_div.children(".remove").remove();
+ this.tiles_div.children(".remove").remove();
// Only do postdraw actions for tiles; instances where tiles may not be drawn include:
// (a) ReferenceTrack without sufficient resolution;
// (b) data_fetch = false.
tiles = _.filter(tiles, t => t !== null);
if (tiles.length !== 0) {
- track.postdraw_actions(tiles, width, w_scale, clear_after);
+ this.postdraw_actions(tiles, width, w_scale, clear_after);
}
});
},
@@ -3124,18 +3133,17 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
* Add a maximum/minimum label to track.
*/
_add_yaxis_label: function(type, on_change) {
- var track = this;
var css_class = type === "max" ? "top" : "bottom";
var text = type === "max" ? "max" : "min";
var pref_name = type === "max" ? "max_value" : "min_value";
var label = this.container_div.find(`.yaxislabel.${css_class}`);
- var value = round(track.config.get_value(pref_name), 1);
+ var value = round(this.config.get_value(pref_name), 1);
// Default action for on_change is to redraw track.
on_change =
on_change ||
(() => {
- track.request_draw({ clear_tile_cache: true });
+ this.request_draw({ clear_tile_cache: true });
});
if (label.length !== 0) {
@@ -3149,7 +3157,7 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
num_cols: 12,
on_finish: function(new_val) {
$(".tooltip").remove();
- track.config.set_value(pref_name, round(new_val, 1));
+ this.config.set_value(pref_name, round(new_val, 1));
on_change();
},
help_text: `Set ${text} value`
@@ -3176,18 +3184,17 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
// Clear because this is set when drawing.
this.max_height_px = 0;
- var track = this;
_.each(tiles, tile => {
if (!(tile instanceof LineTrackTile)) {
tile.html_elt.remove();
- track.draw_helper(tile.region, w_scale, {
+ this.draw_helper(tile.region, w_scale, {
force: true,
mode: "Coverage"
});
}
});
- track._add_yaxis_label("max");
+ this._add_yaxis_label("max");
} else {
// -- Drawing in non-Coverage mode. --
@@ -3247,19 +3254,16 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
var mode = options.mode || this.mode;
var resolution = 1 / w_scale;
- var // Useful vars.
- track = this;
-
var drawables = this._get_drawables();
var key = this._gen_tile_cache_key(w_scale, region);
var is_tile = o => o && "track" in o;
// Check tile cache, if found show existing tile in correct position
- var tile = force ? undefined : track.tile_cache.get_elt(key);
+ var tile = force ? undefined : this.tile_cache.get_elt(key);
if (tile) {
if (is_tile(tile)) {
- track.show_tile(tile, w_scale);
+ this.show_tile(tile, w_scale);
}
return tile;
}
@@ -3277,16 +3281,16 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
// Map drawable object to data needed for drawing.
var tile_data = _.map(drawables, (
d // Get the track data/promise.
- ) => d.data_manager.get_data(region, data_mode, resolution, track.data_url_extra_params));
+ ) => d.data_manager.get_data(region, data_mode, resolution, this.data_url_extra_params));
// Get reference data/promise.
- if (view.reference_track) {
+ if (this.view.reference_track) {
tile_data.push(
- view.reference_track.data_manager.get_data(
+ this.view.reference_track.data_manager.get_data(
region,
mode,
resolution,
- view.reference_track.data_url_extra_params
+ this.view.reference_track.data_url_extra_params
)
);
}
@@ -3298,7 +3302,7 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
// When data is available, draw tile.
//
var tile_drawn = $.Deferred();
- track.tile_cache.set_elt(key, tile_drawn);
+ this.tile_cache.set_elt(key, tile_drawn);
$.when.apply($, get_tile_data()).then(() => {
var tile_data = get_tile_data();
var tracks_data = tile_data;
@@ -3309,41 +3313,36 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
// Deferred, try again from the top. NOTE: this condition could (should?) be handled by the
// GenomeDataManager in visualization module.
if (_.find(tile_data, d => util.is_deferred(d))) {
- track.tile_cache.set_elt(key, undefined);
- $.when(track.draw_helper(region, w_scale, options)).then(tile => {
+ this.tile_cache.set_elt(key, undefined);
+ $.when(this.draw_helper(region, w_scale, options)).then(tile => {
tile_drawn.resolve(tile);
});
return;
}
// If sequence data is available, subset to get only data in region.
- if (view.reference_track) {
- seq_data = view.reference_track.data_manager.subset_entry(tile_data.pop(), region);
+ if (this.view.reference_track) {
+ seq_data = this.view.reference_track.data_manager.subset_entry(tile_data.pop(), region);
}
// Get drawing modes, heights for all tracks.
var drawing_modes = [];
-
var drawing_heights = [];
_.each(drawables, (d, i) => {
- var mode = d.mode;
var data = tracks_data[i];
- if (mode === "Auto") {
- mode = d.get_mode(data);
- d.update_auto_mode(mode);
+ if (d.mode === "Auto") {
+ d.mode = d.get_mode(data);
+ d.update_auto_mode(d.mode);
}
- drawing_modes.push(mode);
- drawing_heights.push(d.get_canvas_height(data, mode, w_scale, width));
+ drawing_modes.push(d.mode);
+ drawing_heights.push(d.get_canvas_height(data, d.mode, w_scale, width));
});
- var canvas = track.view.canvas_manager.new_canvas();
+ var canvas = this.view.canvas_manager.new_canvas();
var tile_low = region.get("start");
var tile_high = region.get("end");
- var all_data_index = 0;
-
- var width = Math.ceil((tile_high - tile_low) * w_scale) + track.left_offset;
-
+ var width = Math.ceil((tile_high - tile_low) * w_scale) + this.left_offset;
var height = _.max(drawing_heights);
var tile;
@@ -3354,7 +3353,7 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
// Height is specified in options or is the height found above.
canvas.height = options.height || height;
var ctx = canvas.getContext("2d");
- ctx.translate(track.left_offset, 0);
+ ctx.translate(this.left_offset, 0);
if (drawables.length > 1) {
ctx.globalAlpha = 0.5;
ctx.globalCompositeOperation = "source-over";
@@ -3365,8 +3364,8 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
// Don't cache, show if no tile.
if (tile !== undefined) {
- track.tile_cache.set_elt(key, tile);
- track.show_tile(tile, w_scale);
+ this.tile_cache.set_elt(key, tile);
+ this.show_tile(tile, w_scale);
}
tile_drawn.resolve(tile);
@@ -3428,7 +3427,6 @@ extend(TiledTrack.prototype, Drawable.prototype, Track.prototype, {
* an existing tile rather than reshowing it.
*/
show_tile: function(tile, w_scale) {
- var track = this;
var tile_element = tile.html_elt;
// -- Show/move tile element. --
@@ -3785,7 +3783,7 @@ extend(CompositeTrack.prototype, TiledTrack.prototype, {
}
// Replace track with group.
- var index = this.container.replace_drawable(this, group, true);
+ this.container.replace_drawable(this, group, true);
group.request_draw({ clear_tile_cache: true });
},
@@ -3819,7 +3817,6 @@ extend(CompositeTrack.prototype, TiledTrack.prototype, {
* Update minimum, maximum for component tracks.
*/
update_all_min_max: function() {
- var track = this;
var min_value = this.config.get_value("min_value");
var max_value = this.config.get_value("max_value");
_.each(this.drawables, d => {
@@ -4127,7 +4124,8 @@ extend(DiagonalHeatmapTrack.prototype, Drawable.prototype, TiledTrack.prototype,
hda_ldda: track.dataset.get("hda_ldda")
},
result => {
- var data = result.data;
+ // What does this do? Is it meant to be attached to some higher scope state object?
+ // var data = result.data;
}
);
},
@@ -4163,10 +4161,9 @@ var FeatureTrack = function(view, container, obj_dict) {
this.slotters = {};
this.start_end_dct = {};
this.left_offset = 200;
-
- // this.painter = painters.LinkedFeaturePainter;
this.set_painter_from_config();
};
+
extend(FeatureTrack.prototype, Drawable.prototype, TiledTrack.prototype, {
display_modes: ["Auto", "Coverage", "Dense", "Squish", "Pack"],
@@ -4258,9 +4255,6 @@ extend(FeatureTrack.prototype, Drawable.prototype, TiledTrack.prototype, {
postdraw_actions: function(tiles, width, w_scale, clear_after) {
TiledTrack.prototype.postdraw_actions.call(this, tiles, width, w_scale, clear_after);
- var track = this;
- var i;
-
var line_track_tiles = _.filter(tiles, t => t instanceof LineTrackTile);
//
@@ -4277,7 +4271,6 @@ extend(FeatureTrack.prototype, Drawable.prototype, TiledTrack.prototype, {
});
// Draw incomplete features on each tile.
- var self = this;
_.each(tiles, tile => {
// Remove features already drawn on tile originally.
var tile_incomplete_features = _.omit(
@@ -4296,16 +4289,16 @@ extend(FeatureTrack.prototype, Drawable.prototype, TiledTrack.prototype, {
data: _.values(tile_incomplete_features)
};
- var new_canvas = self.view.canvas_manager.new_canvas();
+ var new_canvas = this.view.canvas_manager.new_canvas();
var new_canvas_ctx = new_canvas.getContext("2d");
new_canvas.height = Math.max(
tile.canvas.height,
- self.get_canvas_height(features, tile.mode, tile.w_scale, 100)
+ this.get_canvas_height(features, tile.mode, tile.w_scale, 100)
);
new_canvas.width = tile.canvas.width;
new_canvas_ctx.drawImage(tile.canvas, 0, 0);
- new_canvas_ctx.translate(track.left_offset, 0);
- var new_tile = self.draw_tile(
+ new_canvas_ctx.translate(this.left_offset, 0);
+ var new_tile = this.draw_tile(
features,
new_canvas_ctx,
tile.mode,
@@ -4348,8 +4341,8 @@ extend(FeatureTrack.prototype, Drawable.prototype, TiledTrack.prototype, {
//
// Update filtering UI.
- if (track.filters_manager) {
- var filters = track.filters_manager.filters;
+ if (this.filters_manager) {
+ var filters = this.filters_manager.filters;
var f;
for (f = 0; f < filters.length; f++) {
filters[f].update_ui_elt();
@@ -4361,7 +4354,7 @@ extend(FeatureTrack.prototype, Drawable.prototype, TiledTrack.prototype, {
var example_feature;
var filter;
- for (i = 0; i < tiles.length; i++) {
+ for (let i = 0; i < tiles.length; i++) {
if (tiles[i].data.length) {
example_feature = tiles[i].data[0];
for (f = 0; f < filters.length; f++) {
@@ -4375,12 +4368,12 @@ extend(FeatureTrack.prototype, Drawable.prototype, TiledTrack.prototype, {
}
// If filter availability changed, hide filter div if necessary and update menu.
- if (track.filters_available !== filters_available) {
- track.filters_available = filters_available;
- if (!track.filters_available) {
- track.filters_manager.hide();
+ if (this.filters_available !== filters_available) {
+ this.filters_available = filters_available;
+ if (!this.filters_available) {
+ this.filters_manager.hide();
}
- track.update_icons();
+ this.update_icons();
}
}
@@ -4389,7 +4382,7 @@ extend(FeatureTrack.prototype, Drawable.prototype, TiledTrack.prototype, {
//
if (tiles[0] instanceof FeatureTrackTile) {
var all_slotted = true;
- for (i = 0; i < tiles.length; i++) {
+ for (let i = 0; i < tiles.length; i++) {
if (!tiles[i].all_slotted) {
all_slotted = false;
break;
@@ -4557,6 +4550,7 @@ extend(FeatureTrack.prototype, Drawable.prototype, TiledTrack.prototype, {
);
var feature_mapper = null;
+ var incomplete_features = null;
ctx.fillStyle = this.config.get_value("block_color");
ctx.font = ctx.canvas.manager.default_font;
diff --git a/client/galaxy/scripts/viz/visualization.js b/client/galaxy/scripts/viz/visualization.js
index 7ba8c775ad3..b3140462fe8 100644
--- a/client/galaxy/scripts/viz/visualization.js
+++ b/client/galaxy/scripts/viz/visualization.js
@@ -5,7 +5,6 @@ import util_mod from "viz/trackster/util";
import config_mod from "utils/config";
import GridView from "mvc/grid/grid-view";
import Tabs from "mvc/ui/ui-tabs";
-import Ui from "mvc/ui/ui-misc";
/**
* Mixin for returning custom JSON representation from toJSON. Class attribute to_json_keys defines a set of attributes
* to include in the representation; to_json_mappers defines mappers for returned objects.
@@ -1109,12 +1108,11 @@ var TrackBrowserRouter = Backbone.Router.extend({
// Can't put regular expression in routes dictionary.
// NOTE: parentheses are used to denote parameters returned to callback.
this.route(/([\w]+)$/, "change_location");
- this.route(/([\w\+]+\:[\d,]+-[\d,]+)$/, "change_location");
+ this.route(/([\w+]+:[\d,]+-[\d,]+)$/, "change_location");
// Handle navigate events from view.
- var self = this;
- self.view.on("navigate", new_loc => {
- self.navigate(new_loc);
+ this.view.on("navigate", new_loc => {
+ this.navigate(new_loc);
});
},
diff --git a/client/galaxy/style/less/ui.less b/client/galaxy/style/less/ui.less
index 4e3d3eaa1bd..1e48cea2a26 100644
--- a/client/galaxy/style/less/ui.less
+++ b/client/galaxy/style/less/ui.less
@@ -486,6 +486,9 @@
.ui-form-composite {
max-width: 900px;
+ height: 100%;
+ flex-direction: column;
+ display: flex;
.ui-form-header {
&:extend(h3);
margin-top: 0px;
@@ -906,10 +909,6 @@
float: right;
width: ~'calc(100% - 76px)';
}
- .ui-gs-token-textbox {
- float: right;
- width: ~'calc(100% - 76px)';
- }
.ui-gs-browse-button {
float: left;
.ui-button-icon {
@@ -917,15 +916,9 @@
margin-right: 5px;
}
}
- .ui-gs-token-label {
- margin-top: 5px;
- margin-left: 32px;
- margin-right: 5px;
- float: left;
- }
- .ui-gs-token-field {
- clear: both;
- padding-top: 5px;
+ .ui-gs-browse-field {
+ height: @input-height-base;
+ line-height: @line-height-base;
}
}
diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample
index a6b87e2c82c..4da482becf8 100644
--- a/config/datatypes_conf.xml.sample
+++ b/config/datatypes_conf.xml.sample
@@ -21,7 +21,7 @@
-
+
@@ -299,7 +299,7 @@
-
+
diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample
index aa412bf3d72..6951fc2762e 100644
--- a/config/galaxy.yml.sample
+++ b/config/galaxy.yml.sample
@@ -812,6 +812,11 @@ galaxy:
# documentation for the corresponding nginx configuration.
#nginx_upload_job_files_path: false
+ # Galaxy can upload user files in chunks without using nginx. Enable
+ # the chunk uploader by specifying a chunk size larger than 0. The
+ # chunk size is specified in bytes (default: 100MB).
+ #chunk_upload_size: 104857600
+
# Have Galaxy manage dynamic proxy component for routing requests to
# other services based on Galaxy's session cookie. It will attempt to
# do this by default though you do need to install node+npm and do an
@@ -1167,6 +1172,15 @@ galaxy:
# Set maximum size of ngrams
#tool_ngram_maxsize: 4
+ # Set tool test data directory. The test framework sets this value to
+ # 'test-data,https://github.com/galaxyproject/galaxy-test-data.git'
+ # which will cause Galaxy to clone down extra test data on the fly for
+ # certain tools distributed with Galaxy but this is likely not
+ # appropriate for production systems. Instead one can simply clone
+ # that repository directly and specify a path here instead of a Git
+ # HTTP repository.
+ #tool_test_data_directories: test-data
+
# Galaxy encodes various internal values when these values will be
# output in some format (for example, in a URL or cookie). You should
# set a key to be used by the algorithm that encodes and decodes these
@@ -1621,23 +1635,6 @@ galaxy:
# processors, memory and walltime.
#job_resource_params_file: config/job_resource_params_conf.xml
- # Similar to the above parameter, workflows can describe parameters
- # used to influence scheduling of jobs within the workflow. This
- # requires both a description of the fields available (which defaults
- # to the definitions in job_resource_params_file if not set).
- #workflow_resource_params_file: config/workflow_resource_params_conf.xml
-
- # This parameter describes how to map users and workflows to a set of
- # workflow resource parameter to present (typically input IDs from
- # workflow_resource_params_file). If this this is a function reference
- # it will be passed various inputs (workflow model object and user)
- # and it should produce a list of input IDs. If it is a path it is
- # expected to an XML or YAML file describing how to map group names to
- # parameter descriptions (additional types of mappings via these files
- # could be implemented but haven't yet - for instance using workflow
- # tags to do the mapping).
- #workflow_resource_params_mapper: config/workflow_resource_mapper_conf.yml
-
# If using job concurrency limits (configured in job_config_file),
# several extra database queries must be performed to determine the
# number of jobs a user has dispatched to a given destination. By
diff --git a/config/job_metrics_conf.xml.sample b/config/job_metrics_conf.xml.sample
index 1fdfb3849a9..5824cb4b39c 100644
--- a/config/job_metrics_conf.xml.sample
+++ b/config/job_metrics_conf.xml.sample
@@ -15,7 +15,7 @@
-
+
-
+
+
+
+
#
#
#
- dataset_collectors = map(dataset_collector, output_collection_def.dataset_collector_descriptions)
- output_name = output_collection_def.name
- filenames = self.find_files(output_name, collection, dataset_collectors)
+ if name is None:
+ name = "unnamed output"
element_datasets = []
for filename, discovered_file in filenames.items():
@@ -241,6 +406,8 @@ class JobContext(object):
# Create new primary dataset
name = fields_match.name or designation
+ link_data = discovered_file.match.link_data
+
dataset = self.create_dataset(
ext=ext,
designation=designation,
@@ -248,14 +415,15 @@ class JobContext(object):
dbkey=dbkey,
name=name,
filename=filename,
- metadata_source_name=output_collection_def.metadata_source,
+ metadata_source_name=metadata_source_name,
+ link_data=link_data,
)
log.debug(
"(%s) Created dynamic collection dataset for path [%s] with element identifier [%s] for output [%s] %s",
self.job.id,
filename,
designation,
- output_collection_def.name,
+ name,
create_dataset_timer,
)
element_datasets.append((element_identifiers, dataset))
@@ -270,7 +438,7 @@ class JobContext(object):
log.debug(
"(%s) Add dynamic collection datsets to history for output [%s] %s",
self.job.id,
- output_collection_def.name,
+ name,
add_datasets_timer,
)
@@ -300,12 +468,24 @@ class JobContext(object):
dbkey,
name,
filename,
- metadata_source_name,
+ metadata_source_name=None,
+ info=None,
+ library_folder=None,
+ link_data=False,
+ primary_data=None,
):
app = self.app
sa_session = self.sa_session
- primary_data = _new_hda(app, sa_session, ext, designation, visible, dbkey, self.permissions)
+ if primary_data is None:
+ if not library_folder:
+ primary_data = _new_hda(app, sa_session, ext, designation, visible, dbkey, self.permissions)
+ else:
+ primary_data = _new_ldda(self.work_context, name, ext, visible, dbkey, library_folder)
+ else:
+ primary_data.extension = ext
+ primary_data.visible = visible
+ primary_data.dbkey = dbkey
# Copy metadata from one of the inputs if requested.
metadata_source = None
@@ -314,7 +494,11 @@ class JobContext(object):
sa_session.flush()
# Move data from temp location to dataset location
- app.object_store.update_from_file(primary_data.dataset, file_name=filename, create=True)
+ if not link_data:
+ app.object_store.update_from_file(primary_data.dataset, file_name=filename, create=True)
+ else:
+ primary_data.link_to(filename)
+
primary_data.set_size()
# If match specified a name use otherwise generate one from
# designation.
@@ -325,6 +509,9 @@ class JobContext(object):
else:
primary_data.init_meta()
+ if info is not None:
+ primary_data.info = info
+
primary_data.set_meta()
primary_data.set_peek()
@@ -491,6 +678,20 @@ def discover_files(output_name, tool_provided_metadata, extra_file_collectors, j
yield DiscoveredFile(match.path, collector, match)
+def discovered_file_for_unnamed_output(dataset, job_working_directory, parent_identifiers=[]):
+ extra_file_collector = DEFAULT_TOOL_PROVIDED_DATASET_COLLECTOR
+ target_directory = discover_target_directory(extra_file_collector.directory, job_working_directory)
+ filename = dataset["filename"]
+ # handle link_data_only here, verify filename is in directory if not linking...
+ if not dataset.get("link_data_only"):
+ path = os.path.join(target_directory, filename)
+ if not util.in_directory(path, target_directory):
+ raise Exception("Problem with tool configuration, attempting to pull in datasets from outside working directory.")
+ else:
+ path = filename
+ return DiscoveredFile(path, extra_file_collector, JsonCollectedDatasetMatch(dataset, extra_file_collector, filename, path=path, parent_identifiers=parent_identifiers))
+
+
def discover_target_directory(dir_name, job_working_directory):
if dir_name:
directory = os.path.join(job_working_directory, dir_name)
@@ -605,11 +806,12 @@ def _compose(f, g):
class JsonCollectedDatasetMatch(object):
- def __init__(self, as_dict, collector, filename, path=None):
+ def __init__(self, as_dict, collector, filename, path=None, parent_identifiers=[]):
self.as_dict = as_dict
self.collector = collector
self.filename = filename
self.path = path
+ self._parent_identifiers = parent_identifiers
@property
def designation(self):
@@ -627,7 +829,7 @@ class JsonCollectedDatasetMatch(object):
@property
def element_identifiers(self):
- return self.raw_element_identifiers or [self.designation]
+ return self._parent_identifiers + (self.raw_element_identifiers or [self.designation])
@property
def raw_element_identifiers(self):
@@ -664,6 +866,14 @@ class JsonCollectedDatasetMatch(object):
except KeyError:
return self.collector.default_visible
+ @property
+ def link_data(self):
+ return bool(self.as_dict.get("link_data_only", False))
+
+ @property
+ def object_id(self):
+ return self.as_dict.get("object_id", None)
+
class RegexCollectedDatasetMatch(JsonCollectedDatasetMatch):
@@ -676,6 +886,42 @@ class RegexCollectedDatasetMatch(JsonCollectedDatasetMatch):
UNSET = object()
+def _new_ldda(
+ trans,
+ name,
+ ext,
+ visible,
+ dbkey,
+ library_folder,
+):
+ ld = trans.app.model.LibraryDataset(folder=library_folder, name=name)
+ trans.sa_session.add(ld)
+ trans.sa_session.flush()
+ trans.app.security_agent.copy_library_permissions(trans, library_folder, ld)
+
+ ldda = trans.app.model.LibraryDatasetDatasetAssociation(name=name,
+ extension=ext,
+ dbkey=dbkey,
+ library_dataset=ld,
+ user=trans.user,
+ create_dataset=True,
+ sa_session=trans.sa_session)
+ trans.sa_session.add(ldda)
+ ldda.state = ldda.states.OK
+ # Permissions must be the same on the LibraryDatasetDatasetAssociation and the associated LibraryDataset
+ trans.app.security_agent.copy_library_permissions(trans, ld, ldda)
+ # Copy the current user's DefaultUserPermissions to the new LibraryDatasetDatasetAssociation.dataset
+ trans.app.security_agent.set_all_dataset_permissions(ldda.dataset, trans.app.security_agent.user_get_default_permissions(trans.user))
+ library_folder.add_library_dataset(ld, genome_build=dbkey)
+ trans.sa_session.add(library_folder)
+ trans.sa_session.flush()
+
+ ld.library_dataset_dataset_association_id = ldda.id
+ trans.sa_session.add(ld)
+ trans.sa_session.flush()
+ return ldda
+
+
def _new_hda(
app,
sa_session,
@@ -702,3 +948,4 @@ def _new_hda(
DEFAULT_DATASET_COLLECTOR = DatasetCollector(DEFAULT_DATASET_COLLECTOR_DESCRIPTION)
+DEFAULT_TOOL_PROVIDED_DATASET_COLLECTOR = ToolMetadataDatasetCollector(ToolProvidedMetadataDatasetCollection())
diff --git a/lib/galaxy/tools/parser/interface.py b/lib/galaxy/tools/parser/interface.py
index 626a8f2114c..0ca8f33187e 100644
--- a/lib/galaxy/tools/parser/interface.py
+++ b/lib/galaxy/tools/parser/interface.py
@@ -357,26 +357,77 @@ class ToolStdioExitCode(object):
class TestCollectionDef(object):
- # TODO: do not require XML directly here.
- def __init__(self, elem, parse_param_elem):
- self.elements = []
+ def __init__(self, attrib, name, collection_type, elements):
+ self.attrib = attrib
+ self.collection_type = collection_type
+ self.elements = elements
+ self.name = name
+
+ @staticmethod
+ def from_xml(elem, parse_param_elem):
+ elements = []
attrib = dict(elem.attrib)
- self.collection_type = attrib["type"]
- self.name = attrib.get("name", "Unnamed Collection")
+ collection_type = attrib["type"]
+ name = attrib.get("name", "Unnamed Collection")
for element in elem.findall("element"):
element_attrib = dict(element.attrib)
element_identifier = element_attrib["name"]
nested_collection_elem = element.find("collection")
if nested_collection_elem is not None:
- self.elements.append((element_identifier, TestCollectionDef(nested_collection_elem, parse_param_elem)))
+ element_definition = TestCollectionDef.from_xml(nested_collection_elem, parse_param_elem)
else:
- self.elements.append((element_identifier, parse_param_elem(element)))
+ element_definition = parse_param_elem(element)
+ elements.append({"element_identifier": element_identifier, "element_definition": element_definition})
+
+ return TestCollectionDef(
+ attrib=attrib,
+ collection_type=collection_type,
+ elements=elements,
+ name=name,
+ )
+
+ def to_dict(self):
+ def element_to_dict(element_dict):
+ element_identifier, element_def = element_dict["element_identifier"], element_dict["element_definition"]
+ if isinstance(element_def, TestCollectionDef):
+ element_def = element_def.to_dict()
+ return {
+ "element_identifier": element_identifier,
+ "element_definition": element_def,
+ }
+
+ return {
+ "model_class": "TestCollectionDef",
+ "attributes": self.attrib,
+ "collection_type": self.collection_type,
+ "elements": map(element_to_dict, self.elements or []),
+ "name": self.name,
+ }
+
+ @staticmethod
+ def from_dict(as_dict):
+ assert as_dict["model_class"] == "TestCollectionDef"
+
+ def element_from_dict(element_dict):
+ if "element_definition" not in element_dict:
+ raise Exception("Invalid element_dict %s" % element_dict)
+ element_def = element_dict["element_definition"]
+ if element_def.get("model_class", None) == "TestCollectionDef":
+ element_def = TestCollectionDef.from_dict(element_def)
+ return {"element_identifier": element_dict["element_identifier"], "element_definition": element_def}
+
+ return TestCollectionDef(
+ attrib=as_dict["attributes"],
+ name=as_dict["name"],
+ elements=list(map(element_from_dict, as_dict["elements"] or [])),
+ collection_type=as_dict["collection_type"],
+ )
def collect_inputs(self):
inputs = []
for element in self.elements:
- value = element[1]
+ value = element["element_definition"]
if isinstance(value, TestCollectionDef):
inputs.extend(value.collect_inputs())
else:
@@ -393,3 +444,18 @@ class TestCollectionOutputDef(object):
self.count = int(count) if count is not None else None
self.attrib = attrib
self.element_tests = element_tests
+
+ @staticmethod
+ def from_dict(as_dict):
+ return TestCollectionOutputDef(
+ name=as_dict["name"],
+ attrib=as_dict["attributes"],
+ element_tests=as_dict["element_tests"],
+ )
+
+ def to_dict(self):
+ return dict(
+ name=self.name,
+ attributes=self.attrib,
+ element_tests=self.element_tests
+ )
diff --git a/lib/galaxy/tools/parser/xml.py b/lib/galaxy/tools/parser/xml.py
index 6edce860323..fde2b532935 100644
--- a/lib/galaxy/tools/parser/xml.py
+++ b/lib/galaxy/tools/parser/xml.py
@@ -391,8 +391,6 @@ class XmlToolSource(ToolSource):
for i, test_elem in enumerate(tests_elem.findall("test")):
tests.append(_test_elem_to_dict(test_elem, i))
- _copy_to_dict_if_present(tests_elem, rval, ["interactor"])
-
return rval
def parse_profile(self):
@@ -418,7 +416,7 @@ def _test_elem_to_dict(test_elem, i):
expect_failure=string_as_bool(test_elem.get("expect_failure", False)),
maxseconds=test_elem.get("maxseconds", None),
)
- _copy_to_dict_if_present(test_elem, rval, ["interactor", "num_outputs"])
+ _copy_to_dict_if_present(test_elem, rval, ["num_outputs"])
return rval
@@ -431,7 +429,7 @@ def __parse_output_elems(test_elem):
outputs = []
for output_elem in test_elem.findall("output"):
name, file, attributes = __parse_output_elem(output_elem)
- outputs.append((name, file, attributes))
+ outputs.append({"name": name, "value": file, "attributes": attributes})
return outputs
@@ -464,7 +462,7 @@ def __parse_output_collection_elem(output_collection_elem):
if name is None:
raise Exception("Test output collection does not have a 'name'")
element_tests = __parse_element_tests(output_collection_elem)
- return TestCollectionOutputDef(name, attrib, element_tests)
+ return TestCollectionOutputDef(name, attrib, element_tests).to_dict()
def __parse_element_tests(parent_element):
@@ -565,7 +563,12 @@ def __parse_extra_files_elem(extra):
assert extra_type == 'directory' or extra_name is not None, \
'extra_files type (%s) requires a name attribute' % extra_type
extra_value, extra_attributes = __parse_test_attributes(extra, attrib)
- return extra_type, extra_value, extra_name, extra_attributes
+ return {
+ "value": extra_value,
+ "name": extra_name,
+ "type": extra_type,
+ "attributes": extra_attributes
+ }
def __expand_input_elems(root_elem, prefix=""):
@@ -628,8 +631,8 @@ def _copy_to_dict_if_present(elem, rval, attributes):
def __parse_inputs_elems(test_elem, i):
raw_inputs = []
for param_elem in test_elem.findall("param"):
- name, value, attrib = __parse_param_elem(param_elem, i)
- raw_inputs.append((name, value, attrib))
+ raw_inputs.append(__parse_param_elem(param_elem, i))
+
return raw_inputs
@@ -641,40 +644,43 @@ def __parse_param_elem(param_elem, i=0):
value = attrib['value']
else:
value = None
- attrib['children'] = param_elem
- if attrib['children'] is not None:
+ children_elem = param_elem
+ if children_elem is not None:
# At this time, we can assume having children only
# occurs on DataToolParameter test items but this could
# change and would cause the below parsing to change
# based upon differences in children items
- attrib['metadata'] = []
+ attrib['metadata'] = {}
attrib['composite_data'] = []
attrib['edit_attributes'] = []
# Composite datasets need to be renamed uniquely
composite_data_name = None
- for child in attrib['children']:
+ for child in children_elem:
if child.tag == 'composite_data':
- attrib['composite_data'].append(child)
+ file_name = child.get("value")
+ attrib['composite_data'].append(file_name)
if composite_data_name is None:
# Generate a unique name; each test uses a
# fresh history.
composite_data_name = '_COMPOSITE_RENAMED_t%d_%s' \
% (i, uuid.uuid1().hex)
elif child.tag == 'metadata':
- attrib['metadata'].append(child)
- elif child.tag == 'metadata':
- attrib['metadata'].append(child)
+ attrib['metadata'][child.get("name")] = child.get("value")
elif child.tag == 'edit_attributes':
attrib['edit_attributes'].append(child)
elif child.tag == 'collection':
- attrib['collection'] = TestCollectionDef(child, __parse_param_elem)
+ attrib['collection'] = TestCollectionDef.from_xml(child, __parse_param_elem)
if composite_data_name:
# Composite datasets need implicit renaming;
# inserted at front of list so explicit declarations
# take precedence
attrib['edit_attributes'].insert(0, {'type': 'name', 'value': composite_data_name})
name = attrib.pop('name')
- return (name, value, attrib)
+ return {
+ "name": name,
+ "value": value,
+ "attributes": attrib
+ }
class StdioParser(object):
diff --git a/lib/galaxy/tools/parser/yaml.py b/lib/galaxy/tools/parser/yaml.py
index 71a593e3cf6..4a8d796e273 100644
--- a/lib/galaxy/tools/parser/yaml.py
+++ b/lib/galaxy/tools/parser/yaml.py
@@ -188,7 +188,7 @@ def _parse_test(i, test_dict):
if _is_dict(inputs):
new_inputs = []
for key, value in inputs.items():
- new_inputs.append((key, value, {}))
+ new_inputs.append({"name": key, "value": value, "attributes": {}})
test_dict["inputs"] = new_inputs
outputs = test_dict["outputs"]
@@ -202,7 +202,11 @@ def _parse_test(i, test_dict):
else:
file = value
attributes = {}
- new_outputs.append((key, file, attributes))
+ new_outputs.append({
+ "name": key,
+ "value": file,
+ "attributes": attributes
+ })
else:
for output in outputs:
name = output["name"]
@@ -211,7 +215,7 @@ def _parse_test(i, test_dict):
new_outputs.append((name, value, attributes))
for output in new_outputs:
- attributes = output[2]
+ attributes = output["attributes"]
defaults = {
'compare': 'diff',
'lines_diff': 0,
diff --git a/lib/galaxy/tools/special_tools.py b/lib/galaxy/tools/special_tools.py
index 953e69dee64..129b7064a94 100644
--- a/lib/galaxy/tools/special_tools.py
+++ b/lib/galaxy/tools/special_tools.py
@@ -4,6 +4,7 @@ log = logging.getLogger(__name__)
SPECIAL_TOOLS = {
"history export": "galaxy/tools/imp_exp/exp_history_to_archive.xml",
"history import": "galaxy/tools/imp_exp/imp_history_from_archive.xml",
+ "data fetch": "galaxy/tools/data_fetch.xml",
}
diff --git a/lib/galaxy/tools/test.py b/lib/galaxy/tools/test.py
index 82ab9061dee..a26f26fea9d 100644
--- a/lib/galaxy/tools/test.py
+++ b/lib/galaxy/tools/test.py
@@ -6,6 +6,7 @@ from six import string_types
import galaxy.tools.parameters.basic
import galaxy.tools.parameters.grouping
+from galaxy.tools.verify.interactor import ToolTestDescription
from galaxy.util import string_as_bool
try:
@@ -16,11 +17,6 @@ except ImportError:
log = logging.getLogger(__name__)
-DEFAULT_FTYPE = 'auto'
-DEFAULT_DBKEY = 'hg17'
-DEFAULT_INTERACTOR = "api" # Default mechanism test code uses for interacting with Galaxy instance.
-DEFAULT_MAX_SECS = None
-
@nottest
def parse_tests(tool, tests_source):
@@ -28,207 +24,128 @@ def parse_tests(tool, tests_source):
Build ToolTestBuilder objects for each "" elements and
return default interactor (if any).
"""
- default_interactor = os.environ.get('GALAXY_TEST_DEFAULT_INTERACTOR', DEFAULT_INTERACTOR)
- tests_dict = tests_source.parse_tests_to_dict()
- tests_default_interactor = tests_dict.get('interactor', default_interactor)
+ raw_tests_dict = tests_source.parse_tests_to_dict()
tests = []
- for i, test_dict in enumerate(tests_dict.get('tests', [])):
- test = ToolTestBuilder(tool, test_dict, i, default_interactor=tests_default_interactor)
+ for i, raw_test_dict in enumerate(raw_tests_dict.get('tests', [])):
+ test = description_from_tool_object(tool, i, raw_test_dict)
tests.append(test)
return tests
-class ToolTestBuilder(object):
+def description_from_tool_object(tool, test_index, raw_test_dict):
+ required_files = []
+
+ num_outputs = raw_test_dict.get('expect_num_outputs', None)
+ if num_outputs:
+ num_outputs = int(num_outputs)
+
+ try:
+ processed_inputs = _process_raw_inputs(tool, tool.inputs, raw_test_dict["inputs"], required_files)
+ processed_test_dict = {
+ "inputs": processed_inputs,
+ "outputs": raw_test_dict["outputs"],
+ "output_collections": raw_test_dict["output_collections"],
+ "num_outputs": num_outputs,
+ "command_line": raw_test_dict.get("command", None),
+ "stdout": raw_test_dict.get("stdout", None),
+ "stderr": raw_test_dict.get("stderr", None),
+ "expect_exit_code": raw_test_dict.get("expect_exit_code", None),
+ "expect_failure": raw_test_dict.get("expect_failure", False),
+ "required_files": required_files,
+ "tool_id": tool.id,
+ "test_index": test_index,
+ "error": False,
+ }
+ except Exception as e:
+ log.exception("Failed to load tool test number [%d] for %s" % (test_index, tool.id))
+ processed_test_dict = {
+ "tool_id": tool.id,
+ "test_index": test_index,
+ "inputs": {},
+ "error": True,
+ "exception": str(e),
+ }
+
+ return ToolTestDescription(processed_test_dict)
+
+
+def _process_raw_inputs(tool, tool_inputs, raw_inputs, required_files, parent_context=None):
"""
- Encapsulates information about a tool test, and allows creation of a
- dynamic TestCase class (the unittest framework is very class oriented,
- doing dynamic tests in this way allows better integration)
+ Recursively expand flat list of inputs into "tree" form of flat list
+ (| using to nest to new levels) structure and expand dataset
+ information as proceeding to populate self.required_files.
"""
-
- def __init__(self, tool, test_dict, i, default_interactor):
- name = test_dict.get('name', 'Test-%d' % (i + 1))
- maxseconds = test_dict.get('maxseconds', DEFAULT_MAX_SECS)
- if maxseconds is not None:
- maxseconds = int(maxseconds)
-
- self.tool = tool
- self.name = name
- self.maxseconds = maxseconds
- self.required_files = []
- self.inputs = {}
- self.outputs = []
- # By default do not making assertions on number of outputs - but to
- # test filtering allow explicitly state number of outputs.
- self.num_outputs = None
- self.error = False
- self.exception = None
-
- self.__handle_test_dict(test_dict, i, default_interactor)
-
- def test_data(self):
- """
- Iterator over metadata representing the required files for upload.
- """
- return test_data_iter(self.required_files)
-
- def __matching_case_for_value(self, cond, declared_value):
- test_param = cond.test_param
- if isinstance(test_param, galaxy.tools.parameters.basic.BooleanToolParameter):
- if declared_value is None:
- # No explicit value for param in test case, determine from default
- query_value = test_param.checked
- else:
- query_value = _process_bool_param_value(test_param, declared_value)
-
- def matches_declared_value(case_value):
- return _process_bool_param_value(test_param, case_value) == query_value
- elif isinstance(test_param, galaxy.tools.parameters.basic.SelectToolParameter):
- if declared_value is not None:
- # Test case supplied explicit value to check against.
-
- def matches_declared_value(case_value):
- return case_value == declared_value
- elif test_param.static_options:
- # No explicit value in test case, not much to do if options are dynamic but
- # if static options are available can find the one specified as default or
- # fallback on top most option (like GUI).
- for (name, value, selected) in test_param.static_options:
- if selected:
- default_option = name
- else:
- first_option = test_param.static_options[0]
- first_option_value = first_option[1]
- default_option = first_option_value
-
- def matches_declared_value(case_value):
- return case_value == default_option
- else:
- # No explicit value for this param and cannot determine a
- # default - give up. Previously this would just result in a key
- # error exception.
- msg = "Failed to find test parameter value specification required for conditional %s" % cond.name
- raise Exception(msg)
-
- # Check the tool's defined cases against predicate to determine
- # selected or default.
- for i, case in enumerate(cond.cases):
- if matches_declared_value(case.value):
- return case
- else:
- msg_template = "%s - Failed to find case matching value (%s) for test parameter specification for conditional %s. Remainder of test behavior is unspecified."
- msg = msg_template % (self.tool.id, declared_value, cond.name)
- log.info(msg)
-
- def __split_if_str(self, value):
- split = isinstance(value, string_types)
- if split:
- value = value.split(",")
- return value
-
- def __handle_test_dict(self, test_dict, i, default_interactor):
- try:
- # Mechanism test code uses for interacting with Galaxy instance,
- # until 'api' is the default switch this to API to use its new
- # features. Once 'api' is the default set to 'twill' to use legacy
- # features or workarounds.
- self.interactor = test_dict.get('interactor', default_interactor)
-
- self.inputs = self.__process_raw_inputs(self.tool.inputs, test_dict["inputs"])
- self.outputs = test_dict["outputs"]
- self.output_collections = test_dict["output_collections"]
- num_outputs = test_dict.get('expect_num_outputs', None)
- if num_outputs:
- num_outputs = int(num_outputs)
- self.num_outputs = num_outputs
- self.command_line = test_dict.get("command", None)
- self.stdout = test_dict.get("stdout", None)
- self.stderr = test_dict.get("stderr", None)
- self.expect_exit_code = test_dict.get("expect_exit_code", None)
- self.expect_failure = test_dict.get("expect_failure", False)
- self.md5 = test_dict.get("md5", None)
- except Exception as e:
- self.inputs = {}
- self.error = True
- self.exception = e
-
- def __process_raw_inputs(self, tool_inputs, raw_inputs, parent_context=None):
- """
- Recursively expand flat list of inputs into "tree" form of flat list
- (| using to nest to new levels) structure and expand dataset
- information as proceeding to populate self.required_files.
- """
- parent_context = parent_context or RootParamContext()
- expanded_inputs = {}
- for key, value in tool_inputs.items():
- if isinstance(value, galaxy.tools.parameters.grouping.Conditional):
- cond_context = ParamContext(name=value.name, parent_context=parent_context)
- case_context = ParamContext(name=value.test_param.name, parent_context=cond_context)
- raw_input = case_context.extract_value(raw_inputs)
- case_value = raw_input[1] if raw_input else None
- case = self.__matching_case_for_value(value, case_value)
- if case:
- for input_name, input_value in case.inputs.items():
- case_inputs = self.__process_raw_inputs({input_name: input_value}, raw_inputs, parent_context=cond_context)
- expanded_inputs.update(case_inputs)
- if not value.type == "text":
- expanded_case_value = self.__split_if_str(case.value)
- if case_value is not None:
- # A bit tricky here - we are growing inputs with value
- # that may be implicit (i.e. not defined by user just
- # a default defined in tool). So we do not want to grow
- # expanded_inputs and risk repeat block viewing this
- # as a new instance with value defined and hence enter
- # an infinite loop - hence the "case_value is not None"
- # check.
- processed_value = _process_simple_value(value.test_param, expanded_case_value)
- expanded_inputs[case_context.for_state()] = processed_value
- elif isinstance(value, galaxy.tools.parameters.grouping.Section):
- context = ParamContext(name=value.name, parent_context=parent_context)
+ parent_context = parent_context or RootParamContext()
+ expanded_inputs = {}
+ for key, value in tool_inputs.items():
+ if isinstance(value, galaxy.tools.parameters.grouping.Conditional):
+ cond_context = ParamContext(name=value.name, parent_context=parent_context)
+ case_context = ParamContext(name=value.test_param.name, parent_context=cond_context)
+ raw_input_dict = case_context.extract_value(raw_inputs)
+ case_value = raw_input_dict["value"] if raw_input_dict else None
+ case = _matching_case_for_value(tool, value, case_value)
+ if case:
+ for input_name, input_value in case.inputs.items():
+ case_inputs = _process_raw_inputs(tool, {input_name: input_value}, raw_inputs, required_files, parent_context=cond_context)
+ expanded_inputs.update(case_inputs)
+ if not value.type == "text":
+ expanded_case_value = _split_if_str(case.value)
+ if case_value is not None:
+ # A bit tricky here - we are growing inputs with value
+ # that may be implicit (i.e. not defined by user just
+ # a default defined in tool). So we do not want to grow
+ # expanded_inputs and risk repeat block viewing this
+ # as a new instance with value defined and hence enter
+ # an infinite loop - hence the "case_value is not None"
+ # check.
+ processed_value = _process_simple_value(value.test_param, expanded_case_value)
+ expanded_inputs[case_context.for_state()] = processed_value
+ elif isinstance(value, galaxy.tools.parameters.grouping.Section):
+ context = ParamContext(name=value.name, parent_context=parent_context)
+ for r_name, r_value in value.inputs.items():
+ expanded_input = _process_raw_inputs(tool, {context.for_state(): r_value}, raw_inputs, required_files, parent_context=context)
+ if expanded_input:
+ expanded_inputs.update(expanded_input)
+ elif isinstance(value, galaxy.tools.parameters.grouping.Repeat):
+ repeat_index = 0
+ while True:
+ context = ParamContext(name=value.name, index=repeat_index, parent_context=parent_context)
+ updated = False
for r_name, r_value in value.inputs.items():
- expanded_input = self.__process_raw_inputs({context.for_state(): r_value}, raw_inputs, parent_context=context)
+ expanded_input = _process_raw_inputs(tool, {context.for_state(): r_value}, raw_inputs, required_files, parent_context=context)
if expanded_input:
expanded_inputs.update(expanded_input)
- elif isinstance(value, galaxy.tools.parameters.grouping.Repeat):
- repeat_index = 0
- while True:
- context = ParamContext(name=value.name, index=repeat_index, parent_context=parent_context)
- updated = False
- for r_name, r_value in value.inputs.items():
- expanded_input = self.__process_raw_inputs({context.for_state(): r_value}, raw_inputs, parent_context=context)
- if expanded_input:
- expanded_inputs.update(expanded_input)
- updated = True
- if not updated:
- break
- repeat_index += 1
- else:
- context = ParamContext(name=value.name, parent_context=parent_context)
- raw_input = context.extract_value(raw_inputs)
- if raw_input:
- (name, param_value, param_extra) = raw_input
- if not value.type == "text":
- param_value = self.__split_if_str(param_value)
- if isinstance(value, galaxy.tools.parameters.basic.DataToolParameter):
- if not isinstance(param_value, list):
- param_value = [param_value]
- map(lambda v: self.__add_uploaded_dataset(context.for_state(), v, param_extra, value), param_value)
- processed_value = param_value
- elif isinstance(value, galaxy.tools.parameters.basic.DataCollectionToolParameter):
- assert 'collection' in param_extra
- collection_def = param_extra['collection']
- for (name, value, extra) in collection_def.collect_inputs():
- require_file(name, value, extra, self.required_files)
- processed_value = collection_def
- else:
- processed_value = _process_simple_value(value, param_value)
- expanded_inputs[context.for_state()] = processed_value
- return expanded_inputs
-
- def __add_uploaded_dataset(self, name, value, extra, input_parameter):
- if value is None:
- assert input_parameter.optional, '%s is not optional. You must provide a valid filename.' % name
- return value
- return require_file(name, value, extra, self.required_files)
+ updated = True
+ if not updated:
+ break
+ repeat_index += 1
+ else:
+ context = ParamContext(name=value.name, parent_context=parent_context)
+ raw_input_dict = context.extract_value(raw_inputs)
+ if raw_input_dict:
+ name = raw_input_dict["name"]
+ param_value = raw_input_dict["value"]
+ param_extra = raw_input_dict["attributes"]
+ if not value.type == "text":
+ param_value = _split_if_str(param_value)
+ if isinstance(value, galaxy.tools.parameters.basic.DataToolParameter):
+ if not isinstance(param_value, list):
+ param_value = [param_value]
+ map(lambda v: _add_uploaded_dataset(context.for_state(), v, param_extra, value, required_files), param_value)
+ processed_value = param_value
+ elif isinstance(value, galaxy.tools.parameters.basic.DataCollectionToolParameter):
+ assert 'collection' in param_extra
+ collection_def = param_extra['collection']
+ for input_dict in collection_def.collect_inputs():
+ name = input_dict["name"]
+ value = input_dict["value"]
+ attributes = input_dict["attributes"]
+ require_file(name, value, attributes, required_files)
+ processed_value = collection_def
+ else:
+ processed_value = _process_simple_value(value, param_value)
+ expanded_inputs[context.for_state()] = processed_value
+ return expanded_inputs
def _process_simple_value(param, param_value):
@@ -266,6 +183,69 @@ def _process_simple_value(param, param_value):
return processed_value
+def _matching_case_for_value(tool, cond, declared_value):
+ test_param = cond.test_param
+ if isinstance(test_param, galaxy.tools.parameters.basic.BooleanToolParameter):
+ if declared_value is None:
+ # No explicit value for param in test case, determine from default
+ query_value = test_param.checked
+ else:
+ query_value = _process_bool_param_value(test_param, declared_value)
+
+ def matches_declared_value(case_value):
+ return _process_bool_param_value(test_param, case_value) == query_value
+ elif isinstance(test_param, galaxy.tools.parameters.basic.SelectToolParameter):
+ if declared_value is not None:
+ # Test case supplied explicit value to check against.
+
+ def matches_declared_value(case_value):
+ return case_value == declared_value
+ elif test_param.static_options:
+ # No explicit value in test case, not much to do if options are dynamic but
+ # if static options are available can find the one specified as default or
+ # fallback on top most option (like GUI).
+ for (name, value, selected) in test_param.static_options:
+ if selected:
+ default_option = name
+ else:
+ first_option = test_param.static_options[0]
+ first_option_value = first_option[1]
+ default_option = first_option_value
+
+ def matches_declared_value(case_value):
+ return case_value == default_option
+ else:
+ # No explicit value for this param and cannot determine a
+ # default - give up. Previously this would just result in a key
+ # error exception.
+ msg = "Failed to find test parameter value specification required for conditional %s" % cond.name
+ raise Exception(msg)
+
+ # Check the tool's defined cases against predicate to determine
+ # selected or default.
+ for i, case in enumerate(cond.cases):
+ if matches_declared_value(case.value):
+ return case
+ else:
+ msg_template = "%s - Failed to find case matching value (%s) for test parameter specification for conditional %s. Remainder of test behavior is unspecified."
+ msg = msg_template % (tool.id, declared_value, cond.name)
+ log.info(msg)
+
+
+def _add_uploaded_dataset(name, value, extra, input_parameter, required_files):
+ if value is None:
+ assert input_parameter.optional, '%s is not optional. You must provide a valid filename.' % name
+ return value
+ return require_file(name, value, extra, required_files)
+
+
+def _split_if_str(value):
+ split = isinstance(value, string_types)
+ if split:
+ value = value.split(",")
+ return value
+
+
def _process_bool_param_value(param, param_value):
assert isinstance(param, galaxy.tools.parameters.basic.BooleanToolParameter)
was_list = False
@@ -281,30 +261,6 @@ def _process_bool_param_value(param, param_value):
return [processed_value] if was_list else processed_value
-@nottest
-def test_data_iter(required_files):
- for fname, extra in required_files:
- data_dict = dict(
- fname=fname,
- metadata=extra.get('metadata', []),
- composite_data=extra.get('composite_data', []),
- ftype=extra.get('ftype', DEFAULT_FTYPE),
- dbkey=extra.get('dbkey', DEFAULT_DBKEY),
- )
- edit_attributes = extra.get('edit_attributes', [])
-
- # currently only renaming is supported
- for edit_att in edit_attributes:
- if edit_att.get('type', None) == 'name':
- new_name = edit_att.get('value', None)
- assert new_name, 'You must supply the new dataset name as the value tag of the edit_attributes tag'
- data_dict['name'] = new_name
- else:
- raise Exception('edit_attributes type (%s) is unimplemented' % edit_att.get('type', None))
-
- yield data_dict
-
-
def require_file(name, value, extra, required_files):
if (value, extra) not in required_files:
required_files.append((value, extra)) # these files will be uploaded
@@ -359,13 +315,13 @@ class ParamContext(object):
def __raw_param_found(self, param_name, raw_inputs):
index = None
- for i, raw_input in enumerate(raw_inputs):
- if raw_input[0] == param_name:
+ for i, raw_input_dict in enumerate(raw_inputs):
+ if raw_input_dict["name"] == param_name:
index = i
if index is not None:
- raw_input = raw_inputs[index]
+ raw_input_dict = raw_inputs[index]
del raw_inputs[index]
- return raw_input
+ return raw_input_dict
else:
return None
diff --git a/lib/galaxy/tools/toolbox/filters/__init__.py b/lib/galaxy/tools/toolbox/filters/__init__.py
index 8e9a1677f11..2184e37e846 100644
--- a/lib/galaxy/tools/toolbox/filters/__init__.py
+++ b/lib/galaxy/tools/toolbox/filters/__init__.py
@@ -34,7 +34,7 @@ class FilterFactory(object):
filters = deepcopy(self.default_filters)
if trans.user:
for name, value in trans.user.preferences.items():
- if value.strip():
+ if value and value.strip():
user_filters = listify(value, do_strip=True)
category = ''
if name == 'toolbox_tool_filters':
diff --git a/lib/galaxy/tools/verify/asserts/__init__.py b/lib/galaxy/tools/verify/asserts/__init__.py
index 626a4dd7309..59b90d32479 100644
--- a/lib/galaxy/tools/verify/asserts/__init__.py
+++ b/lib/galaxy/tools/verify/asserts/__init__.py
@@ -2,6 +2,8 @@ import inspect
import logging
import sys
+from galaxy.util import unicodify
+
log = logging.getLogger(__name__)
assertion_module_names = ['text', 'tabular', 'xml']
@@ -66,7 +68,11 @@ def verify_assertion(data, assertion_description):
# - .)
if "output" in assert_function_args:
- args["output"] = data
+ # This was read in as bytes for checksum and such, but all current
+ # assertions expect text data. If binary assertions are added at
+ # some point, just checkout for "output_bytes" for instance and pass
+ # data in unchanged.
+ args["output"] = unicodify(data)
if "verify_assertions_function" in assert_function_args:
args["verify_assertions_function"] = verify_assertions
diff --git a/lib/galaxy/tools/verify/interactor.py b/lib/galaxy/tools/verify/interactor.py
new file mode 100644
index 00000000000..5623a7faa44
--- /dev/null
+++ b/lib/galaxy/tools/verify/interactor.py
@@ -0,0 +1,982 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+import os
+import re
+import sys
+import time
+from json import dumps
+from logging import getLogger
+
+try:
+ from nose.tools import nottest
+except ImportError:
+ def nottest(x):
+ return x
+try:
+ import requests
+except ImportError:
+ requests = None
+from six import StringIO, text_type
+
+from galaxy import util
+from galaxy.tools.parser.interface import TestCollectionDef, TestCollectionOutputDef
+from galaxy.util.bunch import Bunch
+from galaxy.util.odict import odict
+from .asserts import verify_assertions
+from ..verify import verify
+
+log = getLogger(__name__)
+
+# Off by default because it can pound the database pretty heavily
+# and result in sqlite errors on larger tests or larger numbers of
+# tests.
+VERBOSE_ERRORS = util.asbool(os.environ.get("GALAXY_TEST_VERBOSE_ERRORS", False))
+UPLOAD_ASYNC = util.asbool(os.environ.get("GALAXY_TEST_UPLOAD_ASYNC", True))
+ERROR_MESSAGE_DATASET_SEP = "--------------------------------------"
+DEFAULT_TOOL_TEST_WAIT = os.environ.get("GALAXY_TEST_DEFAULT_WAIT", 86400)
+
+DEFAULT_FTYPE = 'auto'
+# This following default dbkey was traditionally hg17 before Galaxy 18.05,
+# restore this behavior by setting GALAXY_TEST_DEFAULT_DBKEY to hg17.
+DEFAULT_DBKEY = os.environ.get("GALAXY_TEST_DEFAULT_DBKEY", "?")
+DEFAULT_MAX_SECS = DEFAULT_TOOL_TEST_WAIT
+
+
+def stage_data_in_history(galaxy_interactor, tool_id, all_test_data, history):
+ # Upload any needed files
+ upload_waits = []
+
+ assert tool_id
+
+ if UPLOAD_ASYNC:
+ for test_data in all_test_data:
+ upload_waits.append(galaxy_interactor.stage_data_async(test_data, history, tool_id))
+ for upload_wait in upload_waits:
+ upload_wait()
+ else:
+ for test_data in all_test_data:
+ upload_wait = galaxy_interactor.stage_data_async(test_data, history, tool_id)
+ upload_wait()
+
+
+class GalaxyInteractorApi(object):
+
+ def __init__(self, **kwds):
+ self.api_url = "%s/api" % kwds["galaxy_url"].rstrip("/")
+ self.master_api_key = kwds["master_api_key"]
+ self.api_key = self.__get_user_key(kwds.get("api_key"), kwds.get("master_api_key"), test_user=kwds.get("test_user"))
+ self.keep_outputs_dir = kwds["keep_outputs_dir"]
+
+ self.uploads = {}
+
+ def __get_user_key(self, user_key, admin_key, test_user=None):
+ if not test_user:
+ test_user = "test@bx.psu.edu"
+ if user_key:
+ return user_key
+ test_user = self.ensure_user_with_email(test_user)
+ return self._post("users/%s/api_key" % test_user['id'], key=admin_key).json()
+
+ # def get_tools(self):
+ # response = self._get("tools?in_panel=false")
+ # assert response.status_code == 200, "Non 200 response from tool index API. [%s]" % response.content
+ # return response.json()
+
+ def get_tests_summary(self):
+ response = self._get("tools/tests_summary")
+ assert response.status_code == 200, "Non 200 response from tool tests available API. [%s]" % response.content
+ return response.json()
+
+ def get_tool_tests(self, tool_id, tool_version=None):
+ url = "tools/%s/test_data" % tool_id
+ if tool_version is not None:
+ url += "?tool_version=%s" % tool_version
+ response = self._get(url)
+ assert response.status_code == 200, "Non 200 response from tool test API. [%s]" % response.content
+ return response.json()
+
+ def verify_output(self, history_id, jobs, output_data, output_testdef, tool_id, maxseconds):
+ outfile = output_testdef.outfile
+ attributes = output_testdef.attributes
+ name = output_testdef.name
+ self.wait_for_jobs(history_id, jobs, maxseconds)
+ hid = self.__output_id(output_data)
+ # TODO: Twill version verifys dataset is 'ok' in here.
+ self.verify_output_dataset(history_id=history_id, hda_id=hid, outfile=outfile, attributes=attributes, tool_id=tool_id)
+
+ primary_datasets = attributes.get('primary_datasets', {})
+ if primary_datasets:
+ job_id = self._dataset_provenance(history_id, hid)["job_id"]
+ outputs = self._get("jobs/%s/outputs" % (job_id)).json()
+
+ for designation, (primary_outfile, primary_attributes) in primary_datasets.items():
+ primary_output = None
+ for output in outputs:
+ if output["name"] == '__new_primary_file_%s|%s__' % (name, designation):
+ primary_output = output
+ break
+
+ if not primary_output:
+ msg_template = "Failed to find primary dataset with designation [%s] for output with name [%s]"
+ msg_args = (designation, name)
+ raise Exception(msg_template % msg_args)
+
+ primary_hda_id = primary_output["dataset"]["id"]
+ self.verify_output_dataset(history_id, primary_hda_id, primary_outfile, primary_attributes, tool_id=tool_id)
+
+ def wait_for_jobs(self, history_id, jobs, maxseconds):
+ for job in jobs:
+ self.wait_for_job(job['id'], history_id, maxseconds)
+
+ def verify_output_dataset(self, history_id, hda_id, outfile, attributes, tool_id):
+ fetcher = self.__dataset_fetcher(history_id)
+ test_data_path_builder = self.__test_data_path_builder(tool_id)
+ verify_hid(
+ outfile,
+ hda_id=hda_id,
+ attributes=attributes,
+ dataset_fetcher=fetcher,
+ test_data_path_builder=test_data_path_builder,
+ keep_outputs_dir=self.keep_outputs_dir
+ )
+ self._verify_metadata(history_id, hda_id, attributes)
+
+ def _verify_metadata(self, history_id, hid, attributes):
+ """Check dataset metadata.
+
+ ftype on output maps to `file_ext` on the hda's API description, `name`, `info`,
+ `dbkey` and `tags` all map to the API description directly. Other metadata attributes
+ are assumed to be datatype-specific and mapped with a prefix of `metadata_`.
+ """
+ metadata = attributes.get('metadata', {}).copy()
+ for key, value in metadata.copy().items():
+ if key not in ['name', 'info', 'tags']:
+ new_key = "metadata_%s" % key
+ metadata[new_key] = metadata[key]
+ del metadata[key]
+ elif key == "info":
+ metadata["misc_info"] = metadata["info"]
+ del metadata["info"]
+ expected_file_type = attributes.get('ftype', None)
+ if expected_file_type:
+ metadata["file_ext"] = expected_file_type
+
+ if metadata:
+ time.sleep(5)
+ dataset = self._get("histories/%s/contents/%s" % (history_id, hid)).json()
+ for key, value in metadata.items():
+ try:
+ dataset_value = dataset.get(key, None)
+
+ def compare(val, expected):
+ if text_type(val) != text_type(expected):
+ msg = "Dataset metadata verification for [%s] failed, expected [%s] but found [%s]. Dataset API value was [%s]."
+ msg_params = (key, value, dataset_value, dataset)
+ msg = msg % msg_params
+ raise Exception(msg)
+
+ if isinstance(dataset_value, list):
+ value = text_type(value).split(",")
+ if len(value) != len(dataset_value):
+ msg = "Dataset metadata verification for [%s] failed, expected [%s] but found [%s], lists differ in length. Dataset API value was [%s]."
+ msg_params = (key, value, dataset_value, dataset)
+ msg = msg % msg_params
+ raise Exception(msg)
+ for val, expected in zip(dataset_value, value):
+ compare(val, expected)
+ else:
+ compare(dataset_value, value)
+ except KeyError:
+ msg = "Failed to verify dataset metadata, metadata key [%s] was not found." % key
+ raise Exception(msg)
+
+ def wait_for_job(self, job_id, history_id, maxseconds):
+ self.wait_for(lambda: not self.__job_ready(job_id, history_id), maxseconds=maxseconds)
+
+ def wait_for(self, func, **kwd):
+ sleep_amount = 0.2
+ slept = 0
+ walltime_exceeded = kwd.get("maxseconds", DEFAULT_TOOL_TEST_WAIT)
+
+ while slept <= walltime_exceeded:
+ result = func()
+ if result:
+ time.sleep(sleep_amount)
+ slept += sleep_amount
+ sleep_amount *= 2
+ else:
+ return
+
+ message = 'Tool test run exceeded walltime [total %s, max %s], terminating.' % (slept, walltime_exceeded)
+ log.info(message)
+ raise AssertionError(message)
+
+ def get_job_stdio(self, job_id):
+ job_stdio = self.__get_job_stdio(job_id).json()
+ return job_stdio
+
+ def __get_job(self, job_id):
+ return self._get('jobs/%s' % job_id)
+
+ def __get_job_stdio(self, job_id):
+ return self._get('jobs/%s?full=true' % job_id)
+
+ def new_history(self):
+ history_json = self._post("histories", {"name": "test_history"}).json()
+ return history_json['id']
+
+ @nottest
+ def test_data_path(self, tool_id, filename):
+ return self._get("tools/%s/test_data_path?filename=%s" % (tool_id, filename)).json()
+
+ def __output_id(self, output_data):
+ # Allow data structure coming out of tools API - {id: , output_name: , etc...}
+ # or simple id as comes out of workflow API.
+ try:
+ output_id = output_data.get('id')
+ except AttributeError:
+ output_id = output_data
+ return output_id
+
+ def stage_data_async(self, test_data, history_id, tool_id, async=True):
+ fname = test_data['fname']
+ tool_input = {
+ "file_type": test_data['ftype'],
+ "dbkey": test_data['dbkey'],
+ }
+ metadata = test_data.get("metadata", {})
+ if not hasattr(metadata, "items"):
+ raise Exception("Invalid metadata description found for input [%s] - [%s]" % (fname, metadata))
+ for name, value in test_data.get('metadata', {}).items():
+ tool_input["files_metadata|%s" % name] = value
+
+ composite_data = test_data['composite_data']
+ if composite_data:
+ files = {}
+ for i, file_name in enumerate(composite_data):
+ file_name = self.test_data_path(tool_id, file_name)
+ files["files_%s|file_data" % i] = open(file_name, 'rb')
+ tool_input.update({
+ "files_%d|type" % i: "upload_dataset",
+ })
+ name = test_data['name']
+ else:
+ file_name = self.test_data_path(tool_id, fname)
+ name = test_data.get('name', None)
+ if not name:
+ name = os.path.basename(file_name)
+
+ tool_input.update({
+ "files_0|NAME": name,
+ "files_0|type": "upload_dataset",
+ })
+ # TODO: Option to upload by path since we are getting the paths from Galaxy now it makes more
+ # sense to move this there.
+ files = {
+ "files_0|file_data": open(file_name, 'rb')
+ }
+ submit_response_object = self.__submit_tool(history_id, "upload1", tool_input, extra_data={"type": "upload_dataset"}, files=files)
+ if submit_response_object.status_code != 200:
+ raise Exception("Request to upload dataset failed [%s]" % submit_response_object.content)
+ submit_response = submit_response_object.json()
+ assert "outputs" in submit_response, "Invalid response from server [%s], expecteding outputs in response." % submit_response
+ outputs = submit_response["outputs"]
+ assert len(outputs) > 0, "Invalid response from server [%s], expecting an output dataset." % submit_response
+ dataset = outputs[0]
+ hid = dataset['id']
+ self.uploads[os.path.basename(fname)] = self.uploads[fname] = self.uploads[name] = {"src": "hda", "id": hid}
+ assert "jobs" in submit_response, "Invalid response from server [%s], expecting jobs in response." % submit_response
+ jobs = submit_response["jobs"]
+ assert len(jobs) > 0, "Invalid response from server [%s], expecting a job." % submit_response
+ return lambda: self.wait_for_job(jobs[0]["id"], history_id, DEFAULT_TOOL_TEST_WAIT)
+
+ def run_tool(self, testdef, history_id, resource_parameters={}):
+ # We need to handle the case where we've uploaded a valid compressed file since the upload
+ # tool will have uncompressed it on the fly.
+
+ inputs_tree = testdef.inputs.copy()
+ for key, value in inputs_tree.items():
+ values = [value] if not isinstance(value, list) else value
+ new_values = []
+ for value in values:
+ if isinstance(value, TestCollectionDef):
+ hdca_id = self._create_collection(history_id, value)
+ new_values = [dict(src="hdca", id=hdca_id)]
+ elif value in self.uploads:
+ new_values.append(self.uploads[value])
+ else:
+ new_values.append(value)
+ inputs_tree[key] = new_values
+
+ if resource_parameters:
+ inputs_tree["__job_resource|__job_resource__select"] = "yes"
+ for key, value in resource_parameters.items():
+ inputs_tree["__job_resource|%s" % key] = value
+
+ # HACK: Flatten single-value lists. Required when using expand_grouping
+ for key, value in inputs_tree.items():
+ if isinstance(value, list) and len(value) == 1:
+ inputs_tree[key] = value[0]
+
+ submit_response = self.__submit_tool(history_id, tool_id=testdef.tool_id, tool_input=inputs_tree)
+ submit_response_object = submit_response.json()
+ try:
+ return Bunch(
+ inputs=inputs_tree,
+ outputs=self.__dictify_outputs(submit_response_object),
+ output_collections=self.__dictify_output_collections(submit_response_object),
+ jobs=submit_response_object['jobs'],
+ )
+ except KeyError:
+ message = "Error creating a job for these tool inputs - %s" % submit_response_object['err_msg']
+ raise RunToolException(message, inputs_tree)
+
+ def _create_collection(self, history_id, collection_def):
+ create_payload = dict(
+ name=collection_def.name,
+ element_identifiers=dumps(self._element_identifiers(collection_def)),
+ collection_type=collection_def.collection_type,
+ history_id=history_id,
+ )
+ return self._post("dataset_collections", data=create_payload).json()["id"]
+
+ def _element_identifiers(self, collection_def):
+ element_identifiers = []
+ for element_dict in collection_def.elements:
+ element_identifier = element_dict["element_identifier"]
+ element_def = element_dict["element_definition"]
+ if isinstance(element_def, TestCollectionDef):
+ subelement_identifiers = self._element_identifiers(element_def)
+ element = dict(
+ name=element_identifier,
+ src="new_collection",
+ collection_type=element_def.collection_type,
+ element_identifiers=subelement_identifiers
+ )
+ else:
+ element = self.uploads[element_def["value"]].copy()
+ element["name"] = element_identifier
+ element_identifiers.append(element)
+ return element_identifiers
+
+ def __dictify_output_collections(self, submit_response):
+ output_collections_dict = odict()
+ for output_collection in submit_response['output_collections']:
+ output_collections_dict[output_collection.get("output_name")] = output_collection
+ return output_collections_dict
+
+ def __dictify_outputs(self, datasets_object):
+ # Convert outputs list to a dictionary that can be accessed by
+ # output_name so can be more flexiable about ordering of outputs
+ # but also allows fallback to legacy access as list mode.
+ outputs_dict = odict()
+ index = 0
+ for output in datasets_object['outputs']:
+ outputs_dict[index] = outputs_dict[output.get("output_name")] = output
+ index += 1
+ # Adding each item twice (once with index for backward compat),
+ # overiding length to reflect the real number of outputs.
+ outputs_dict.__len__ = lambda: index
+ return outputs_dict
+
+ def output_hid(self, output_data):
+ return output_data['id']
+
+ def delete_history(self, history):
+ return None
+
+ def __job_ready(self, job_id, history_id):
+ if job_id is None:
+ raise ValueError("__job_ready passed empty job_id")
+ job_json = self._get("jobs/%s" % job_id).json()
+ state = job_json['state']
+ try:
+ return self._state_ready(state, error_msg="Job in error state.")
+ except Exception:
+ if VERBOSE_ERRORS:
+ self._summarize_history(history_id)
+ raise
+
+ def _summarize_history(self, history_id):
+ if history_id is None:
+ raise ValueError("_summarize_history passed empty history_id")
+ print("Problem in history with id %s - summary of datasets below." % history_id)
+ try:
+ history_contents = self.__contents(history_id)
+ except Exception:
+ print("*TEST FRAMEWORK FAILED TO FETCH HISTORY DETAILS*")
+
+ for history_content in history_contents:
+
+ dataset = history_content
+
+ print(ERROR_MESSAGE_DATASET_SEP)
+ dataset_id = dataset.get('id', None)
+ print("| %d - %s (HID - NAME) " % (int(dataset['hid']), dataset['name']))
+ if history_content['history_content_type'] == 'dataset_collection':
+ history_contents_json = self._get("histories/%s/contents/dataset_collections/%s" % (history_id, history_content["id"])).json()
+ print("| Dataset Collection: %s" % history_contents_json)
+ continue
+
+ try:
+ dataset_info = self._dataset_info(history_id, dataset_id)
+ print("| Dataset State:")
+ print(self.format_for_summary(dataset_info.get("state"), "Dataset state is unknown."))
+ print("| Dataset Blurb:")
+ print(self.format_for_summary(dataset_info.get("misc_blurb", ""), "Dataset blurb was empty."))
+ print("| Dataset Info:")
+ print(self.format_for_summary(dataset_info.get("misc_info", ""), "Dataset info is empty."))
+ print("| Peek:")
+ print(self.format_for_summary(dataset_info.get("peek", ""), "Peek unavilable."))
+ except Exception:
+ print("| *TEST FRAMEWORK ERROR FETCHING DATASET DETAILS*")
+ try:
+ provenance_info = self._dataset_provenance(history_id, dataset_id)
+ print("| Dataset Job Standard Output:")
+ print(self.format_for_summary(provenance_info.get("stdout", ""), "Standard output was empty."))
+ print("| Dataset Job Standard Error:")
+ print(self.format_for_summary(provenance_info.get("stderr", ""), "Standard error was empty."))
+ except Exception:
+ print("| *TEST FRAMEWORK ERROR FETCHING JOB DETAILS*")
+ print("|")
+ print(ERROR_MESSAGE_DATASET_SEP)
+
+ def format_for_summary(self, blob, empty_message, prefix="| "):
+ contents = "\n".join(["%s%s" % (prefix, line.strip()) for line in StringIO(blob).readlines() if line.rstrip("\n\r")])
+ return contents or "%s*%s*" % (prefix, empty_message)
+
+ def _dataset_provenance(self, history_id, id):
+ provenance = self._get("histories/%s/contents/%s/provenance" % (history_id, id)).json()
+ return provenance
+
+ def _dataset_info(self, history_id, id):
+ dataset_json = self._get("histories/%s/contents/%s" % (history_id, id)).json()
+ return dataset_json
+
+ def __contents(self, history_id):
+ history_contents_json = self._get("histories/%s/contents" % history_id).json()
+ return history_contents_json
+
+ def _state_ready(self, state_str, error_msg):
+ if state_str == 'ok':
+ return True
+ elif state_str == 'error':
+ raise Exception(error_msg)
+ return False
+
+ def __submit_tool(self, history_id, tool_id, tool_input, extra_data={}, files=None):
+ data = dict(
+ history_id=history_id,
+ tool_id=tool_id,
+ inputs=dumps(tool_input),
+ **extra_data
+ )
+ return self._post("tools", files=files, data=data)
+
+ def ensure_user_with_email(self, email, password=None):
+ admin_key = self.master_api_key
+ all_users = self._get('users', key=admin_key).json()
+ try:
+ test_user = [user for user in all_users if user["email"] == email][0]
+ except IndexError:
+ username = re.sub('[^a-z-]', '--', email.lower())
+ password = password or 'testpass'
+ # If remote user middleware is enabled - this endpoint consumes
+ # ``remote_user_email`` otherwise it requires ``email``, ``password``
+ # and ``username``.
+ data = dict(
+ remote_user_email=email,
+ email=email,
+ password=password,
+ username=username,
+ )
+ test_user = self._post('users', data, key=admin_key).json()
+ return test_user
+
+ def __test_data_path_builder(self, tool_id):
+ return lambda filename: self.test_data_path(tool_id, filename)
+
+ def __dataset_fetcher(self, history_id):
+ def fetcher(hda_id, base_name=None):
+ url = "histories/%s/contents/%s/display?raw=true" % (history_id, hda_id)
+ if base_name:
+ url += "&filename=%s" % base_name
+ return self._get(url).content
+
+ return fetcher
+
+ def _post(self, path, data={}, files=None, key=None, admin=False, anon=False):
+ if not anon:
+ if not key:
+ key = self.api_key if not admin else self.master_api_key
+ data = data.copy()
+ data['key'] = key
+ return requests.post("%s/%s" % (self.api_url, path), data=data, files=files)
+
+ def _delete(self, path, data={}, key=None, admin=False, anon=False):
+ if not anon:
+ if not key:
+ key = self.api_key if not admin else self.master_api_key
+ data = data.copy()
+ data['key'] = key
+ return requests.delete("%s/%s" % (self.api_url, path), params=data)
+
+ def _patch(self, path, data={}, key=None, admin=False, anon=False):
+ if not anon:
+ if not key:
+ key = self.api_key if not admin else self.master_api_key
+ params = dict(key=key)
+ data = data.copy()
+ data['key'] = key
+ else:
+ params = {}
+ return requests.patch("%s/%s" % (self.api_url, path), params=params, data=data)
+
+ def _put(self, path, data={}, key=None, admin=False, anon=False):
+ if not anon:
+ if not key:
+ key = self.api_key if not admin else self.master_api_key
+ params = dict(key=key)
+ data = data.copy()
+ data['key'] = key
+ else:
+ params = {}
+ return requests.put("%s/%s" % (self.api_url, path), params=params, data=data)
+
+ def _get(self, path, data={}, key=None, admin=False, anon=False):
+ if not anon:
+ if not key:
+ key = self.api_key if not admin else self.master_api_key
+ data = data.copy()
+ data['key'] = key
+ if path.startswith("/api"):
+ path = path[len("/api"):]
+ url = "%s/%s" % (self.api_url, path)
+ return requests.get(url, params=data)
+
+
+class RunToolException(Exception):
+
+ def __init__(self, message, inputs=None):
+ super(RunToolException, self).__init__(message)
+ self.inputs = inputs
+
+
+# Galaxy specific methods - rest of this can be used with arbitrary files and such.
+def verify_hid(filename, hda_id, attributes, test_data_path_builder, hid="", dataset_fetcher=None, keep_outputs_dir=False):
+ assert dataset_fetcher is not None
+
+ def verify_extra_files(extra_files):
+ _verify_extra_files_content(extra_files, hda_id, dataset_fetcher=dataset_fetcher, test_data_path_builder=test_data_path_builder, keep_outputs_dir=keep_outputs_dir)
+
+ data = dataset_fetcher(hda_id)
+ item_label = "History item %s" % hid
+ verify(
+ item_label,
+ data,
+ attributes=attributes,
+ filename=filename,
+ get_filename=test_data_path_builder,
+ keep_outputs_dir=keep_outputs_dir,
+ verify_extra_files=verify_extra_files,
+ )
+
+
+def _verify_composite_datatype_file_content(file_name, hda_id, base_name=None, attributes=None, dataset_fetcher=None, test_data_path_builder=None, keep_outputs_dir=False):
+ assert dataset_fetcher is not None
+
+ data = dataset_fetcher(hda_id, base_name)
+ item_label = "History item %s" % hda_id
+ try:
+ verify(
+ item_label,
+ data,
+ attributes=attributes,
+ filename=file_name,
+ get_filename=test_data_path_builder,
+ keep_outputs_dir=keep_outputs_dir,
+ )
+ except AssertionError as err:
+ errmsg = 'Composite file (%s) of %s different than expected, difference:\n' % (base_name, item_label)
+ errmsg += str(err)
+ raise AssertionError(errmsg)
+
+
+def _verify_extra_files_content(extra_files, hda_id, dataset_fetcher, test_data_path_builder, keep_outputs_dir):
+ files_list = []
+ for extra_file_dict in extra_files:
+ extra_file_type = extra_file_dict["type"]
+ extra_file_name = extra_file_dict["name"]
+ extra_file_attributes = extra_file_dict["attributes"]
+ extra_file_value = extra_file_dict["value"]
+
+ if extra_file_type == 'file':
+ files_list.append((extra_file_name, extra_file_value, extra_file_attributes))
+ elif extra_file_type == 'directory':
+ for filename in os.listdir(test_data_path_builder(extra_file_value)):
+ files_list.append((filename, os.path.join(extra_file_value, filename), extra_file_attributes))
+ else:
+ raise ValueError('unknown extra_files type: %s' % extra_file_type)
+ for filename, filepath, attributes in files_list:
+ _verify_composite_datatype_file_content(filepath, hda_id, base_name=filename, attributes=attributes, dataset_fetcher=dataset_fetcher, test_data_path_builder=test_data_path_builder, keep_outputs_dir=keep_outputs_dir)
+
+
+def verify_tool(tool_id, galaxy_interactor, resource_parameters={}, register_job_data=None, test_index=0, tool_version=None, quiet=False):
+ tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version)
+ tool_test_dict = tool_test_dicts[test_index]
+ testdef = ToolTestDescription(tool_test_dict)
+
+ _handle_def_errors(testdef)
+
+ test_history = galaxy_interactor.new_history()
+
+ stage_data_in_history(galaxy_interactor, tool_id, testdef.test_data(), test_history)
+
+ # Once data is ready, run the tool and check the outputs - record API
+ # input, job info, tool run exception, as well as exceptions related to
+ # job output checking and register they with the test plugin so it can
+ # record structured information.
+ tool_inputs = None
+ job_stdio = None
+ job_output_exceptions = None
+ tool_execution_exception = None
+ expected_failure_occurred = False
+ begin_time = time.time()
+ try:
+ try:
+ tool_response = galaxy_interactor.run_tool(testdef, test_history, resource_parameters=resource_parameters)
+ data_list, jobs, tool_inputs = tool_response.outputs, tool_response.jobs, tool_response.inputs
+ data_collection_list = tool_response.output_collections
+ except RunToolException as e:
+ tool_inputs = e.inputs
+ tool_execution_exception = e
+ if not testdef.expect_failure:
+ raise e
+ else:
+ expected_failure_occurred = True
+ except Exception as e:
+ tool_execution_exception = e
+ raise e
+
+ if not expected_failure_occurred:
+ assert data_list or data_collection_list
+
+ try:
+ job_stdio = _verify_outputs(testdef, test_history, jobs, tool_id, data_list, data_collection_list, galaxy_interactor, quiet=quiet)
+ except JobOutputsError as e:
+ job_stdio = e.job_stdio
+ job_output_exceptions = e.output_exceptions
+ raise e
+ except Exception as e:
+ job_output_exceptions = [e]
+ raise e
+ finally:
+ if register_job_data is not None:
+ end_time = time.time()
+ job_data = {
+ "tool_id": tool_id,
+ "tool_version": tool_version,
+ "test_index": test_index,
+ "time_seconds": end_time - begin_time,
+ }
+ if tool_inputs is not None:
+ job_data["inputs"] = tool_inputs
+ if job_stdio is not None:
+ job_data["job"] = job_stdio
+ status = "success"
+ if job_output_exceptions:
+ job_data["output_problems"] = [str(_) for _ in job_output_exceptions]
+ status = "failure"
+ if tool_execution_exception:
+ job_data["execution_problem"] = str(tool_execution_exception)
+ status = "error"
+ job_data["status"] = status
+ register_job_data(job_data)
+
+ galaxy_interactor.delete_history(test_history)
+
+
+def _handle_def_errors(testdef):
+ # If the test generation had an error, raise
+ if testdef.error:
+ if testdef.exception:
+ if isinstance(testdef.exception, Exception):
+ raise testdef.exception
+ else:
+ raise Exception(testdef.exception)
+ else:
+ raise Exception("Test parse failure")
+
+
+def _verify_outputs(testdef, history, jobs, tool_id, data_list, data_collection_list, galaxy_interactor, quiet=False):
+ assert len(jobs) == 1, "Test framework logic error, somehow tool test resulted in more than one job."
+ job = jobs[0]
+
+ maxseconds = testdef.maxseconds
+ if testdef.num_outputs is not None:
+ expected = testdef.num_outputs
+ actual = len(data_list)
+ if expected != actual:
+ messaage_template = "Incorrect number of outputs - expected %d, found %s."
+ message = messaage_template % (expected, actual)
+ raise Exception(message)
+ found_exceptions = []
+
+ def register_exception(e):
+ if not found_exceptions and not quiet:
+ # Only print this stuff out once.
+ for stream in ['stdout', 'stderr']:
+ if stream in job_stdio:
+ print(_format_stream(job_stdio[stream], stream=stream, format=True), file=sys.stderr)
+ found_exceptions.append(e)
+
+ if testdef.expect_failure:
+ if testdef.outputs:
+ raise Exception("Cannot specify outputs in a test expecting failure.")
+
+ # Wait for the job to complete and register expections if the final
+ # status was not what test was expecting.
+ job_failed = False
+ try:
+ galaxy_interactor.wait_for_job(job['id'], history, maxseconds)
+ except Exception as e:
+ job_failed = True
+ if not testdef.expect_failure:
+ found_exceptions.append(e)
+
+ job_stdio = galaxy_interactor.get_job_stdio(job['id'])
+
+ if not job_failed and testdef.expect_failure:
+ error = AssertionError("Expected job to fail but Galaxy indicated the job successfully completed.")
+ register_exception(error)
+
+ expect_exit_code = testdef.expect_exit_code
+ if expect_exit_code is not None:
+ exit_code = job_stdio["exit_code"]
+ if str(expect_exit_code) != str(exit_code):
+ error = AssertionError("Expected job to complete with exit code %s, found %s" % (expect_exit_code, exit_code))
+ register_exception(error)
+
+ for output_index, output_dict in enumerate(testdef.outputs):
+ # Get the correct hid
+ name = output_dict["name"]
+ outfile = output_dict["value"]
+ attributes = output_dict["attributes"]
+ output_testdef = Bunch(name=name, outfile=outfile, attributes=attributes)
+ try:
+ output_data = data_list[name]
+ except (TypeError, KeyError):
+ # Legacy - fall back on ordered data list access if data_list is
+ # just a list (case with twill variant or if output changes its
+ # name).
+ if hasattr(data_list, "values"):
+ output_data = list(data_list.values())[output_index]
+ else:
+ output_data = data_list[len(data_list) - len(testdef.outputs) + output_index]
+ assert output_data is not None
+ try:
+ galaxy_interactor.verify_output(history, jobs, output_data, output_testdef=output_testdef, tool_id=tool_id, maxseconds=maxseconds)
+ except Exception as e:
+ register_exception(e)
+
+ other_checks = {
+ "command_line": "Command produced by the job",
+ "stdout": "Standard output of the job",
+ "stderr": "Standard error of the job",
+ }
+ for what, description in other_checks.items():
+ if getattr(testdef, what, None) is not None:
+ try:
+ data = job_stdio[what]
+ verify_assertions(data, getattr(testdef, what))
+ except AssertionError as err:
+ errmsg = '%s different than expected\n' % description
+ errmsg += str(err)
+ register_exception(AssertionError(errmsg))
+
+ for output_collection_def in testdef.output_collections:
+ try:
+ name = output_collection_def.name
+ # TODO: data_collection_list is clearly a bad name for dictionary.
+ if name not in data_collection_list:
+ template = "Failed to find output [%s], tool outputs include [%s]"
+ message = template % (name, ",".join(data_collection_list.keys()))
+ raise AssertionError(message)
+
+ # Data collection returned from submission, elements may have been populated after
+ # the job completed so re-hit the API for more information.
+ data_collection_returned = data_collection_list[name]
+ data_collection = galaxy_interactor._get("dataset_collections/%s" % data_collection_returned["id"], data={"instance_type": "history"}).json()
+
+ def get_element(elements, id):
+ for element in elements:
+ if element["element_identifier"] == id:
+ return element
+ return False
+
+ expected_collection_type = output_collection_def.collection_type
+ if expected_collection_type:
+ collection_type = data_collection["collection_type"]
+ if expected_collection_type != collection_type:
+ template = "Expected output collection [%s] to be of type [%s], was of type [%s]."
+ message = template % (name, expected_collection_type, collection_type)
+ raise AssertionError(message)
+
+ expected_element_count = output_collection_def.count
+ if expected_element_count:
+ actual_element_count = len(data_collection["elements"])
+ if expected_element_count != actual_element_count:
+ template = "Expected output collection [%s] to have %s elements, but it had %s."
+ message = template % (name, expected_element_count, actual_element_count)
+ raise AssertionError(message)
+
+ def verify_elements(element_objects, element_tests):
+ for element_identifier, (element_outfile, element_attrib) in element_tests.items():
+ element = get_element(element_objects, element_identifier)
+ if not element:
+ template = "Failed to find identifier [%s] for testing, tool generated collection elements [%s]"
+ message = template % (element_identifier, element_objects)
+ raise AssertionError(message)
+
+ element_type = element["element_type"]
+ if element_type != "dataset_collection":
+ hda = element["object"]
+ galaxy_interactor.verify_output_dataset(
+ history,
+ hda_id=hda["id"],
+ outfile=element_outfile,
+ attributes=element_attrib,
+ tool_id=tool_id
+ )
+ if element_type == "dataset_collection":
+ elements = element["object"]["elements"]
+ verify_elements(elements, element_attrib.get("elements", {}))
+
+ verify_elements(data_collection["elements"], output_collection_def.element_tests)
+ except Exception as e:
+ register_exception(e)
+
+ if found_exceptions:
+ raise JobOutputsError(found_exceptions, job_stdio)
+ else:
+ return job_stdio
+
+
+def _format_stream(output, stream, format):
+ output = output or ''
+ if format:
+ msg = "---------------------- >> begin tool %s << -----------------------\n" % stream
+ msg += output + "\n"
+ msg += "----------------------- >> end tool %s << ------------------------\n" % stream
+ else:
+ msg = output
+ return msg
+
+
+class JobOutputsError(AssertionError):
+
+ def __init__(self, output_exceptions, job_stdio):
+ big_message = "\n".join(map(str, output_exceptions))
+ super(JobOutputsError, self).__init__(big_message)
+ self.job_stdio = job_stdio
+ self.output_exceptions = output_exceptions
+
+
+class ToolTestDescription(object):
+ """
+ Encapsulates information about a tool test, and allows creation of a
+ dynamic TestCase class (the unittest framework is very class oriented,
+ doing dynamic tests in this way allows better integration)
+ """
+
+ def __init__(self, processed_test_dict):
+ test_index = processed_test_dict["test_index"]
+ name = processed_test_dict.get('name', 'Test-%d' % (test_index + 1))
+ maxseconds = processed_test_dict.get('maxseconds', DEFAULT_MAX_SECS)
+ if maxseconds is not None:
+ maxseconds = int(maxseconds)
+
+ self.test_index = test_index
+ self.tool_id = processed_test_dict["tool_id"]
+ self.name = name
+ self.maxseconds = maxseconds
+ self.required_files = processed_test_dict.get("required_files", [])
+
+ inputs = processed_test_dict.get("inputs", {})
+ loaded_inputs = {}
+ for key, value in inputs.items():
+ if isinstance(value, dict) and value.get("model_class"):
+ loaded_inputs[key] = TestCollectionDef.from_dict(value)
+ else:
+ loaded_inputs[key] = value
+
+ self.inputs = loaded_inputs
+ self.outputs = processed_test_dict.get("outputs", [])
+ self.num_outputs = processed_test_dict.get("num_outputs", 0)
+
+ self.error = processed_test_dict.get("error", False)
+ self.exception = processed_test_dict.get("exception", None)
+
+ self.output_collections = map(TestCollectionOutputDef.from_dict, processed_test_dict.get("output_collections", []))
+ self.command_line = processed_test_dict.get("command", None)
+ self.stdout = processed_test_dict.get("stdout", None)
+ self.stderr = processed_test_dict.get("stderr", None)
+ self.expect_exit_code = processed_test_dict.get("expect_exit_code", None)
+ self.expect_failure = processed_test_dict.get("expect_failure", False)
+
+ def test_data(self):
+ """
+ Iterator over metadata representing the required files for upload.
+ """
+ return test_data_iter(self.required_files)
+
+ def to_dict(self):
+ inputs_dict = {}
+ for key, value in self.inputs.items():
+ if hasattr(value, "to_dict"):
+ inputs_dict[key] = value.to_dict()
+ else:
+ inputs_dict[key] = value
+
+ return {
+ "inputs": inputs_dict,
+ "outputs": self.outputs,
+ "output_collections": map(lambda o: o.to_dict(), self.output_collections),
+ "num_outputs": self.num_outputs,
+ "command_line": self.command_line,
+ "stdout": self.stdout,
+ "stderr": self.stderr,
+ "expect_exit_code": self.expect_exit_code,
+ "expect_failure": self.expect_failure,
+ "name": self.name,
+ "test_index": self.test_index,
+ "tool_id": self.tool_id,
+ "required_files": self.required_files,
+ "error": self.error,
+ "exception": self.exception,
+ }
+
+
+@nottest
+def test_data_iter(required_files):
+ for fname, extra in required_files:
+ data_dict = dict(
+ fname=fname,
+ metadata=extra.get('metadata', {}),
+ composite_data=extra.get('composite_data', []),
+ ftype=extra.get('ftype', DEFAULT_FTYPE),
+ dbkey=extra.get('dbkey', DEFAULT_DBKEY),
+ )
+ edit_attributes = extra.get('edit_attributes', [])
+
+ # currently only renaming is supported
+ for edit_att in edit_attributes:
+ if edit_att.get('type', None) == 'name':
+ new_name = edit_att.get('value', None)
+ assert new_name, 'You must supply the new dataset name as the value tag of the edit_attributes tag'
+ data_dict['name'] = new_name
+ else:
+ raise Exception('edit_attributes type (%s) is unimplemented' % edit_att.get('type', None))
+
+ yield data_dict
diff --git a/lib/galaxy/tools/verify/script.py b/lib/galaxy/tools/verify/script.py
new file mode 100644
index 00000000000..64873a368f4
--- /dev/null
+++ b/lib/galaxy/tools/verify/script.py
@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+from __future__ import print_function
+
+import argparse
+import json
+import sys
+
+from galaxy.tools.verify.interactor import GalaxyInteractorApi, verify_tool
+
+DESCRIPTION = """Script to quickly run a tool test against a running Galaxy instance."""
+ALL_TESTS = "*all_tests*"
+
+
+def main(argv=None):
+ if argv is None:
+ argv = sys.argv[1:]
+
+ args = _arg_parser().parse_args(argv)
+ galaxy_interactor_kwds = {
+ "galaxy_url": args.galaxy_url,
+ "master_api_key": args.admin_key,
+ "api_key": args.key,
+ "keep_outputs_dir": args.output,
+ }
+ tool_id = args.tool_id
+ tool_version = args.tool_version
+
+ galaxy_interactor = GalaxyInteractorApi(**galaxy_interactor_kwds)
+ raw_test_index = args.test_index
+ if raw_test_index == ALL_TESTS:
+ tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version)
+ test_indices = list(range(len(tool_test_dicts)))
+ else:
+ test_indices = [int(raw_test_index)]
+
+ test_results = []
+
+ if args.append:
+ with open(args.output_json, "r") as f:
+ previous_results = json.load(f)
+ test_results = previous_results["tests"]
+
+ exceptions = []
+ verbose = args.verbose
+ for test_index in test_indices:
+ if tool_version:
+ tool_id_and_version = "%s/%s" % (tool_id, tool_version)
+ else:
+ tool_id_and_version = tool_id
+
+ test_identifier = "tool %s test # %d" % (tool_id_and_version, test_index)
+
+ def register(job_data):
+ test_results.append({
+ 'id': tool_id + "-" + str(test_index),
+ 'has_data': True,
+ 'data': job_data,
+ })
+
+ try:
+ verify_tool(
+ tool_id, galaxy_interactor, test_index=test_index, tool_version=tool_version,
+ register_job_data=register, quiet=not verbose
+ )
+
+ if verbose:
+ print("%s passed" % test_identifier)
+
+ except Exception as e:
+ if verbose:
+ print("%s failed, %s" % (test_identifier, e))
+ exceptions.append(e)
+
+ report_obj = {
+ 'version': '0.1',
+ 'tests': test_results,
+ }
+ output_json = args.output_json
+ if output_json:
+ if args.output_json == "-":
+ assert not args.append
+ print(json.dumps(report_obj))
+ else:
+ with open(args.output_json, "w") as f:
+ json.dump(report_obj, f)
+
+ if exceptions:
+ raise exceptions[0]
+
+
+def _arg_parser():
+ parser = argparse.ArgumentParser(description=DESCRIPTION)
+ parser.add_argument('-u', '--galaxy-url', default="http://localhost:8080", help='Galaxy URL')
+ parser.add_argument('-k', '--key', default=None, help='Galaxy User API Key')
+ parser.add_argument('-a', '--admin-key', default=None, help='Galaxy Admin API Key')
+ parser.add_argument('-t', '--tool-id', default=None, help='Tool ID')
+ parser.add_argument('--tool-version', default=None, help='Tool Version')
+ parser.add_argument('-i', '--test-index', default=ALL_TESTS, help='Tool Test Index (starting at 0) - by default all tests will run.')
+ parser.add_argument('-o', '--output', default=None, help='directory to dump outputs to')
+ parser.add_argument('--append', default=False, action="store_true", help="Extend a test record json (created with --output-json) with additional tests.")
+ parser.add_argument('-j', '--output-json', default=None, help='output metadata json')
+ parser.add_argument('--verbose', default=False, action="store_true", help="Verbose logging.")
+ return parser
+
+
+if __name__ == "__main__":
+ main()
diff --git a/lib/galaxy/tools/verify/test_data.py b/lib/galaxy/tools/verify/test_data.py
index 1c629741ee1..ea63d864aa8 100644
--- a/lib/galaxy/tools/verify/test_data.py
+++ b/lib/galaxy/tools/verify/test_data.py
@@ -8,6 +8,7 @@ from string import Template
from galaxy.util import (
asbool,
+ in_directory,
smart_str
)
@@ -27,32 +28,25 @@ LIST_SEP = re.compile("\s*,\s*")
class TestDataResolver(object):
- def __init__(self, env_var='GALAXY_TEST_FILE_DIR', environ=os.environ):
- file_dirs = environ.get(env_var, None)
+ def __init__(self, file_dirs=None, env_var='GALAXY_TEST_FILE_DIR', environ=os.environ):
+ if file_dirs is None:
+ file_dirs = environ.get(env_var, None)
+ if file_dirs is None:
+ file_dirs = "test-data,https://github.com/galaxyproject/galaxy-test-data.git"
if file_dirs:
self.resolvers = [build_resolver(u, environ) for u in LIST_SEP.split(file_dirs)]
else:
self.resolvers = []
def get_filename(self, name):
- if not self.resolvers:
- filename = None
- else:
- resolver = self.resolvers[0]
+ filename = None
+ for resolver in self.resolvers or []:
+ if not resolver.exists(name):
+ continue
filename = resolver.path(name)
- if not resolver.exists(filename):
- for resolver in self.resolvers[1:]:
- if resolver.exists(name):
- filename = resolver.path(name)
- else:
- # For backward compat. returning first path if none
- # exist - though I don't know if this function is ever
- # actually used in a context where one should return
- # a file even if it doesn't exist (e.g. a prefix or
- # or something) - I am pretty sure it is not used in
- # such a fashion in the context of tool tests.
- filename = resolver.path(name)
- return os.path.abspath(filename)
+
+ if filename:
+ return os.path.abspath(filename)
def build_resolver(uri, environ):
@@ -68,7 +62,8 @@ class FileDataResolver(object):
self.file_dir = file_dir
def exists(self, filename):
- return os.path.exists(self.path(filename))
+ path = os.path.abspath(self.path(filename))
+ return os.path.exists(path) and in_directory(path, self.file_dir)
def path(self, filename):
return os.path.join(self.file_dir, filename)
diff --git a/lib/galaxy/util/compression_utils.py b/lib/galaxy/util/compression_utils.py
index 7c70aaad644..e4e4734d4d3 100644
--- a/lib/galaxy/util/compression_utils.py
+++ b/lib/galaxy/util/compression_utils.py
@@ -1,5 +1,10 @@
+from __future__ import absolute_import
+
import gzip
import io
+import logging
+import os
+import tarfile
import zipfile
from .checkers import (
@@ -8,6 +13,8 @@ from .checkers import (
is_gzip
)
+log = logging.getLogger(__name__)
+
def get_fileobj(filename, mode="r", compressed_formats=None):
"""
@@ -45,3 +52,121 @@ def get_fileobj(filename, mode="r", compressed_formats=None):
return io.TextIOWrapper(fh, encoding='utf-8')
else:
return fh
+
+
+class CompressedFile(object):
+
+ def __init__(self, file_path, mode='r'):
+ if tarfile.is_tarfile(file_path):
+ self.file_type = 'tar'
+ elif zipfile.is_zipfile(file_path) and not file_path.endswith('.jar'):
+ self.file_type = 'zip'
+ self.file_name = os.path.splitext(os.path.basename(file_path))[0]
+ if self.file_name.endswith('.tar'):
+ self.file_name = os.path.splitext(self.file_name)[0]
+ self.type = self.file_type
+ method = 'open_%s' % self.file_type
+ if hasattr(self, method):
+ self.archive = getattr(self, method)(file_path, mode)
+ else:
+ raise NameError('File type %s specified, no open method found.' % self.file_type)
+
+ def extract(self, path):
+ '''Determine the path to which the archive should be extracted.'''
+ contents = self.getmembers()
+ extraction_path = path
+ common_prefix = ''
+ if len(contents) == 1:
+ # The archive contains a single file, return the extraction path.
+ if self.isfile(contents[0]):
+ extraction_path = os.path.join(path, self.file_name)
+ if not os.path.exists(extraction_path):
+ os.makedirs(extraction_path)
+ self.archive.extractall(extraction_path)
+ else:
+ # Get the common prefix for all the files in the archive. If the common prefix ends with a slash,
+ # or self.isdir() returns True, the archive contains a single directory with the desired contents.
+ # Otherwise, it contains multiple files and/or directories at the root of the archive.
+ common_prefix = os.path.commonprefix([self.getname(item) for item in contents])
+ if len(common_prefix) >= 1 and not common_prefix.endswith(os.sep) and self.isdir(self.getmember(common_prefix)):
+ common_prefix += os.sep
+ if not common_prefix.endswith(os.sep):
+ common_prefix = ''
+ extraction_path = os.path.join(path, self.file_name)
+ if not os.path.exists(extraction_path):
+ os.makedirs(extraction_path)
+ self.archive.extractall(extraction_path)
+ # Since .zip files store unix permissions separately, we need to iterate through the zip file
+ # and set permissions on extracted members.
+ if self.file_type == 'zip':
+ for zipped_file in contents:
+ filename = self.getname(zipped_file)
+ absolute_filepath = os.path.join(extraction_path, filename)
+ external_attributes = self.archive.getinfo(filename).external_attr
+ # The 2 least significant bytes are irrelevant, the next two contain unix permissions.
+ unix_permissions = external_attributes >> 16
+ if unix_permissions != 0:
+ if os.path.exists(absolute_filepath):
+ os.chmod(absolute_filepath, unix_permissions)
+ else:
+ log.warning("Unable to change permission on extracted file '%s' as it does not exist" % absolute_filepath)
+ return os.path.abspath(os.path.join(extraction_path, common_prefix))
+
+ def getmembers_tar(self):
+ return self.archive.getmembers()
+
+ def getmembers_zip(self):
+ return self.archive.infolist()
+
+ def getname_tar(self, item):
+ return item.name
+
+ def getname_zip(self, item):
+ return item.filename
+
+ def getmember(self, name):
+ for member in self.getmembers():
+ if self.getname(member) == name:
+ return member
+
+ def getmembers(self):
+ return getattr(self, 'getmembers_%s' % self.type)()
+
+ def getname(self, member):
+ return getattr(self, 'getname_%s' % self.type)(member)
+
+ def isdir(self, member):
+ return getattr(self, 'isdir_%s' % self.type)(member)
+
+ def isdir_tar(self, member):
+ return member.isdir()
+
+ def isdir_zip(self, member):
+ if member.filename.endswith(os.sep):
+ return True
+ return False
+
+ def isfile(self, member):
+ if not self.isdir(member):
+ return True
+ return False
+
+ def open_tar(self, filepath, mode):
+ return tarfile.open(filepath, mode, errorlevel=0)
+
+ def open_zip(self, filepath, mode):
+ return zipfile.ZipFile(filepath, mode)
+
+ def zipfile_ok(self, path_to_archive):
+ """
+ This function is a bit pedantic and not functionally necessary. It checks whether there is
+ no file pointing outside of the extraction, because ZipFile.extractall() has some potential
+ security holes. See python zipfile documentation for more details.
+ """
+ basename = os.path.realpath(os.path.dirname(path_to_archive))
+ zip_archive = zipfile.ZipFile(path_to_archive)
+ for member in zip_archive.namelist():
+ member_path = os.path.realpath(os.path.join(basename, member))
+ if not member_path.startswith(basename):
+ return False
+ return True
diff --git a/lib/galaxy/util/path/__init__.py b/lib/galaxy/util/path/__init__.py
index b1756514d24..9424c3965de 100644
--- a/lib/galaxy/util/path/__init__.py
+++ b/lib/galaxy/util/path/__init__.py
@@ -6,7 +6,10 @@ import errno
import imp
import logging
from functools import partial
-from grp import getgrgid
+try:
+ from grp import getgrgid
+except ImportError:
+ getgrgid = None
from itertools import starmap
from operator import getitem
from os import (
@@ -28,7 +31,10 @@ from os.path import (
relpath,
sep as separator,
)
-from pwd import getpwuid
+try:
+ from pwd import getpwuid
+except ImportError:
+ getpwuid = None
from six import iteritems, string_types
from six.moves import filter, map, zip
@@ -165,6 +171,9 @@ def __path_permission_for_user(path, username):
:type username: string
:param username: a username matching the systems username
"""
+ if getpwuid is None:
+ raise NotImplementedError("This functionality is not implemented for Windows.")
+
group_id_of_file = stat(path).st_gid
file_owner = getpwuid(stat(path).st_uid)
group_members = getgrgid(group_id_of_file).gr_mem
diff --git a/lib/galaxy/util/sanitize_html.py b/lib/galaxy/util/sanitize_html.py
index 525c0d8ed4d..3dd2d4c185f 100644
--- a/lib/galaxy/util/sanitize_html.py
+++ b/lib/galaxy/util/sanitize_html.py
@@ -1,427 +1,45 @@
"""
-HTML Sanitizer (ripped from feedparser)
+HTML Sanitizer (lists of acceptable_* ripped from feedparser)
"""
+import bleach
-import re
-import sgmllib
+_acceptable_elements = ['a', 'abbr', 'acronym', 'address', 'area', 'article',
+ 'aside', 'audio', 'b', 'big', 'blockquote', 'br', 'button', 'canvas',
+ 'caption', 'center', 'cite', 'code', 'col', 'colgroup', 'command',
+ 'datagrid', 'datalist', 'dd', 'del', 'details', 'dfn', 'dialog', 'dir',
+ 'div', 'dl', 'dt', 'em', 'event-source', 'fieldset', 'figure',
+ 'footer', 'font', 'form', 'header', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
+ 'hr', 'i', 'img', 'input', 'ins', 'keygen', 'kbd', 'label', 'legend',
+ 'li', 'm', 'map', 'menu', 'meter', 'multicol', 'nav', 'nextid', 'ol',
+ 'output', 'optgroup', 'option', 'p', 'pre', 'progress', 'q', 's',
+ 'samp', 'section', 'select', 'small', 'sound', 'source', 'spacer',
+ 'span', 'strike', 'strong', 'sub', 'sup', 'table', 'tbody', 'td',
+ 'textarea', 'time', 'tfoot', 'th', 'thead', 'tr', 'tt', 'u', 'ul',
+ 'var', 'video', 'noscript']
-from six import unichr
-from six.moves.html_entities import name2codepoint
-
-from galaxy.util import unicodify
-
-_cp1252 = {
- unichr(128): unichr(8364), # euro sign
- unichr(130): unichr(8218), # single low-9 quotation mark
- unichr(131): unichr(402), # latin small letter f with hook
- unichr(132): unichr(8222), # double low-9 quotation mark
- unichr(133): unichr(8230), # horizontal ellipsis
- unichr(134): unichr(8224), # dagger
- unichr(135): unichr(8225), # double dagger
- unichr(136): unichr(710), # modifier letter circumflex accent
- unichr(137): unichr(8240), # per mille sign
- unichr(138): unichr(352), # latin capital letter s with caron
- unichr(139): unichr(8249), # single left-pointing angle quotation mark
- unichr(140): unichr(338), # latin capital ligature oe
- unichr(142): unichr(381), # latin capital letter z with caron
- unichr(145): unichr(8216), # left single quotation mark
- unichr(146): unichr(8217), # right single quotation mark
- unichr(147): unichr(8220), # left double quotation mark
- unichr(148): unichr(8221), # right double quotation mark
- unichr(149): unichr(8226), # bullet
- unichr(150): unichr(8211), # en dash
- unichr(151): unichr(8212), # em dash
- unichr(152): unichr(732), # small tilde
- unichr(153): unichr(8482), # trade mark sign
- unichr(154): unichr(353), # latin small letter s with caron
- unichr(155): unichr(8250), # single right-pointing angle quotation mark
- unichr(156): unichr(339), # latin small ligature oe
- unichr(158): unichr(382), # latin small letter z with caron
- unichr(159): unichr(376)} # latin capital letter y with diaeresis
+_acceptable_attributes = ['abbr', 'accept', 'accept-charset', 'accesskey',
+ 'action', 'align', 'alt', 'autocomplete', 'autofocus', 'axis',
+ 'background', 'balance', 'bgcolor', 'bgproperties', 'border',
+ 'bordercolor', 'bordercolordark', 'bordercolorlight', 'bottompadding',
+ 'cellpadding', 'cellspacing', 'ch', 'challenge', 'char', 'charoff',
+ 'choff', 'charset', 'checked', 'cite', 'class', 'clear', 'color',
+ 'cols', 'colspan', 'compact', 'contenteditable', 'controls', 'coords',
+ 'data', 'datafld', 'datapagesize', 'datasrc', 'datetime', 'default',
+ 'delay', 'dir', 'disabled', 'draggable', 'dynsrc', 'enctype', 'end',
+ 'face', 'for', 'form', 'frame', 'galleryimg', 'gutter', 'headers',
+ 'height', 'hidefocus', 'hidden', 'high', 'href', 'hreflang', 'hspace',
+ 'icon', 'id', 'inputmode', 'ismap', 'keytype', 'label', 'leftspacing',
+ 'lang', 'list', 'longdesc', 'loop', 'loopcount', 'loopend',
+ 'loopstart', 'low', 'lowsrc', 'max', 'maxlength', 'media', 'method',
+ 'min', 'multiple', 'name', 'nohref', 'noshade', 'nowrap', 'open',
+ 'optimum', 'pattern', 'ping', 'point-size', 'prompt', 'pqg',
+ 'radiogroup', 'readonly', 'rel', 'repeat-max', 'repeat-min', 'replace',
+ 'required', 'rev', 'rightspacing', 'rows', 'rowspan', 'rules', 'scope',
+ 'selected', 'shape', 'size', 'span', 'src', 'start', 'step', 'summary',
+ 'suppress', 'tabindex', 'target', 'template', 'title', 'toppadding',
+ 'type', 'unselectable', 'usemap', 'urn', 'valign', 'value', 'variable',
+ 'volume', 'vspace', 'vrml', 'width', 'wrap', 'xml:lang']
-class _BaseHTMLProcessor(sgmllib.SGMLParser):
- special = re.compile('''[<>'"]''')
- bare_ampersand = re.compile("&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)")
- elements_no_end_tag = ['area', 'base', 'basefont', 'br', 'col', 'frame', 'hr',
- 'img', 'input', 'isindex', 'link', 'meta', 'param']
-
- def __init__(self, encoding, type):
- self.encoding = encoding
- self.type = type
- # if _debug: sys.stderr.write('entering BaseHTMLProcessor, encoding=%s\n' % self.encoding)
- sgmllib.SGMLParser.__init__(self)
-
- def reset(self):
- self.pieces = []
- sgmllib.SGMLParser.reset(self)
-
- def _shorttag_replace(self, match):
- tag = match.group(1)
- if tag in self.elements_no_end_tag:
- return '<' + tag + ' />'
- else:
- return '<' + tag + '>'
-
- def parse_starttag(self, i):
- j = sgmllib.SGMLParser.parse_starttag(self, i)
- if self.type == 'application/xhtml+xml':
- if j > 2 and self.rawdata[j - 2:j] == '/>':
- self.unknown_endtag(self.lasttag)
- return j
-
- def feed(self, data):
- data = re.compile(r'\s]+?)\s*/>', self._shorttag_replace, data)
- data = data.replace(''', "'")
- data = data.replace('"', '"')
- sgmllib.SGMLParser.feed(self, data)
- sgmllib.SGMLParser.close(self)
-
- def normalize_attrs(self, attrs):
- if not attrs:
- return attrs
- # utility method to be called by descendants
- attrs = dict([(k.lower(), v) for k, v in attrs]).items()
- attrs = [(k, k in ('rel', 'type') and v.lower() or v) for k, v in attrs]
- attrs.sort()
- return attrs
-
- def unknown_starttag(self, tag, attrs):
- # called for each start tag
- # attrs is a list of (attr, value) tuples
- # e.g. for , tag='pre', attrs=[('class', 'screen')]
- uattrs = []
- strattrs = ''
- if attrs:
- for key, value in attrs:
- value = value.replace('>', '>').replace('<', '<').replace('"', '"')
- value = self.bare_ampersand.sub("&", value)
- uattrs.append((key, value))
- strattrs = ''.join([' %s="%s"' % (k, v) for k, v in uattrs])
- if tag in self.elements_no_end_tag:
- self.pieces.append('<%s%s />' % (tag, strattrs))
- else:
- self.pieces.append('<%s%s>' % (tag, strattrs))
-
- def unknown_endtag(self, tag):
- # called for each end tag, e.g. for , tag will be 'pre'
- # Reconstruct the original end tag.
- if tag not in self.elements_no_end_tag:
- self.pieces.append("" % locals())
-
- def handle_charref(self, ref):
- # called for each character reference, e.g. for ' ', ref will be '160'
- # Reconstruct the original character reference.
- if ref.startswith('x'):
- value = unichr(int(ref[1:], 16))
- else:
- value = unichr(int(ref))
-
- if value in _cp1252.keys():
- self.pieces.append('&#%s;' % hex(ord(_cp1252[value]))[1:])
- else:
- self.pieces.append('&#%(ref)s;' % locals())
-
- def handle_entityref(self, ref):
- # called for each entity reference, e.g. for '©', ref will be 'copy'
- # Reconstruct the original entity reference.
- if ref in name2codepoint:
- self.pieces.append('&%(ref)s;' % locals())
- else:
- self.pieces.append('&%(ref)s' % locals())
-
- def handle_data(self, text):
- # called for each block of plain text, i.e. outside of any tag and
- # not containing any character or entity references
- # Store the original text verbatim.
- self.pieces.append(text)
-
- def handle_comment(self, text):
- # called for each HTML comment, e.g.
- # Reconstruct the original comment.
- self.pieces.append('' % locals())
-
- def handle_pi(self, text):
- # called for each processing instruction, e.g.
- # Reconstruct original processing instruction.
- self.pieces.append('' % locals())
-
- def handle_decl(self, text):
- # called for the DOCTYPE, if present, e.g.
- #
- # Reconstruct original DOCTYPE
- self.pieces.append('' % locals())
-
- _new_declname_match = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9:]*\s*').match
-
- def _scan_name(self, i, declstartpos):
- rawdata = self.rawdata
- n = len(rawdata)
- if i == n:
- return None, -1
- m = self._new_declname_match(rawdata, i)
- if m:
- s = m.group()
- name = s.strip()
- if (i + len(s)) == n:
- return None, -1 # end of buffer
- return name.lower(), m.end()
- else:
- self.handle_data(rawdata)
-# self.updatepos(declstartpos, i)
- return None, -1
-
- def convert_charref(self, name):
- return '&#%s;' % name
-
- def convert_entityref(self, name):
- return '&%s;' % name
-
- def output(self):
- '''Return processed HTML as a single string'''
- return ''.join(self.pieces)
-
-
-class _HTMLSanitizer(_BaseHTMLProcessor):
- acceptable_elements = ['a', 'abbr', 'acronym', 'address', 'area', 'article',
- 'aside', 'audio', 'b', 'big', 'blockquote', 'br', 'button', 'canvas',
- 'caption', 'center', 'cite', 'code', 'col', 'colgroup', 'command',
- 'datagrid', 'datalist', 'dd', 'del', 'details', 'dfn', 'dialog', 'dir',
- 'div', 'dl', 'dt', 'em', 'event-source', 'fieldset', 'figure', 'footer',
- 'font', 'form', 'header', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i',
- 'img', 'input', 'ins', 'keygen', 'kbd', 'label', 'legend', 'li', 'm', 'map',
- 'menu', 'meter', 'multicol', 'nav', 'nextid', 'ol', 'output', 'optgroup',
- 'option', 'p', 'pre', 'progress', 'q', 's', 'samp', 'section', 'select',
- 'small', 'sound', 'source', 'spacer', 'span', 'strike', 'strong', 'sub',
- 'sup', 'table', 'tbody', 'td', 'textarea', 'time', 'tfoot', 'th', 'thead',
- 'tr', 'tt', 'u', 'ul', 'var', 'video', 'noscript']
-
- acceptable_attributes = ['abbr', 'accept', 'accept-charset', 'accesskey',
- 'action', 'align', 'alt', 'autocomplete', 'autofocus', 'axis',
- 'background', 'balance', 'bgcolor', 'bgproperties', 'border',
- 'bordercolor', 'bordercolordark', 'bordercolorlight', 'bottompadding',
- 'cellpadding', 'cellspacing', 'ch', 'challenge', 'char', 'charoff',
- 'choff', 'charset', 'checked', 'cite', 'class', 'clear', 'color', 'cols',
- 'colspan', 'compact', 'contenteditable', 'controls', 'coords', 'data',
- 'datafld', 'datapagesize', 'datasrc', 'datetime', 'default', 'delay',
- 'dir', 'disabled', 'draggable', 'dynsrc', 'enctype', 'end', 'face', 'for',
- 'form', 'frame', 'galleryimg', 'gutter', 'headers', 'height', 'hidefocus',
- 'hidden', 'high', 'href', 'hreflang', 'hspace', 'icon', 'id', 'inputmode',
- 'ismap', 'keytype', 'label', 'leftspacing', 'lang', 'list', 'longdesc',
- 'loop', 'loopcount', 'loopend', 'loopstart', 'low', 'lowsrc', 'max',
- 'maxlength', 'media', 'method', 'min', 'multiple', 'name', 'nohref',
- 'noshade', 'nowrap', 'open', 'optimum', 'pattern', 'ping', 'point-size',
- 'prompt', 'pqg', 'radiogroup', 'readonly', 'rel', 'repeat-max',
- 'repeat-min', 'replace', 'required', 'rev', 'rightspacing', 'rows',
- 'rowspan', 'rules', 'scope', 'selected', 'shape', 'size', 'span', 'src',
- 'start', 'step', 'summary', 'suppress', 'tabindex', 'target', 'template',
- 'title', 'toppadding', 'type', 'unselectable', 'usemap', 'urn', 'valign',
- 'value', 'variable', 'volume', 'vspace', 'vrml', 'width', 'wrap',
- 'xml:lang']
-
- unacceptable_elements_with_end_tag = ['script', 'applet', 'style']
-
- acceptable_css_properties = ['azimuth', 'background-color',
- 'border-bottom-color', 'border-collapse', 'border-color',
- 'border-left-color', 'border-right-color', 'border-top-color', 'clear',
- 'color', 'cursor', 'direction', 'display', 'elevation', 'float', 'font',
- 'font-family', 'font-size', 'font-style', 'font-variant', 'font-weight',
- 'height', 'letter-spacing', 'line-height', 'overflow', 'pause',
- 'pause-after', 'pause-before', 'pitch', 'pitch-range', 'richness',
- 'speak', 'speak-header', 'speak-numeral', 'speak-punctuation',
- 'speech-rate', 'stress', 'text-align', 'text-decoration', 'text-indent',
- 'unicode-bidi', 'vertical-align', 'voice-family', 'volume',
- 'white-space', 'width']
-
- # survey of common keywords found in feeds
- acceptable_css_keywords = ['auto', 'aqua', 'black', 'block', 'blue',
- 'bold', 'both', 'bottom', 'brown', 'center', 'collapse', 'dashed',
- 'dotted', 'fuchsia', 'gray', 'green', '!important', 'italic', 'left',
- 'lime', 'maroon', 'medium', 'none', 'navy', 'normal', 'nowrap', 'olive',
- 'pointer', 'purple', 'red', 'right', 'solid', 'silver', 'teal', 'top',
- 'transparent', 'underline', 'white', 'yellow']
-
- valid_css_values = re.compile('^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|' +
- '\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$')
-
- mathml_elements = ['annotation', 'annotation-xml', 'maction', 'math',
- 'merror', 'mfenced', 'mfrac', 'mi', 'mmultiscripts', 'mn', 'mo', 'mover', 'mpadded',
- 'mphantom', 'mprescripts', 'mroot', 'mrow', 'mspace', 'msqrt', 'mstyle',
- 'msub', 'msubsup', 'msup', 'mtable', 'mtd', 'mtext', 'mtr', 'munder',
- 'munderover', 'none', 'semantics']
-
- mathml_attributes = ['actiontype', 'align', 'columnalign', 'columnalign',
- 'columnalign', 'close', 'columnlines', 'columnspacing', 'columnspan', 'depth',
- 'display', 'displaystyle', 'encoding', 'equalcolumns', 'equalrows',
- 'fence', 'fontstyle', 'fontweight', 'frame', 'height', 'linethickness',
- 'lspace', 'mathbackground', 'mathcolor', 'mathvariant', 'mathvariant',
- 'maxsize', 'minsize', 'open', 'other', 'rowalign', 'rowalign', 'rowalign',
- 'rowlines', 'rowspacing', 'rowspan', 'rspace', 'scriptlevel', 'selection',
- 'separator', 'separators', 'stretchy', 'width', 'width', 'xlink:href',
- 'xlink:show', 'xlink:type', 'xmlns', 'xmlns:xlink']
-
- # svgtiny - foreignObject + linearGradient + radialGradient + stop
- svg_elements = ['a', 'animate', 'animateColor', 'animateMotion',
- 'animateTransform', 'circle', 'defs', 'desc', 'ellipse', 'foreignObject',
- 'font-face', 'font-face-name', 'font-face-src', 'g', 'glyph', 'hkern',
- 'linearGradient', 'line', 'marker', 'metadata', 'missing-glyph', 'mpath',
- 'path', 'polygon', 'polyline', 'radialGradient', 'rect', 'set', 'stop',
- 'svg', 'switch', 'text', 'title', 'tspan', 'use']
-
- # svgtiny + class + opacity + offset + xmlns + xmlns:xlink
- svg_attributes = ['accent-height', 'accumulate', 'additive', 'alphabetic',
- 'arabic-form', 'ascent', 'attributeName', 'attributeType',
- 'baseProfile', 'bbox', 'begin', 'by', 'calcMode', 'cap-height',
- 'class', 'color', 'color-rendering', 'content', 'cx', 'cy', 'd', 'dx',
- 'dy', 'descent', 'display', 'dur', 'end', 'fill', 'fill-opacity',
- 'fill-rule', 'font-family', 'font-size', 'font-stretch', 'font-style',
- 'font-variant', 'font-weight', 'from', 'fx', 'fy', 'g1', 'g2',
- 'glyph-name', 'gradientUnits', 'hanging', 'height', 'horiz-adv-x',
- 'horiz-origin-x', 'id', 'ideographic', 'k', 'keyPoints', 'keySplines',
- 'keyTimes', 'lang', 'mathematical', 'marker-end', 'marker-mid',
- 'marker-start', 'markerHeight', 'markerUnits', 'markerWidth', 'max',
- 'min', 'name', 'offset', 'opacity', 'orient', 'origin',
- 'overline-position', 'overline-thickness', 'panose-1', 'path',
- 'pathLength', 'points', 'preserveAspectRatio', 'r', 'refX', 'refY',
- 'repeatCount', 'repeatDur', 'requiredExtensions', 'requiredFeatures',
- 'restart', 'rotate', 'rx', 'ry', 'slope', 'stemh', 'stemv',
- 'stop-color', 'stop-opacity', 'strikethrough-position',
- 'strikethrough-thickness', 'stroke', 'stroke-dasharray',
- 'stroke-dashoffset', 'stroke-linecap', 'stroke-linejoin',
- 'stroke-miterlimit', 'stroke-opacity', 'stroke-width', 'systemLanguage',
- 'target', 'text-anchor', 'to', 'transform', 'type', 'u1', 'u2',
- 'underline-position', 'underline-thickness', 'unicode', 'unicode-range',
- 'units-per-em', 'values', 'version', 'viewBox', 'visibility', 'width',
- 'widths', 'x', 'x-height', 'x1', 'x2', 'xlink:actuate', 'xlink:arcrole',
- 'xlink:href', 'xlink:role', 'xlink:show', 'xlink:title', 'xlink:type',
- 'xml:base', 'xml:lang', 'xml:space', 'xmlns', 'xmlns:xlink', 'y', 'y1',
- 'y2', 'zoomAndPan']
-
- svg_attr_map = None
- svg_elem_map = None
-
- acceptable_svg_properties = ['fill', 'fill-opacity', 'fill-rule',
- 'stroke', 'stroke-width', 'stroke-linecap', 'stroke-linejoin',
- 'stroke-opacity']
-
- def reset(self):
- _BaseHTMLProcessor.reset(self)
- self.unacceptablestack = 0
- self.mathmlOK = 0
- self.svgOK = 0
-
- def unknown_starttag(self, tag, attrs):
- acceptable_attributes = self.acceptable_attributes
- keymap = {}
- if tag not in self.acceptable_elements or self.svgOK:
- if tag in self.unacceptable_elements_with_end_tag:
- self.unacceptablestack += 1
-
- # not otherwise acceptable, perhaps it is MathML or SVG?
- if tag == 'math' and ('xmlns', 'http://www.w3.org/1998/Math/MathML') in attrs:
- self.mathmlOK += 1
- if tag == 'svg' and ('xmlns', 'http://www.w3.org/2000/svg') in attrs:
- self.svgOK += 1
-
- # chose acceptable attributes based on tag class, else bail
- if self.mathmlOK and tag in self.mathml_elements:
- acceptable_attributes = self.mathml_attributes
- elif self.svgOK and tag in self.svg_elements:
- # for most vocabularies, lowercasing is a good idea. Many
- # svg elements, however, are camel case
- if not self.svg_attr_map:
- lower = [attr.lower() for attr in self.svg_attributes]
- mix = [a for a in self.svg_attributes if a not in lower]
- self.svg_attributes = lower
- self.svg_attr_map = dict([(a.lower(), a) for a in mix])
-
- lower = [attr.lower() for attr in self.svg_elements]
- mix = [a for a in self.svg_elements if a not in lower]
- self.svg_elements = lower
- self.svg_elem_map = dict([(a.lower(), a) for a in mix])
- acceptable_attributes = self.svg_attributes
- tag = self.svg_elem_map.get(tag, tag)
- keymap = self.svg_attr_map
- elif tag not in self.acceptable_elements:
- return
-
- # declare xlink namespace, if needed
- if self.mathmlOK or self.svgOK:
- if any(map(lambda n, v: n.startswith('xlink:'), attrs)):
- if not ('xmlns:xlink', 'http://www.w3.org/1999/xlink') in attrs:
- attrs.append(('xmlns:xlink', 'http://www.w3.org/1999/xlink'))
-
- clean_attrs = []
- for key, value in self.normalize_attrs(attrs):
- if key == "href" and value.strip().startswith("javascript"):
- pass
- elif key in acceptable_attributes:
- key = keymap.get(key, key)
- clean_attrs.append((key, value))
- elif key == 'style':
- pass
- # clean_value = self.sanitize_style(value)
- # if clean_value: clean_attrs.append((key,clean_value))
- _BaseHTMLProcessor.unknown_starttag(self, tag, clean_attrs)
-
- def unknown_endtag(self, tag):
- if tag not in self.acceptable_elements:
- if tag in self.unacceptable_elements_with_end_tag:
- self.unacceptablestack -= 1
- if self.mathmlOK and tag in self.mathml_elements:
- if tag == 'math' and self.mathmlOK:
- self.mathmlOK -= 1
- elif self.svgOK and tag in self.svg_elements:
- tag = self.svg_elem_map.get(tag, tag)
- if tag == 'svg' and self.svgOK:
- self.svgOK -= 1
- else:
- return
- _BaseHTMLProcessor.unknown_endtag(self, tag)
-
- def handle_pi(self, text):
- pass
-
- def handle_decl(self, text):
- pass
-
- def handle_data(self, text):
- if not self.unacceptablestack:
- _BaseHTMLProcessor.handle_data(self, text)
-
- def sanitize_style(self, style):
- # disallow urls
- style = re.compile('url\s*\(\s*[^\s)]+?\s*\)\s*').sub(' ', style)
-
- # gauntlet
- if not re.match("""^([:,;#%.\sa-zA-Z0-9!]|\w-\w|'[\s\w]+'|"[\s\w]+"|\([\d,\s]+\))*$""", style):
- return ''
- if not re.match("^(\s*[-\w]+\s*:\s*[^:;]*(;|$))*$", style):
- return ''
-
- clean = []
- for prop, value in re.findall("([-\w]+)\s*:\s*([^:;]*)", style):
- if not value:
- continue
- if prop.lower() in self.acceptable_css_properties:
- clean.append(prop + ': ' + value + ';')
- elif prop.split('-')[0].lower() in ['background', 'border', 'margin', 'padding']:
- for keyword in value.split():
- if keyword not in self.acceptable_css_keywords and \
- not self.valid_css_values.match(keyword):
- break
- else:
- clean.append(prop + ': ' + value + ';')
- elif self.svgOK and prop.lower() in self.acceptable_svg_properties:
- clean.append(prop + ': ' + value + ';')
- return ' '.join(clean)
-
-
-def sanitize_html(htmlSource, encoding="utf-8", type="text/html"):
- p = _HTMLSanitizer(encoding, type)
- p.feed(unicodify(htmlSource, encoding))
- data = p.output()
- data = data.strip().replace('\r\n', '\n')
- return data
+def sanitize_html(htmlSource):
+ return bleach.clean(htmlSource, tags=_acceptable_elements, attributes=_acceptable_attributes, strip=True)
diff --git a/lib/galaxy/util/validation.py b/lib/galaxy/util/validation.py
index a851148c5e5..1ecf60d129e 100644
--- a/lib/galaxy/util/validation.py
+++ b/lib/galaxy/util/validation.py
@@ -12,13 +12,13 @@ def validate_and_sanitize_basestring(key, val):
if not isinstance(val, string_types):
raise exceptions.RequestParameterInvalidException('%s must be a string or unicode: %s'
% (key, str(type(val))))
- return sanitize_html(val, 'utf-8', 'text/html')
+ return sanitize_html(val)
def validate_and_sanitize_basestring_list(key, val):
try:
assert isinstance(val, list)
- return [sanitize_html(t, 'utf-8', 'text/html') for t in val]
+ return [sanitize_html(t) for t in val]
except (AssertionError, TypeError):
raise exceptions.RequestParameterInvalidException('%s must be a list of strings: %s'
% (key, str(type(val))))
@@ -32,7 +32,7 @@ def validate_boolean(key, val):
# TODO:
-# def validate_integer( self, key, val, min, max ):
-# def validate_float( self, key, val, min, max ):
-# def validate_number( self, key, val, min, max ):
-# def validate_genome_build( self, key, val ):
+# def validate_integer(self, key, val, min, max):
+# def validate_float(self, key, val, min, max):
+# def validate_number(self, key, val, min, max):
+# def validate_genome_build(self, key, val):
diff --git a/lib/galaxy/visualization/plugins/interactive_environments.py b/lib/galaxy/visualization/plugins/interactive_environments.py
index edb108eb864..10cb067282d 100644
--- a/lib/galaxy/visualization/plugins/interactive_environments.py
+++ b/lib/galaxy/visualization/plugins/interactive_environments.py
@@ -6,6 +6,7 @@ import shlex
import stat
import string
import tempfile
+import time
import uuid
from itertools import product
from subprocess import PIPE, Popen
@@ -442,7 +443,20 @@ class InteractiveEnvironmentRequest(object):
"""
run_args = self.container_run_args(image, env_override, volumes)
container = self.attr.container_interface.run_in_container(None, **run_args)
- container_port = self._find_port_mapping(container.ports)
+ attempt = 0
+ container_ports = container.ports
+ while container_ports is None and attempt < 30:
+ # TODO: it would be better to do this in /interactive_environments/ready so the client doesn't block here,
+ # but _find_port_mapping needs certain non-persisted data (the port configured to be published) and the
+ # proxy manager doesn't have an update method, so that'd require bigger changes than I have the time for
+ # right now
+ attempt += 1
+ log.warning("Sleeping for 2 seconds while waiting for container %s ports", container.id)
+ time.sleep(2)
+ container_ports = container.ports
+ if container_ports is None:
+ raise Exception("Failed to determine ports for container '%s' after 30 attempts" % container.id)
+ container_port = self._find_port_mapping(container_ports)
log.debug("Container '%s' accessible at: %s:%s", container.id, container_port.hostaddr, container_port.hostport)
self.attr.proxy_request = self.trans.app.proxy_manager.setup_proxy(
self.trans,
diff --git a/lib/galaxy/visualization/plugins/plugin.py b/lib/galaxy/visualization/plugins/plugin.py
index 443dbfdf7c7..c5ba7fbb583 100644
--- a/lib/galaxy/visualization/plugins/plugin.py
+++ b/lib/galaxy/visualization/plugins/plugin.py
@@ -269,6 +269,14 @@ class InteractiveEnvironmentPlugin(VisualizationPlugin):
context['base_url'] = 'interactive_environments'
super(InteractiveEnvironmentPlugin, self).__init__(app, path, name, config, context=context, **kwargs)
+ def _error_template(self, trans):
+ return trans.fill_template('message.mako',
+ message='Loading the interactive environment failed, please contact the {admin_tag} for assistance'.format(
+ admin_tag='Galaxy administrator'.format(
+ admin_mail=trans.app.config.error_email_to)
+ if trans.app.config.error_email_to else 'Galaxy administrator'),
+ status='error')
+
def _render(self, render_vars, trans=None, embedded=None, **kwargs):
"""
Override to add interactive environment specific template vars.
@@ -294,16 +302,15 @@ class InteractiveEnvironmentPlugin(VisualizationPlugin):
request = self.INTENV_REQUEST_FACTORY(trans, self)
except Exception:
log.exception("IE plugin request handling failed")
- return trans.fill_template('message.mako',
- message='Loading the interactive environment failed, please contact the {admin_tag} for assistance'.format(
- admin_tag='Galaxy administrator'.format(
- admin_mail=trans.app.config.error_email_to)
- if trans.app.config.error_email_to else 'Galaxy administrator'),
- status='error')
+ return self._error_template(trans)
render_vars["ie_request"] = request
template_filename = self.config['entry_point']['file']
- return trans.fill_template(template_filename, template_lookup=self.template_lookup, **render_vars)
+ try:
+ return trans.fill_template(template_filename, template_lookup=self.template_lookup, **render_vars)
+ except Exception:
+ log.exception("IE plugin template fill failed")
+ return self._error_template(trans)
class ScriptVisualizationPlugin(VisualizationPlugin):
diff --git a/lib/galaxy/visualization/plugins/resource_parser.py b/lib/galaxy/visualization/plugins/resource_parser.py
index 1ac6b65464a..ebc1754a277 100644
--- a/lib/galaxy/visualization/plugins/resource_parser.py
+++ b/lib/galaxy/visualization/plugins/resource_parser.py
@@ -29,7 +29,7 @@ class ResourceParser(object):
new keys (e.g. dataset_id="NNN" -> hda=).
"""
primitive_parsers = {
- 'str' : lambda param: galaxy.util.sanitize_html.sanitize_html(param, 'utf-8'),
+ 'str' : lambda param: galaxy.util.sanitize_html.sanitize_html(param),
'bool' : lambda param: galaxy.util.string_as_bool(param),
'int' : int,
'float' : float,
@@ -217,7 +217,7 @@ class ResourceParser(object):
# TODO: ideally this would check v. a list of valid dbkeys
elif param_type == 'dbkey':
dbkey = query_param
- parsed_param = galaxy.util.sanitize_html.sanitize_html(dbkey, 'utf-8')
+ parsed_param = galaxy.util.sanitize_html.sanitize_html(dbkey)
return parsed_param
diff --git a/lib/galaxy/web/base/controller.py b/lib/galaxy/web/base/controller.py
index 10c2fd4c01b..a7f25f5844e 100644
--- a/lib/galaxy/web/base/controller.py
+++ b/lib/galaxy/web/base/controller.py
@@ -517,7 +517,7 @@ class UsesLibraryMixinItems(SharableItemSecurityMixin):
# PRECONDITION: folder_id has already been altered to remove the folder prefix ('F')
# TODO: allow name and other, editable ldda attrs?
if ldda_message:
- ldda_message = util.sanitize_html.sanitize_html(ldda_message, 'utf-8')
+ ldda_message = sanitize_html(ldda_message)
# check permissions on (all three?) resources: hda, library, folder
# TODO: do we really need the library??
@@ -1121,7 +1121,7 @@ class UsesVisualizationMixin(UsesLibraryMixinItems):
else:
self.create_item_slug(trans.sa_session, visualization)
if annotation:
- annotation = sanitize_html(annotation, 'utf-8', 'text/html')
+ annotation = sanitize_html(annotation)
# TODO: if this is to stay in the mixin, UsesAnnotations should be added to the superclasses
# right now this is depending on the classes that include this mixin to have UsesAnnotations
self.add_item_annotation(trans.sa_session, trans.user, visualization, annotation)
diff --git a/lib/galaxy/web/framework/decorators.py b/lib/galaxy/web/framework/decorators.py
index ef23ed0a127..9ce0ea099c2 100644
--- a/lib/galaxy/web/framework/decorators.py
+++ b/lib/galaxy/web/framework/decorators.py
@@ -134,7 +134,7 @@ def expose_api(func, to_json=True, user_required=True):
trans.response.headers['Cache-Control'] = "max-age=0,no-cache,no-store"
# Perform api_run_as processing, possibly changing identity
- if 'payload' in kwargs and 'run_as' in kwargs['payload']:
+ if 'payload' in kwargs and isinstance(kwargs['payload'], dict) and 'run_as' in kwargs['payload']:
if not trans.user_can_do_run_as():
error_message = 'User does not have permissions to run jobs as another user'
return error
diff --git a/lib/galaxy/webapps/galaxy/api/_fetch_util.py b/lib/galaxy/webapps/galaxy/api/_fetch_util.py
new file mode 100644
index 00000000000..7c5e2ea8e54
--- /dev/null
+++ b/lib/galaxy/webapps/galaxy/api/_fetch_util.py
@@ -0,0 +1,217 @@
+import logging
+import os
+
+from galaxy.actions.library import (
+ validate_path_upload,
+ validate_server_directory_upload,
+)
+from galaxy.exceptions import (
+ RequestParameterInvalidException
+)
+from galaxy.tools.actions.upload_common import validate_url
+from galaxy.util import (
+ relpath,
+)
+
+log = logging.getLogger(__name__)
+
+VALID_DESTINATION_TYPES = ["library", "library_folder", "hdca", "hdas"]
+ELEMENTS_FROM_TYPE = ["archive", "bagit", "bagit_archive", "directory"]
+# These elements_from cannot be sym linked to because they only exist during upload.
+ELEMENTS_FROM_TRANSIENT_TYPES = ["archive", "bagit_archive"]
+
+
+def validate_and_normalize_targets(trans, payload):
+ """Validate and normalize all src references in fetch targets.
+
+ - Normalize ftp_import and server_dir src entries into simple path entires
+ with the relevant paths resolved and permissions / configuration checked.
+ - Check for file:// URLs in items src of "url" and convert them into path
+ src items - after verifying path pastes are allowed and user is admin.
+ - Check for valid URLs to be fetched for http and https entries.
+ - Based on Galaxy configuration and upload types set purge_source and in_place
+ as needed for each upload.
+ """
+ targets = payload.get("targets", [])
+
+ for target in targets:
+ destination = _get_required_item(target, "destination", "Each target must specify a 'destination'")
+ destination_type = _get_required_item(destination, "type", "Each target destination must specify a 'type'")
+ if "object_id" in destination:
+ raise RequestParameterInvalidException("object_id not allowed to appear in the request.")
+
+ if destination_type not in VALID_DESTINATION_TYPES:
+ template = "Invalid target destination type [%s] encountered, must be one of %s"
+ msg = template % (destination_type, VALID_DESTINATION_TYPES)
+ raise RequestParameterInvalidException(msg)
+ if destination_type == "library":
+ library_name = _get_required_item(destination, "name", "Must specify a library name")
+ description = destination.get("description", "")
+ synopsis = destination.get("synopsis", "")
+ library = trans.app.library_manager.create(
+ trans, library_name, description=description, synopsis=synopsis
+ )
+ destination["type"] = "library_folder"
+ for key in ["name", "description", "synopsis"]:
+ if key in destination:
+ del destination[key]
+ destination["library_folder_id"] = trans.app.security.encode_id(library.root_folder.id)
+
+ # Unlike upload.py we don't transmit or use run_as_real_user in the job - we just make sure
+ # in_place and purge_source are set on the individual upload fetch sources as needed based
+ # on this.
+ run_as_real_user = trans.app.config.external_chown_script is not None # See comment in upload.py
+ purge_ftp_source = getattr(trans.app.config, 'ftp_upload_purge', True) and not run_as_real_user
+
+ payload["check_content"] = trans.app.config.check_upload_content
+
+ def check_src(item):
+ if "object_id" in item:
+ raise RequestParameterInvalidException("object_id not allowed to appear in the request.")
+
+ # Normalize file:// URLs into paths.
+ if item["src"] == "url" and item["url"].startswith("file://"):
+ item["src"] = "path"
+ item["path"] = item["url"][len("file://"):]
+ del item["path"]
+
+ if "in_place" in item:
+ raise RequestParameterInvalidException("in_place cannot be set in the upload request")
+
+ src = item["src"]
+
+ # Check link_data_only can only be set for certain src types and certain elements_from types.
+ _handle_invalid_link_data_only_elements_type(item)
+ if src not in ["path", "server_dir"]:
+ _handle_invalid_link_data_only_type(item)
+ elements_from = item.get("elements_from", None)
+ if elements_from and elements_from not in ELEMENTS_FROM_TYPE:
+ raise RequestParameterInvalidException("Invalid elements_from/items_from found in request")
+
+ if src == "path" or (src == "url" and item["url"].startswith("file:")):
+ # Validate is admin, leave alone.
+ validate_path_upload(trans)
+ elif src == "server_dir":
+ # Validate and replace with path definition.
+ server_dir = item["server_dir"]
+ full_path, _ = validate_server_directory_upload(trans, server_dir)
+ item["src"] = "path"
+ item["path"] = full_path
+ elif src == "ftp_import":
+ ftp_path = item["ftp_path"]
+ full_path = None
+
+ # It'd be nice if this can be de-duplicated with what is in parameters/grouping.py.
+ user_ftp_dir = trans.user_ftp_dir
+ is_directory = False
+
+ assert not os.path.islink(user_ftp_dir), "User FTP directory cannot be a symbolic link"
+ for (dirpath, dirnames, filenames) in os.walk(user_ftp_dir):
+ for filename in filenames:
+ if ftp_path == filename:
+ path = relpath(os.path.join(dirpath, filename), user_ftp_dir)
+ if not os.path.islink(os.path.join(dirpath, filename)):
+ full_path = os.path.abspath(os.path.join(user_ftp_dir, path))
+ break
+
+ for dirname in dirnames:
+ if ftp_path == dirname:
+ path = relpath(os.path.join(dirpath, dirname), user_ftp_dir)
+ if not os.path.islink(os.path.join(dirpath, dirname)):
+ full_path = os.path.abspath(os.path.join(user_ftp_dir, path))
+ is_directory = True
+ break
+
+ if is_directory:
+ # If the target is a directory - make sure no files under it are symbolic links
+ for (dirpath, dirnames, filenames) in os.walk(full_path):
+ for filename in filenames:
+ if ftp_path == filename:
+ path = relpath(os.path.join(dirpath, filename), full_path)
+ if not os.path.islink(os.path.join(dirpath, filename)):
+ full_path = False
+ break
+
+ for dirname in dirnames:
+ if ftp_path == dirname:
+ path = relpath(os.path.join(dirpath, filename), full_path)
+ if not os.path.islink(os.path.join(dirpath, filename)):
+ full_path = False
+ break
+
+ if not full_path:
+ raise RequestParameterInvalidException("Failed to find referenced ftp_path or symbolic link was enountered")
+
+ item["src"] = "path"
+ item["path"] = full_path
+ item["purge_source"] = purge_ftp_source
+ elif src == "url":
+ url = item["url"]
+ looks_like_url = False
+ for url_prefix in ["http://", "https://", "ftp://", "ftps://"]:
+ if url.startswith(url_prefix):
+ looks_like_url = True
+ break
+
+ if not looks_like_url:
+ raise RequestParameterInvalidException("Invalid URL [%s] found in src definition." % url)
+
+ validate_url(url, trans.app.config.fetch_url_whitelist_ips)
+ item["in_place"] = run_as_real_user
+ elif src == "files":
+ item["in_place"] = run_as_real_user
+
+ # Small disagreement with traditional uploads - we purge less by default since whether purging
+ # happens varies based on upload options in non-obvious ways.
+ # https://github.com/galaxyproject/galaxy/issues/5361
+ if "purge_source" not in item:
+ item["purge_source"] = False
+
+ _replace_request_syntax_sugar(targets)
+ _for_each_src(check_src, targets)
+
+
+def _replace_request_syntax_sugar(obj):
+ # For data libraries and hdas to make sense - allow items and items_from in place of elements
+ # and elements_from. This is destructive and modifies the supplied request.
+ if isinstance(obj, list):
+ for el in obj:
+ _replace_request_syntax_sugar(el)
+ elif isinstance(obj, dict):
+ if "items" in obj:
+ obj["elements"] = obj["items"]
+ del obj["items"]
+ if "items_from" in obj:
+ obj["elements_from"] = obj["items_from"]
+ del obj["items_from"]
+ for value in obj.values():
+ _replace_request_syntax_sugar(value)
+
+
+def _handle_invalid_link_data_only_type(item):
+ link_data_only = item.get("link_data_only", False)
+ if link_data_only:
+ raise RequestParameterInvalidException("link_data_only is invalid for src type [%s]" % item.get("src"))
+
+
+def _handle_invalid_link_data_only_elements_type(item):
+ link_data_only = item.get("link_data_only", False)
+ if link_data_only and item.get("elements_from", False) in ELEMENTS_FROM_TRANSIENT_TYPES:
+ raise RequestParameterInvalidException("link_data_only is invalid for derived elements from [%s]" % item.get("elements_from"))
+
+
+def _get_required_item(from_dict, key, message):
+ if key not in from_dict:
+ raise RequestParameterInvalidException(message)
+ return from_dict[key]
+
+
+def _for_each_src(f, obj):
+ if isinstance(obj, list):
+ for item in obj:
+ _for_each_src(f, item)
+ if isinstance(obj, dict):
+ if "src" in obj:
+ f(obj)
+ for key, value in obj.items():
+ _for_each_src(f, value)
diff --git a/lib/galaxy/webapps/galaxy/api/annotations.py b/lib/galaxy/webapps/galaxy/api/annotations.py
index 50ea8f0931c..cfe9872ac61 100644
--- a/lib/galaxy/webapps/galaxy/api/annotations.py
+++ b/lib/galaxy/webapps/galaxy/api/annotations.py
@@ -8,7 +8,7 @@ from galaxy import (
managers
)
from galaxy.model.item_attrs import UsesAnnotations
-from galaxy.util import sanitize_html
+from galaxy.util.sanitize_html import sanitize_html
from galaxy.web import _future_expose_api as expose_api
from galaxy.web.base.controller import (
BaseAPIController,
@@ -36,7 +36,7 @@ class BaseAnnotationsController(BaseAPIController, UsesStoredWorkflowMixin, Uses
if item is not None:
new_annotation = payload.get("text")
# TODO: sanitize on display not entry
- new_annotation = sanitize_html.sanitize_html(new_annotation, 'utf-8', 'text/html')
+ new_annotation = sanitize_html(new_annotation)
self.add_item_annotation(trans.sa_session, trans.get_user(), item, new_annotation)
trans.sa_session.flush()
diff --git a/lib/galaxy/webapps/galaxy/api/folder_contents.py b/lib/galaxy/webapps/galaxy/api/folder_contents.py
index 041892c726b..b012d4a4c59 100644
--- a/lib/galaxy/webapps/galaxy/api/folder_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/folder_contents.py
@@ -275,7 +275,7 @@ class FolderContentsController(BaseAPIController, UsesLibraryMixin, UsesLibraryM
from_hdca_id = payload.pop('from_hdca_id', None)
ldda_message = payload.pop('ldda_message', '')
if ldda_message:
- ldda_message = util.sanitize_html.sanitize_html(ldda_message, 'utf-8')
+ ldda_message = util.sanitize_html.sanitize_html(ldda_message)
try:
if from_hda_id:
decoded_hda_id = self.decode_id(from_hda_id)
diff --git a/lib/galaxy/webapps/galaxy/api/library_contents.py b/lib/galaxy/webapps/galaxy/api/library_contents.py
index 5b873f24087..58201190f5c 100644
--- a/lib/galaxy/webapps/galaxy/api/library_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/library_contents.py
@@ -185,7 +185,8 @@ class LibraryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibrary
* upload_option: (optional) one of 'upload_file' (default), 'upload_directory' or 'upload_paths'
* server_dir: (optional, only if upload_option is
'upload_directory') relative path of the subdirectory of Galaxy
- ``library_import_dir`` to upload. All and only the files (i.e.
+ ``library_import_dir`` (if admin) or ``user_library_import_dir``
+ (if non-admin) to upload. All and only the files (i.e.
no subdirectories) contained in the specified directory will be
uploaded.
* filesystem_paths: (optional, only if upload_option is
diff --git a/lib/galaxy/webapps/galaxy/api/page_revisions.py b/lib/galaxy/webapps/galaxy/api/page_revisions.py
index 31e8969eed5..978585ebec1 100644
--- a/lib/galaxy/webapps/galaxy/api/page_revisions.py
+++ b/lib/galaxy/webapps/galaxy/api/page_revisions.py
@@ -66,7 +66,7 @@ class PageRevisionsController(BaseAPIController, SharableItemSecurityMixin, Uses
else:
title = page.title
- content = sanitize_html(content, 'utf-8', 'text/html')
+ content = sanitize_html(content)
page_revision = trans.app.model.PageRevision()
page_revision.title = title
diff --git a/lib/galaxy/webapps/galaxy/api/pages.py b/lib/galaxy/webapps/galaxy/api/pages.py
index baf046cf86a..73eb5c9cc5a 100644
--- a/lib/galaxy/webapps/galaxy/api/pages.py
+++ b/lib/galaxy/webapps/galaxy/api/pages.py
@@ -81,13 +81,13 @@ class PagesController(BaseAPIController, SharableItemSecurityMixin, UsesAnnotati
raise exceptions.DuplicatedSlugException("Page slug must be unique")
content = payload.get("content", "")
- content = sanitize_html(content, 'utf-8', 'text/html')
+ content = sanitize_html(content)
# Create the new stored page
page = trans.app.model.Page()
page.title = payload['title']
page.slug = payload['slug']
- page_annotation = sanitize_html(payload.get("annotation", ""), 'utf-8', 'text/html')
+ page_annotation = sanitize_html(payload.get("annotation", ""))
self.add_item_annotation(trans.sa_session, trans.get_user(), page, page_annotation)
page.user = user
# And the first (empty) page revision
diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py
index 412cc4082d8..08576ee812d 100644
--- a/lib/galaxy/webapps/galaxy/api/tools.py
+++ b/lib/galaxy/webapps/galaxy/api/tools.py
@@ -6,15 +6,23 @@ from six.moves.urllib.parse import unquote_plus
import galaxy.queue_worker
from galaxy import exceptions, managers, util, web
from galaxy.managers.collections_util import dictify_dataset_collection_instance
+from galaxy.util.json import safe_dumps
+from galaxy.util.odict import odict
from galaxy.visualization.genomes import GenomeRegion
from galaxy.web import _future_expose_api as expose_api
from galaxy.web import _future_expose_api_anonymous as expose_api_anonymous
from galaxy.web import _future_expose_api_anonymous_and_sessionless as expose_api_anonymous_and_sessionless
+from galaxy.web import _future_expose_api_raw_anonymous_and_sessionless as expose_api_raw_anonymous_and_sessionless
from galaxy.web.base.controller import BaseAPIController
from galaxy.web.base.controller import UsesVisualizationMixin
+from ._fetch_util import validate_and_normalize_targets
log = logging.getLogger(__name__)
+# Do not allow these tools to be called directly - they (it) enforces extra security and
+# provides access via a different API endpoint.
+PROTECTED_TOOLS = ["__DATA_FETCH__"]
+
class ToolsController(BaseAPIController, UsesVisualizationMixin):
"""
@@ -98,6 +106,74 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin):
tool = self._get_tool(id, tool_version=tool_version, user=trans.user)
return tool.to_json(trans, kwd.get('inputs', kwd))
+ @expose_api
+ @web.require_admin
+ def test_data_path(self, trans, id, **kwd):
+ """
+ GET /api/tools/{tool_id}/test_data_path?tool_version={tool_version}
+ """
+ # TODO: eliminate copy and paste with above code.
+ if 'payload' in kwd:
+ kwd = kwd.get('payload')
+ tool_version = kwd.get('tool_version', None)
+ tool = self._get_tool(id, tool_version=tool_version, user=trans.user)
+ path = tool.test_data_path(kwd.get("filename"))
+ if path:
+ return path
+ else:
+ raise exceptions.ObjectNotFound("Specified test data path not found.")
+
+ @expose_api_anonymous_and_sessionless
+ def tests_summary(self, trans, **kwd):
+ """
+ GET /api/tools/tests_summary
+
+ Fetch summary information for each tool and version combination with tool tests
+ defined. This summary information currently includes tool name and a count of
+ the tests.
+
+ Fetch complete test data for each tool with /api/tools/{tool_id}/test_data?tool_version=
+ """
+ test_counts_by_tool = {}
+ for id, tool in self.app.toolbox.tools():
+ tests = tool.tests
+ if tests:
+ if tool.id not in test_counts_by_tool:
+ test_counts_by_tool[tool.id] = {}
+ available_versions = test_counts_by_tool[tool.id]
+ available_versions[tool.version] = {
+ "tool_name": tool.name,
+ "count": len(tests),
+ }
+ return test_counts_by_tool
+
+ @expose_api_raw_anonymous_and_sessionless
+ def test_data(self, trans, id, **kwd):
+ """
+ GET /api/tools/{tool_id}/test_data?tool_version={tool_version}
+
+ This API endpoint is unstable and experimental. In particular the format of the
+ response has not been entirely nailed down (it exposes too many Galaxy
+ internals/Pythonisms in a rough way). If this endpoint is being used from outside
+ of scripts shipped with Galaxy let us know and please be prepared for the response
+ from this API to change its format in some ways.
+ """
+ # TODO: eliminate copy and paste with above code.
+ if 'payload' in kwd:
+ kwd = kwd.get('payload')
+ tool_version = kwd.get('tool_version', None)
+ tool = self._get_tool(id, tool_version=tool_version, user=trans.user)
+
+ # Encode in this method to handle odict objects in tool representation.
+ def json_encodeify(obj):
+ if isinstance(obj, odict):
+ return dict(obj)
+ else:
+ return obj
+
+ result = [t.to_dict() for t in tool.tests]
+ return safe_dumps(result, default=json_encodeify)
+
@expose_api
@web.require_admin
def reload(self, trans, id, **kwd):
@@ -290,12 +366,52 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin):
trans.response.headers["Content-Disposition"] = 'attachment; filename="%s.tgz"' % (id)
return download_file
+ @expose_api_anonymous
+ def fetch(self, trans, payload, **kwd):
+ """Adapt clean API to tool-constrained API.
+ """
+ log.info("Keywords are %s" % payload)
+ request_version = '1'
+ history_id = payload.pop("history_id")
+ clean_payload = {}
+ files_payload = {}
+ for key, value in payload.items():
+ if key == "key":
+ continue
+ if key.startswith('files_') or key.startswith('__files_'):
+ files_payload[key] = value
+ continue
+ clean_payload[key] = value
+ log.info("payload %s" % clean_payload)
+ validate_and_normalize_targets(trans, clean_payload)
+ clean_payload["check_content"] = trans.app.config.check_upload_content
+ request = dumps(clean_payload)
+ log.info(request)
+ create_payload = {
+ 'tool_id': "__DATA_FETCH__",
+ 'history_id': history_id,
+ 'inputs': {
+ 'request_version': request_version,
+ 'request_json': request,
+ },
+ }
+ create_payload.update(files_payload)
+ return self._create(trans, create_payload, **kwd)
+
@expose_api_anonymous
def create(self, trans, payload, **kwd):
"""
POST /api/tools
Executes tool using specified inputs and returns tool's outputs.
"""
+ tool_id = payload.get("tool_id")
+ if tool_id in PROTECTED_TOOLS:
+ raise exceptions.RequestParameterInvalidException("Cannot execute tool [%s] directly, must use alternative endpoint." % tool_id)
+ if tool_id is None:
+ raise exceptions.RequestParameterInvalidException("Must specify a valid tool_id to use this endpoint.")
+ return self._create(trans, payload, **kwd)
+
+ def _create(self, trans, payload, **kwd):
# HACK: for now, if action is rerun, rerun tool.
action = payload.get('action', None)
if action == 'rerun':
diff --git a/lib/galaxy/webapps/galaxy/api/uploads.py b/lib/galaxy/webapps/galaxy/api/uploads.py
new file mode 100644
index 00000000000..bedf6daf061
--- /dev/null
+++ b/lib/galaxy/webapps/galaxy/api/uploads.py
@@ -0,0 +1,53 @@
+"""
+API operations for uploaded files in storage.
+"""
+import logging
+import os
+import re
+
+from galaxy.exceptions import MessageException, NotImplemented
+from galaxy.web import expose_api_anonymous
+from galaxy.web.base.controller import BaseAPIController
+
+log = logging.getLogger(__name__)
+
+
+class UploadsAPIController(BaseAPIController):
+
+ READ_CHUNK_SIZE = 2 ** 16
+
+ @expose_api_anonymous
+ def index(self, trans, **kwd):
+ raise NotImplemented("Listing uploads is not implemented.")
+
+ @expose_api_anonymous
+ def create(self, trans, payload, **kwd):
+ """
+ POST /api/uploads/
+ """
+ session_id = payload.get("session_id")
+ session_start = payload.get("session_start")
+ session_chunk = payload.get("session_chunk")
+ if re.match('^[\w-]+$', session_id) is None:
+ raise MessageException("Requires a session id.")
+ if session_start is None:
+ raise MessageException("Requires a session start.")
+ if not hasattr(session_chunk, "file"):
+ raise MessageException("Requires a session chunk.")
+ target_file = os.path.join(trans.app.config.new_file_path, session_id)
+ target_size = 0
+ if os.path.exists(target_file):
+ target_size = os.path.getsize(target_file)
+ if session_start != target_size:
+ raise MessageException("Incorrect session start.")
+ chunk_size = os.fstat(session_chunk.file.fileno()).st_size
+ if chunk_size > trans.app.config.chunk_upload_size:
+ raise MessageException("Invalid chunk size.")
+ with open(target_file, "a") as f:
+ while True:
+ read_chunk = session_chunk.file.read(self.READ_CHUNK_SIZE)
+ if not read_chunk:
+ break
+ f.write(read_chunk)
+ session_chunk.file.close()
+ return {"message": "Successful."}
diff --git a/lib/galaxy/webapps/galaxy/api/visualizations.py b/lib/galaxy/webapps/galaxy/api/visualizations.py
index de44a81fe6a..63f3c42d0e9 100644
--- a/lib/galaxy/webapps/galaxy/api/visualizations.py
+++ b/lib/galaxy/webapps/galaxy/api/visualizations.py
@@ -176,7 +176,7 @@ class VisualizationsController(BaseAPIController, UsesVisualizationMixin, Sharab
if key == 'type':
if not isinstance(val, string_types):
raise ValidationError('%s must be a string or unicode: %s' % (key, str(type(val))))
- val = util.sanitize_html.sanitize_html(val, 'utf-8')
+ val = util.sanitize_html.sanitize_html(val)
elif key == 'config':
if not isinstance(val, dict):
raise ValidationError('%s must be a dictionary: %s' % (key, str(type(val))))
@@ -184,22 +184,22 @@ class VisualizationsController(BaseAPIController, UsesVisualizationMixin, Sharab
elif key == 'annotation':
if not isinstance(val, string_types):
raise ValidationError('%s must be a string or unicode: %s' % (key, str(type(val))))
- val = util.sanitize_html.sanitize_html(val, 'utf-8')
+ val = util.sanitize_html.sanitize_html(val)
# these are keys that actually only be *updated* at the revision level and not here
# (they are still valid for create, tho)
elif key == 'title':
if not isinstance(val, string_types):
raise ValidationError('%s must be a string or unicode: %s' % (key, str(type(val))))
- val = util.sanitize_html.sanitize_html(val, 'utf-8')
+ val = util.sanitize_html.sanitize_html(val)
elif key == 'slug':
if not isinstance(val, string_types):
raise ValidationError('%s must be a string: %s' % (key, str(type(val))))
- val = util.sanitize_html.sanitize_html(val, 'utf-8')
+ val = util.sanitize_html.sanitize_html(val)
elif key == 'dbkey':
if not isinstance(val, string_types):
raise ValidationError('%s must be a string or unicode: %s' % (key, str(type(val))))
- val = util.sanitize_html.sanitize_html(val, 'utf-8')
+ val = util.sanitize_html.sanitize_html(val)
elif key not in valid_but_uneditable_keys:
continue
diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py
index c2c10940df9..f303fbfa92a 100644
--- a/lib/galaxy/webapps/galaxy/buildapp.py
+++ b/lib/galaxy/webapps/galaxy/buildapp.py
@@ -256,6 +256,7 @@ def populate_api_routes(webapp, app):
webapp.mapper.resource('dataset_collection', 'dataset_collections', path_prefix='/api/')
webapp.mapper.resource('form', 'forms', path_prefix='/api')
webapp.mapper.resource('role', 'roles', path_prefix='/api')
+ webapp.mapper.resource('upload', 'uploads', path_prefix='/api')
webapp.mapper.connect('/api/ftp_files', controller='remote_files')
webapp.mapper.resource('remote_file', 'remote_files', path_prefix='/api')
webapp.mapper.resource('group', 'groups', path_prefix='/api')
@@ -271,9 +272,13 @@ def populate_api_routes(webapp, app):
# ====== TOOLS API ======
# =======================
+ webapp.mapper.connect('/api/tools/fetch', action='fetch', controller='tools', conditions=dict(method=["POST"]))
webapp.mapper.connect('/api/tools/all_requirements', action='all_requirements', controller="tools")
webapp.mapper.connect('/api/tools/{id:.+?}/build', action='build', controller="tools")
webapp.mapper.connect('/api/tools/{id:.+?}/reload', action='reload', controller="tools")
+ webapp.mapper.connect('/api/tools/tests_summary', action='tests_summary', controller="tools")
+ webapp.mapper.connect('/api/tools/{id:.+?}/test_data_path', action='test_data_path', controller="tools")
+ webapp.mapper.connect('/api/tools/{id:.+?}/test_data', action='test_data', controller="tools")
webapp.mapper.connect('/api/tools/{id:.+?}/diagnostics', action='diagnostics', controller="tools")
webapp.mapper.connect('/api/tools/{id:.+?}/citations', action='citations', controller="tools")
webapp.mapper.connect('/api/tools/{id:.+?}/download', action='download', controller="tools")
diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml
index 6c365e68b64..5679ffb01f1 100644
--- a/lib/galaxy/webapps/galaxy/config_schema.yml
+++ b/lib/galaxy/webapps/galaxy/config_schema.yml
@@ -1239,6 +1239,15 @@ mapping:
operations on the remote end. See the Galaxy nginx documentation for the
corresponding nginx configuration.
+ chunk_upload_size:
+ type: int
+ default: 104857600
+ required: false
+ desc: |
+ Galaxy can upload user files in chunks without using nginx. Enable the chunk
+ uploader by specifying a chunk size larger than 0. The chunk size is specified
+ in bytes (default: 100MB).
+
dynamic_proxy_manage:
type: bool
default: true
@@ -1817,6 +1826,18 @@ mapping:
desc: |
Set maximum size of ngrams
+ tool_test_data_directories:
+ type: str
+ default: 'test-data'
+ required: false
+ desc: |
+ Set tool test data directory. The test framework sets this value to
+ 'test-data,https://github.com/galaxyproject/galaxy-test-data.git' which will
+ cause Galaxy to clone down extra test data on the fly for certain tools
+ distributed with Galaxy but this is likely not appropriate for production systems.
+ Instead one can simply clone that repository directly and specify a path here
+ instead of a Git HTTP repository.
+
id_secret:
type: str
default: USING THE DEFAULT IS NOT SECURE!
@@ -2518,29 +2539,6 @@ mapping:
overwrite default job resources such as number of processors, memory and
walltime.
- workflow_resource_params_file:
- type: str
- default: config/workflow_resource_params_conf.xml
- required: false
- desc: |
- Similar to the above parameter, workflows can describe parameters used to
- influence scheduling of jobs within the workflow. This requires both a description
- of the fields available (which defaults to the definitions in
- job_resource_params_file if not set).
-
- workflow_resource_params_mapper:
- type: str
- default: config/workflow_resource_mapper_conf.yml
- required: false
- desc: |
- This parameter describes how to map users and workflows to a set of workflow
- resource parameter to present (typically input IDs from workflow_resource_params_file).
- If this this is a function reference it will be passed various inputs (workflow model
- object and user) and it should produce a list of input IDs. If it is a path
- it is expected to an XML or YAML file describing how to map group names to parameter
- descriptions (additional types of mappings via these files could be implemented but
- haven't yet - for instance using workflow tags to do the mapping).
-
cache_user_job_count:
type: bool
default: false
diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py
index 6b1b009aaad..8382ad48ebe 100644
--- a/lib/galaxy/webapps/galaxy/controllers/dataset.py
+++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py
@@ -435,7 +435,7 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE
data.datatype.after_setting_metadata(data)
# Sanitize annotation before adding it.
if payload.get('annotation'):
- annotation = sanitize_html(payload.get('annotation'), 'utf-8', 'text/html')
+ annotation = sanitize_html(payload.get('annotation'))
self.add_item_annotation(trans.sa_session, trans.get_user(), data, annotation)
# if setting metadata previously failed and all required elements have now been set, clear the failed state.
if data._state == trans.model.Dataset.states.FAILED_METADATA and not data.missing_meta():
@@ -700,7 +700,7 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE
web.httpexceptions.HTTPNotFound()
if dataset and new_annotation:
# Sanitize annotation before adding it.
- new_annotation = sanitize_html(new_annotation, 'utf-8', 'text/html')
+ new_annotation = sanitize_html(new_annotation)
self.add_item_annotation(trans.sa_session, trans.get_user(), dataset, new_annotation)
trans.sa_session.flush()
return new_annotation
diff --git a/lib/galaxy/webapps/galaxy/controllers/history.py b/lib/galaxy/webapps/galaxy/controllers/history.py
index 2e2d1909af5..9d51b8a7618 100644
--- a/lib/galaxy/webapps/galaxy/controllers/history.py
+++ b/lib/galaxy/webapps/galaxy/controllers/history.py
@@ -494,11 +494,10 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt
history = trans.sa_session.query(model.History).options(
eagerload_all('active_datasets.creating_job_associations.job.workflow_invocation_step.workflow_invocation.workflow'),
).get(id)
- assert history
- # TODO: formalize to trans.show_error
- assert (history.user and (history.user.id == trans.user.id) or
- (history.id == trans.history.id) or
- (trans.user_is_admin()))
+ if not (history and ((history.user and trans.user and history.user.id == trans.user.id) or
+ (trans.history and history.id == trans.history.id) or
+ trans.user_is_admin())):
+ return trans.show_error_message("Cannot display history structure.")
# Resolve jobs and workflow invocations for the datasets in the history
# items is filled with items (hdas, jobs, or workflows) that go at the
# top level
@@ -1127,7 +1126,7 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt
@web.expose
def purge_deleted_datasets(self, trans):
count = 0
- if trans.app.config.allow_user_dataset_purge:
+ if trans.app.config.allow_user_dataset_purge and trans.history:
for hda in trans.history.datasets:
if not hda.deleted or hda.purged:
continue
@@ -1145,7 +1144,8 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt
except Exception:
log.exception('Unable to purge dataset (%s) on purge of hda (%s):' % (hda.dataset.id, hda.id))
count += 1
- return trans.show_ok_message("%d datasets have been deleted permanently" % count, refresh_frames=['history'])
+ return trans.show_ok_message("%d datasets have been deleted permanently" % count, refresh_frames=['history'])
+ return trans.show_error_message("Cannot purge deleted datasets from this session.")
@web.expose
def delete(self, trans, id, purge=False):
diff --git a/lib/galaxy/webapps/galaxy/controllers/page.py b/lib/galaxy/webapps/galaxy/controllers/page.py
index 2cff6220813..943dd0da10f 100644
--- a/lib/galaxy/webapps/galaxy/controllers/page.py
+++ b/lib/galaxy/webapps/galaxy/controllers/page.py
@@ -1,16 +1,75 @@
+import re
from json import loads
from markupsafe import escape
-from sqlalchemy import and_, desc, false, true
-from sqlalchemy.orm import eagerload, undefer
+from six.moves.html_entities import name2codepoint
+from six.moves.html_parser import HTMLParser
+from sqlalchemy import (
+ and_,
+ desc,
+ false,
+ true
+)
+from sqlalchemy.orm import (
+ eagerload,
+ undefer
+)
-from galaxy import managers, model, util, web
+from galaxy import (
+ managers,
+ model,
+ util,
+ web
+)
from galaxy.model.item_attrs import UsesItemRatings
from galaxy.util import unicodify
-from galaxy.util.sanitize_html import _BaseHTMLProcessor, sanitize_html
-from galaxy.web import error, url_for
-from galaxy.web.base.controller import BaseUIController, SharableMixin, UsesStoredWorkflowMixin, UsesVisualizationMixin
-from galaxy.web.framework.helpers import grids, time_ago
+from galaxy.util.sanitize_html import sanitize_html
+from galaxy.web import (
+ error,
+ url_for
+)
+from galaxy.web.base.controller import (
+ BaseUIController,
+ SharableMixin,
+ UsesStoredWorkflowMixin,
+ UsesVisualizationMixin
+)
+from galaxy.web.framework.helpers import (
+ grids,
+ time_ago
+)
+
+
+# Copied from https://github.com/kurtmckee/feedparser
+_cp1252 = {
+ 128: u'\u20ac', # euro sign
+ 130: u'\u201a', # single low-9 quotation mark
+ 131: u'\u0192', # latin small letter f with hook
+ 132: u'\u201e', # double low-9 quotation mark
+ 133: u'\u2026', # horizontal ellipsis
+ 134: u'\u2020', # dagger
+ 135: u'\u2021', # double dagger
+ 136: u'\u02c6', # modifier letter circumflex accent
+ 137: u'\u2030', # per mille sign
+ 138: u'\u0160', # latin capital letter s with caron
+ 139: u'\u2039', # single left-pointing angle quotation mark
+ 140: u'\u0152', # latin capital ligature oe
+ 142: u'\u017d', # latin capital letter z with caron
+ 145: u'\u2018', # left single quotation mark
+ 146: u'\u2019', # right single quotation mark
+ 147: u'\u201c', # left double quotation mark
+ 148: u'\u201d', # right double quotation mark
+ 149: u'\u2022', # bullet
+ 150: u'\u2013', # en dash
+ 151: u'\u2014', # em dash
+ 152: u'\u02dc', # small tilde
+ 153: u'\u2122', # trade mark sign
+ 154: u'\u0161', # latin small letter s with caron
+ 155: u'\u203a', # single right-pointing angle quotation mark
+ 156: u'\u0153', # latin small ligature oe
+ 158: u'\u017e', # latin small letter z with caron
+ 159: u'\u0178', # latin capital letter y with diaeresis
+}
def format_bool(b):
@@ -223,18 +282,52 @@ class VisualizationSelectionGrid(ItemSelectionGrid):
)
-class _PageContentProcessor(_BaseHTMLProcessor):
- """ Processes page content to produce HTML that is suitable for display. For now, processor renders embedded objects. """
+# Adapted from the _BaseHTMLProcessor class of https://github.com/kurtmckee/feedparser
+class _PageContentProcessor(HTMLParser, object):
+ """
+ Processes page content to produce HTML that is suitable for display.
+ For now, processor renders embedded objects.
+ """
+ bare_ampersand = re.compile("&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)")
+ elements_no_end_tag = set([
+ 'area', 'base', 'basefont', 'br', 'col', 'command', 'embed', 'frame',
+ 'hr', 'img', 'input', 'isindex', 'keygen', 'link', 'meta', 'param',
+ 'source', 'track', 'wbr'
+ ])
- def __init__(self, trans, encoding, type, render_embed_html_fn):
- _BaseHTMLProcessor.__init__(self, encoding, type)
+ def __init__(self, trans, render_embed_html_fn):
+ HTMLParser.__init__(self)
self.trans = trans
self.ignore_content = False
self.num_open_tags_for_ignore = 0
self.render_embed_html_fn = render_embed_html_fn
- def unknown_starttag(self, tag, attrs):
- """ Called for each start tag; attrs is a list of (attr, value) tuples. """
+ def reset(self):
+ self.pieces = []
+ HTMLParser.reset(self)
+
+ def _shorttag_replace(self, match):
+ tag = match.group(1)
+ if tag in self.elements_no_end_tag:
+ return '<' + tag + ' />'
+ else:
+ return '<' + tag + '>'
+
+ def feed(self, data):
+ data = re.compile(r'\s]+?)\s*/>', self._shorttag_replace, data)
+ data = data.replace(''', "'")
+ data = data.replace('"', '"')
+ HTMLParser.feed(self, data)
+ HTMLParser.close(self)
+
+ def handle_starttag(self, tag, attrs):
+ """
+ Called for each start tag
+
+ attrs is a list of (attr, value) tuples, e.g. for ,
+ tag='pre', attrs=[('class', 'screen')]
+ """
# If ignoring content, just increment tag count and ignore.
if self.ignore_content:
@@ -264,17 +357,25 @@ class _PageContentProcessor(_BaseHTMLProcessor):
return
# Default behavior: not ignoring and no embedded content.
- _BaseHTMLProcessor.unknown_starttag(self, tag, attrs)
+ uattrs = []
+ strattrs = ''
+ if attrs:
+ for key, value in attrs:
+ value = value.replace('>', '>').replace('<', '<').replace('"', '"')
+ value = self.bare_ampersand.sub("&", value)
+ uattrs.append((key, value))
+ strattrs = ''.join(' %s="%s"' % (k, v) for k, v in uattrs)
+ if tag in self.elements_no_end_tag:
+ self.pieces.append('<%s%s />' % (tag, strattrs))
+ else:
+ self.pieces.append('<%s%s>' % (tag, strattrs))
- def handle_data(self, text):
- """ Called for each block of plain text. """
- if self.ignore_content:
- return
- _BaseHTMLProcessor.handle_data(self, text)
-
- def unknown_endtag(self, tag):
- """ Called for each end tag. """
+ def handle_endtag(self, tag):
+ """
+ Called for each end tag
+ E.g. for , tag will be 'pre'
+ """
# If ignoring content, see if current tag is the end of content to ignore.
if self.ignore_content:
self.num_open_tags_for_ignore -= 1
@@ -283,8 +384,63 @@ class _PageContentProcessor(_BaseHTMLProcessor):
self.ignore_content = False
return
- # Default behavior:
- _BaseHTMLProcessor.unknown_endtag(self, tag)
+ # Default behavior: reconstruct the original end tag.
+ if tag not in self.elements_no_end_tag:
+ self.pieces.append("" % tag)
+
+ def handle_charref(self, ref):
+ # called for each character reference, e.g. for ' ', ref will be '160'
+ # Reconstruct the original character reference.
+ ref = ref.lower()
+ if ref.startswith('x'):
+ value = int(ref[1:], 16)
+ else:
+ value = int(ref)
+
+ if value in _cp1252:
+ self.pieces.append('&#%s;' % hex(ord(_cp1252[value]))[1:])
+ else:
+ self.pieces.append('&#%s;' % ref)
+
+ def handle_entityref(self, ref):
+ # called for each entity reference, e.g. for '©', ref will be 'copy'
+ # Reconstruct the original entity reference.
+ if ref in name2codepoint or ref == 'apos':
+ self.pieces.append('&%s;' % ref)
+ else:
+ self.pieces.append('&%s' % ref)
+
+ def handle_data(self, text):
+ """
+ Called for each block of plain text
+
+ Called outside of any tag and not containing any character or entity
+ references. Store the original text verbatim.
+ """
+ if self.ignore_content:
+ return
+ self.pieces.append(text)
+
+ def handle_comment(self, text):
+ # called for each HTML comment, e.g.
+ # Reconstruct the original comment.
+ self.pieces.append('' % text)
+
+ def handle_decl(self, text):
+ # called for the DOCTYPE, if present, e.g.
+ #
+ # Reconstruct original DOCTYPE
+ self.pieces.append('' % text)
+
+ def handle_pi(self, text):
+ # called for each processing instruction, e.g.
+ # Reconstruct original processing instruction.
+ self.pieces.append('' % text)
+
+ def output(self):
+ '''Return processed HTML as a single string'''
+ return ''.join(self.pieces)
class PageController(BaseUIController, SharableMixin,
@@ -387,7 +543,7 @@ class PageController(BaseUIController, SharableMixin,
p.slug = p_slug
p.user = user
if p_annotation:
- p_annotation = sanitize_html(p_annotation, 'utf-8', 'text/html')
+ p_annotation = sanitize_html(p_annotation)
self.add_item_annotation(trans.sa_session, user, p, p_annotation)
# And the first (empty) page revision
p_revision = model.PageRevision()
@@ -449,7 +605,7 @@ class PageController(BaseUIController, SharableMixin,
p.title = p_title
p.slug = p_slug
if p_annotation:
- p_annotation = sanitize_html(p_annotation, 'utf-8', 'text/html')
+ p_annotation = sanitize_html(p_annotation)
self.add_item_annotation(trans.sa_session, user, p, p_annotation)
trans.sa_session.add(p)
trans.sa_session.flush()
@@ -550,7 +706,7 @@ class PageController(BaseUIController, SharableMixin,
assert page.user == trans.user
# Sanitize content
- content = sanitize_html(content, 'utf-8', 'text/html')
+ content = sanitize_html(content)
# Add a new revision to the page with the provided content.
page_revision = model.PageRevision()
@@ -567,7 +723,7 @@ class PageController(BaseUIController, SharableMixin,
item = trans.sa_session.query(item_class).filter_by(id=item_id).first()
if not item:
raise RuntimeError("cannot find annotated item")
- text = sanitize_html(annotation_dict['text'], 'utf-8', 'text/html')
+ text = sanitize_html(annotation_dict['text'])
# Add/update annotation.
if item_id and item_class and text:
@@ -615,8 +771,10 @@ class PageController(BaseUIController, SharableMixin,
self.security_check(trans, page, False, True)
# Process page content.
- processor = _PageContentProcessor(trans, 'utf-8', 'text/html', self._get_embed_html)
+ processor = _PageContentProcessor(trans, self._get_embed_html)
processor.feed(page.latest_revision.content)
+ # Output is string, so convert to unicode for display.
+ page_content = unicodify(processor.output(), 'utf-8')
# Get rating data.
user_item_rating = 0
@@ -628,8 +786,6 @@ class PageController(BaseUIController, SharableMixin,
user_item_rating = 0
ave_item_rating, num_ratings = self.get_ave_item_rating_data(trans.sa_session, page)
- # Output is string, so convert to unicode for display.
- page_content = unicodify(processor.output(), 'utf-8')
return trans.fill_template_mako("page/display.mako", item=page,
item_data=page_content,
user_item_rating=user_item_rating,
diff --git a/lib/galaxy/webapps/galaxy/controllers/root.py b/lib/galaxy/webapps/galaxy/controllers/root.py
index ca727ea6889..ec732b7066b 100644
--- a/lib/galaxy/webapps/galaxy/controllers/root.py
+++ b/lib/galaxy/webapps/galaxy/controllers/root.py
@@ -108,7 +108,6 @@ class RootController(controller.JSAppLauncher, UsesAnnotations):
js_options = self._get_js_options(trans)
config = js_options['config']
config.update(self._get_extended_config(trans))
-
return self.template(trans, 'analysis', options=js_options)
@web.expose
diff --git a/lib/galaxy/webapps/galaxy/controllers/user.py b/lib/galaxy/webapps/galaxy/controllers/user.py
index 638659cf031..96ba8b0a786 100644
--- a/lib/galaxy/webapps/galaxy/controllers/user.py
+++ b/lib/galaxy/webapps/galaxy/controllers/user.py
@@ -925,7 +925,9 @@ class User(BaseUIController, UsesFormDefinitionsMixin, CreatesUsersMixin, Create
Check whether token fits the user and then activate the user's account.
"""
params = util.Params(kwd, sanitize=False)
- email = unquote(params.get('email', None))
+ email = params.get('email', None)
+ if email is not None:
+ email = unquote(email)
activation_token = params.get('activation_token', None)
if email is None or activation_token is None:
@@ -934,6 +936,9 @@ class User(BaseUIController, UsesFormDefinitionsMixin, CreatesUsersMixin, Create
else:
# Find the user
user = trans.sa_session.query(trans.app.model.User).filter(trans.app.model.User.table.c.email == email).first()
+ if not user:
+ # Probably wrong email address
+ return trans.show_error_message("You are using an invalid activation link. Try to log in and we will send you a new activation email.
Go to login page.") % web.url_for(controller="root", action="index")
# If the user is active already don't try to activate
if user.active is True:
return trans.show_ok_message("Your account is already active. Nothing has changed.
Go to login page.") % web.url_for(controller='root', action='index')
diff --git a/lib/galaxy/webapps/galaxy/controllers/visualization.py b/lib/galaxy/webapps/galaxy/controllers/visualization.py
index 0232494646e..4d618b36057 100644
--- a/lib/galaxy/webapps/galaxy/controllers/visualization.py
+++ b/lib/galaxy/webapps/galaxy/controllers/visualization.py
@@ -620,7 +620,7 @@ class VisualizationController(BaseUIController, SharableMixin, UsesVisualization
v.slug = v_slug
v.dbkey = v_dbkey
if v_annotation:
- v_annotation = sanitize_html(v_annotation, 'utf-8', 'text/html')
+ v_annotation = sanitize_html(v_annotation)
self.add_item_annotation(trans.sa_session, trans.get_user(), v, v_annotation)
trans.sa_session.add(v)
trans.sa_session.flush()
diff --git a/lib/galaxy/webapps/galaxy/controllers/workflow.py b/lib/galaxy/webapps/galaxy/controllers/workflow.py
index 06b75087eed..8aac37e9638 100644
--- a/lib/galaxy/webapps/galaxy/controllers/workflow.py
+++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py
@@ -4,10 +4,10 @@ import base64
import json
import logging
import os
-import sgmllib
import requests
from markupsafe import escape
+from six.moves.html_parser import HTMLParser
from six.moves.http_client import HTTPConnection
from sqlalchemy import and_
from sqlalchemy.orm import eagerload, joinedload, lazyload, undefer
@@ -153,16 +153,17 @@ class StoredWorkflowAllPublishedGrid(grids.Grid):
self.model_class.deleted == expression.false())
-# Simple SGML parser to get all content in a single tag.
-class SingleTagContentsParser(sgmllib.SGMLParser):
+# Simple HTML parser to get all content in a single tag.
+class SingleTagContentsParser(HTMLParser):
def __init__(self, target_tag):
- sgmllib.SGMLParser.__init__(self)
+ # Cannot use super() because HTMLParser is an old-style class in Python2
+ HTMLParser.__init__(self)
self.target_tag = target_tag
self.cur_tag = None
self.tag_content = ""
- def unknown_starttag(self, tag, attrs):
+ def handle_starttag(self, tag, attrs):
""" Called for each start tag. """
self.cur_tag = tag
@@ -404,7 +405,7 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi
stored = self.get_stored_workflow(trans, id)
if new_annotation:
# Sanitize annotation before adding it.
- new_annotation = sanitize_html(new_annotation, 'utf-8', 'text/html')
+ new_annotation = sanitize_html(new_annotation)
self.add_item_annotation(trans.sa_session, trans.get_user(), stored, new_annotation)
trans.sa_session.flush()
return new_annotation
@@ -547,7 +548,7 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi
workflow.stored_workflow = stored_workflow
stored_workflow.latest_workflow = workflow
# Add annotation.
- workflow_annotation = sanitize_html(workflow_annotation, 'utf-8', 'text/html')
+ workflow_annotation = sanitize_html(workflow_annotation)
self.add_item_annotation(trans.sa_session, trans.get_user(), stored_workflow, workflow_annotation)
# Persist
session = trans.sa_session
@@ -573,7 +574,7 @@ class WorkflowController(BaseUIController, SharableMixin, UsesStoredWorkflowMixi
workflow.stored_workflow = stored_workflow
stored_workflow.latest_workflow = workflow
# Add annotation.
- workflow_annotation = sanitize_html(workflow_annotation, 'utf-8', 'text/html')
+ workflow_annotation = sanitize_html(workflow_annotation)
self.add_item_annotation(trans.sa_session, trans.get_user(), stored_workflow, workflow_annotation)
# Persist
diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py
index 44abc129791..d090ef51280 100644
--- a/lib/galaxy/workflow/modules.py
+++ b/lib/galaxy/workflow/modules.py
@@ -825,6 +825,9 @@ class ToolModule(WorkflowModule):
invocation = invocation_step.workflow_invocation
step = invocation_step.workflow_step
tool = trans.app.toolbox.get_tool(step.tool_id, tool_version=step.tool_version)
+ if not tool.is_workflow_compatible:
+ message = "Specified tool [%s] in workflow is not workflow-compatible." % tool.id
+ raise Exception(message)
tool_state = step.state
# Not strictly needed - but keep Tool state clean by stripping runtime
# metadata parameters from it.
@@ -843,7 +846,6 @@ class ToolModule(WorkflowModule):
else:
iteration_elements_iter = [None]
- resource_parameters = invocation.resource_parameters
for iteration_elements in iteration_elements_iter:
execution_state = tool_state.copy()
# TODO: Move next step into copy()
@@ -916,8 +918,7 @@ class ToolModule(WorkflowModule):
invocation_step=invocation_step,
max_num_jobs=max_num_jobs,
job_callback=lambda job: self._handle_post_job_actions(step, job, invocation.replacement_dict),
- completed_jobs=completed_jobs,
- workflow_resource_parameters=resource_parameters
+ completed_jobs=completed_jobs
)
complete = True
except PartialJobExecution as pje:
diff --git a/lib/galaxy/workflow/run_request.py b/lib/galaxy/workflow/run_request.py
index 2e58141a269..6d67d332146 100644
--- a/lib/galaxy/workflow/run_request.py
+++ b/lib/galaxy/workflow/run_request.py
@@ -7,7 +7,6 @@ from galaxy import (
)
from galaxy.managers import histories
from galaxy.tools.parameters.meta import expand_workflow_inputs
-from galaxy.workflow.resources import get_resource_mapper_function
INPUT_STEP_TYPES = ['data_input', 'data_collection_input', 'parameter_input']
@@ -45,17 +44,15 @@ class WorkflowRunConfig(object):
def __init__(self, target_history,
replacement_dict,
copy_inputs_to_history=False,
- inputs={},
- param_map={},
+ inputs=None,
+ param_map=None,
allow_tool_state_corrections=False,
- use_cached_job=False,
- resource_params={}):
+ use_cached_job=False):
self.target_history = target_history
self.replacement_dict = replacement_dict
self.copy_inputs_to_history = copy_inputs_to_history
- self.inputs = inputs
- self.param_map = param_map
- self.resource_params = resource_params
+ self.inputs = inputs or {}
+ self.param_map = param_map or {}
self.allow_tool_state_corrections = allow_tool_state_corrections
self.use_cached_job = use_cached_job
@@ -171,7 +168,8 @@ def _flatten_step_params(param_dict, prefix=""):
return new_params
-def _get_target_history(trans, workflow, payload, param_keys=[], index=0):
+def _get_target_history(trans, workflow, payload, param_keys=None, index=0):
+ param_keys = param_keys or []
history_name = payload.get('new_history_name', None)
history_id = payload.get('history_id', None)
history_param = payload.get('history', None)
@@ -307,35 +305,6 @@ def build_workflow_run_configs(trans, workflow, payload):
normalized_inputs[key] = value['content']
else:
normalized_inputs[key] = value
- resource_params = payload.get('resource_params', {})
- if resource_params:
- # quick attempt to validate parameters, just handle select options now since is what
- # is needed for DTD - arbitrary plugins can define arbitrary logic at runtime in the
- # destination function. In the future this should be extended to allow arbitrary
- # pluggable validation.
- resource_mapper_function = get_resource_mapper_function(trans.app)
- # TODO: Do we need to do anything with the stored_workflow or can this be removed.
- resource_parameters = resource_mapper_function(trans=trans, stored_workflow=None, workflow=workflow)
- for resource_parameter in resource_parameters:
- if resource_parameter.get("type") == "select":
- name = resource_parameter.get("name")
- if name in resource_params:
- value = resource_params[name]
- valid_option = False
- # TODO: How should be handle the case where no selection is made by the user
- # This can happen when there is a select on the page but the user has no options to select
- # Here I have the validation pass it through. An alternative may be to remove the parameter if
- # it is None.
- if value is None:
- valid_option = True
- else:
- for option_elem in resource_parameter.get('data'):
- option_value = option_elem.get("value")
- if value == option_value:
- valid_option = True
- if not valid_option:
- raise exceptions.RequestParameterInvalidException("Invalid value for parameter '%s' found." % name)
-
run_configs.append(WorkflowRunConfig(
target_history=history,
replacement_dict=payload.get('replacement_params', {}),
@@ -343,7 +312,6 @@ def build_workflow_run_configs(trans, workflow, payload):
param_map=param_map,
allow_tool_state_corrections=allow_tool_state_corrections,
use_cached_job=use_cached_job,
- resource_params=resource_params,
))
return run_configs
@@ -383,8 +351,7 @@ def workflow_run_config_to_request(trans, run_config, workflow):
use_cached_job=run_config.use_cached_job,
inputs={},
param_map={},
- allow_tool_state_corrections=run_config.allow_tool_state_corrections,
- resource_params=run_config.resource_params
+ allow_tool_state_corrections=run_config.allow_tool_state_corrections
)
subworkflow_invocation = workflow_run_config_to_request(
trans,
@@ -406,9 +373,6 @@ def workflow_run_config_to_request(trans, run_config, workflow):
for step_id, content in run_config.inputs.items():
workflow_invocation.add_input(content, step_id)
- resource_parameters = run_config.resource_params
- for key, value in resource_parameters.items():
- add_parameter(key, value, param_types.RESOURCE_PARAMETERS)
add_parameter("copy_inputs_to_history", "true" if run_config.copy_inputs_to_history else "false", param_types.META_PARAMETERS)
add_parameter("use_cached_job", "true" if run_config.use_cached_job else "false", param_types.META_PARAMETERS)
return workflow_invocation
@@ -420,7 +384,6 @@ def workflow_request_to_run_config(work_request_context, workflow_invocation):
replacement_dict = {}
inputs = {}
param_map = {}
- resource_params = {}
copy_inputs_to_history = None
use_cached_job = False
for parameter in workflow_invocation.input_parameters:
@@ -433,8 +396,6 @@ def workflow_request_to_run_config(work_request_context, workflow_invocation):
copy_inputs_to_history = (parameter.value == "true")
if parameter.name == 'use_cached_job':
use_cached_job = (parameter.value == 'true')
- elif parameter_type == param_types.RESOURCE_PARAMETERS:
- resource_params[parameter.name] = parameter.value
for input_association in workflow_invocation.input_datasets:
inputs[input_association.workflow_step_id] = input_association.dataset
for input_association in workflow_invocation.input_dataset_collections:
@@ -450,7 +411,6 @@ def workflow_request_to_run_config(work_request_context, workflow_invocation):
param_map=param_map,
copy_inputs_to_history=copy_inputs_to_history,
use_cached_job=use_cached_job,
- resource_params=resource_params,
)
return workflow_run_config
diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py
index f8dc0ee5cd7..917b2ead48c 100644
--- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py
+++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py
@@ -16,6 +16,7 @@ from galaxy.util import (
asbool,
download_to_file
)
+from galaxy.util.compression_utils import CompressedFile
from galaxy.util.template import fill_template
from tool_shed.galaxy_install.tool_dependencies.env_manager import EnvManager
from tool_shed.util import basic_util, tool_dependency_util
@@ -25,124 +26,6 @@ log = logging.getLogger(__name__)
VIRTUALENV_URL = 'https://pypi.python.org/packages/d4/0c/9840c08189e030873387a73b90ada981885010dd9aea134d6de30cd24cb8/virtualenv-15.1.0.tar.gz'
-class CompressedFile(object):
-
- def __init__(self, file_path, mode='r'):
- if tarfile.is_tarfile(file_path):
- self.file_type = 'tar'
- elif zipfile.is_zipfile(file_path) and not file_path.endswith('.jar'):
- self.file_type = 'zip'
- self.file_name = os.path.splitext(os.path.basename(file_path))[0]
- if self.file_name.endswith('.tar'):
- self.file_name = os.path.splitext(self.file_name)[0]
- self.type = self.file_type
- method = 'open_%s' % self.file_type
- if hasattr(self, method):
- self.archive = getattr(self, method)(file_path, mode)
- else:
- raise NameError('File type %s specified, no open method found.' % self.file_type)
-
- def extract(self, path):
- '''Determine the path to which the archive should be extracted.'''
- contents = self.getmembers()
- extraction_path = path
- common_prefix = ''
- if len(contents) == 1:
- # The archive contains a single file, return the extraction path.
- if self.isfile(contents[0]):
- extraction_path = os.path.join(path, self.file_name)
- if not os.path.exists(extraction_path):
- os.makedirs(extraction_path)
- self.archive.extractall(extraction_path)
- else:
- # Get the common prefix for all the files in the archive. If the common prefix ends with a slash,
- # or self.isdir() returns True, the archive contains a single directory with the desired contents.
- # Otherwise, it contains multiple files and/or directories at the root of the archive.
- common_prefix = os.path.commonprefix([self.getname(item) for item in contents])
- if len(common_prefix) >= 1 and not common_prefix.endswith(os.sep) and self.isdir(self.getmember(common_prefix)):
- common_prefix += os.sep
- if not common_prefix.endswith(os.sep):
- common_prefix = ''
- extraction_path = os.path.join(path, self.file_name)
- if not os.path.exists(extraction_path):
- os.makedirs(extraction_path)
- self.archive.extractall(extraction_path)
- # Since .zip files store unix permissions separately, we need to iterate through the zip file
- # and set permissions on extracted members.
- if self.file_type == 'zip':
- for zipped_file in contents:
- filename = self.getname(zipped_file)
- absolute_filepath = os.path.join(extraction_path, filename)
- external_attributes = self.archive.getinfo(filename).external_attr
- # The 2 least significant bytes are irrelevant, the next two contain unix permissions.
- unix_permissions = external_attributes >> 16
- if unix_permissions != 0:
- if os.path.exists(absolute_filepath):
- os.chmod(absolute_filepath, unix_permissions)
- else:
- log.warning("Unable to change permission on extracted file '%s' as it does not exist" % absolute_filepath)
- return os.path.abspath(os.path.join(extraction_path, common_prefix))
-
- def getmembers_tar(self):
- return self.archive.getmembers()
-
- def getmembers_zip(self):
- return self.archive.infolist()
-
- def getname_tar(self, item):
- return item.name
-
- def getname_zip(self, item):
- return item.filename
-
- def getmember(self, name):
- for member in self.getmembers():
- if self.getname(member) == name:
- return member
-
- def getmembers(self):
- return getattr(self, 'getmembers_%s' % self.type)()
-
- def getname(self, member):
- return getattr(self, 'getname_%s' % self.type)(member)
-
- def isdir(self, member):
- return getattr(self, 'isdir_%s' % self.type)(member)
-
- def isdir_tar(self, member):
- return member.isdir()
-
- def isdir_zip(self, member):
- if member.filename.endswith(os.sep):
- return True
- return False
-
- def isfile(self, member):
- if not self.isdir(member):
- return True
- return False
-
- def open_tar(self, filepath, mode):
- return tarfile.open(filepath, mode, errorlevel=0)
-
- def open_zip(self, filepath, mode):
- return zipfile.ZipFile(filepath, mode)
-
- def zipfile_ok(self, path_to_archive):
- """
- This function is a bit pedantic and not functionally necessary. It checks whether there is
- no file pointing outside of the extraction, because ZipFile.extractall() has some potential
- security holes. See python zipfile documentation for more details.
- """
- basename = os.path.realpath(os.path.dirname(path_to_archive))
- zip_archive = zipfile.ZipFile(path_to_archive)
- for member in zip_archive.namelist():
- member_path = os.path.realpath(os.path.join(basename, member))
- if not member_path.startswith(basename):
- return False
- return True
-
-
class Download(object):
def url_download(self, install_dir, downloaded_file_name, download_url, extract=True, checksums={}):
diff --git a/lib/tool_shed/util/tool_util.py b/lib/tool_shed/util/tool_util.py
index 42dc7132d44..3a6ac2069cb 100644
--- a/lib/tool_shed/util/tool_util.py
+++ b/lib/tool_shed/util/tool_util.py
@@ -43,6 +43,7 @@ def copy_sample_file(app, filename, dest_path=None):
"""
Copies a sample file at `filename` to `the dest_path`
directory and strips the '.sample' extensions from `filename`.
+ Returns the path to the copied file (with the .sample extension).
"""
if dest_path is None:
dest_path = os.path.abspath(app.config.tool_data_path)
@@ -56,8 +57,10 @@ def copy_sample_file(app, filename, dest_path=None):
shutil.copy(full_source_path, full_destination_path)
# Only create the .loc file if it does not yet exist. We don't overwrite it in case it
# contains stuff proprietary to the local instance.
- if not os.path.lexists(os.path.join(dest_path, copied_file)):
+ non_sample_path = os.path.join(dest_path, copied_file)
+ if not os.path.lexists(non_sample_path):
shutil.copy(full_source_path, os.path.join(dest_path, copied_file))
+ return non_sample_path
def copy_sample_files(app, sample_files, tool_path=None, sample_files_copied=None, dest_path=None):
@@ -159,10 +162,10 @@ def handle_missing_index_file(app, tool_path, sample_files, repository_tools_tup
for sample_file in sample_files:
sample_file_name = basic_util.strip_path(sample_file)
if sample_file_name == '%s.sample' % missing_file_name:
- copy_sample_file(app, os.path.join(tool_path, sample_file))
+ target_path = copy_sample_file(app, os.path.join(tool_path, sample_file))
if options.tool_data_table and options.tool_data_table.missing_index_file:
- options.tool_data_table.handle_found_index_file(options.missing_index_file)
- sample_files_copied.append(options.missing_index_file)
+ options.tool_data_table.handle_found_index_file(target_path)
+ sample_files_copied.append(target_path)
break
return repository_tools_tups, sample_files_copied
diff --git a/lib/tool_shed/util/workflow_util.py b/lib/tool_shed/util/workflow_util.py
index 38a7edd8f3e..0604759c040 100644
--- a/lib/tool_shed/util/workflow_util.py
+++ b/lib/tool_shed/util/workflow_util.py
@@ -287,7 +287,7 @@ def get_workflow_from_dict(trans, workflow_dict, tools_metadata, repository_id,
if trans.webapp.name == 'galaxy':
annotation = step_dict.get('annotation', '')
if annotation:
- annotation = sanitize_html(annotation, 'utf-8', 'text/html')
+ annotation = sanitize_html(annotation)
new_step_annotation = trans.model.WorkflowStepAnnotationAssociation()
new_step_annotation.annotation = annotation
new_step_annotation.user = trans.user
@@ -398,7 +398,7 @@ def save_workflow(trans, workflow, workflow_dict=None):
stored.latest_workflow = workflow
stored.user = trans.user
if workflow_dict and workflow_dict.get('annotation', ''):
- annotation = sanitize_html(workflow_dict['annotation'], 'utf-8', 'text/html')
+ annotation = sanitize_html(workflow_dict['annotation'])
new_annotation = trans.model.StoredWorkflowAnnotationAssociation()
new_annotation.annotation = annotation
new_annotation.user = trans.user
diff --git a/scripts/api/fetch_to_library.py b/scripts/api/fetch_to_library.py
new file mode 100644
index 00000000000..6c497bcb402
--- /dev/null
+++ b/scripts/api/fetch_to_library.py
@@ -0,0 +1,33 @@
+import argparse
+import json
+
+import requests
+import yaml
+
+
+def main():
+ parser = argparse.ArgumentParser(description='Upload a directory into a data library')
+ parser.add_argument("-u", "--url", dest="url", required=True, help="Galaxy URL")
+ parser.add_argument("-a", "--api", dest="api_key", required=True, help="API Key")
+ parser.add_argument('target', metavar='FILE', type=str,
+ help='file describing data library to fetch')
+ args = parser.parse_args()
+ with open(args.target, "r") as f:
+ target = yaml.load(f)
+
+ histories_url = args.url + "/api/histories"
+ new_history_response = requests.post(histories_url, data={'key': args.api_key})
+
+ fetch_url = args.url + '/api/tools/fetch'
+ payload = {
+ 'key': args.api_key,
+ 'targets': json.dumps([target]),
+ 'history_id': new_history_response.json()["id"]
+ }
+
+ response = requests.post(fetch_url, data=payload)
+ print(response.content)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/scripts/api/fetch_to_library_example.yml b/scripts/api/fetch_to_library_example.yml
new file mode 100644
index 00000000000..44bc35ef43b
--- /dev/null
+++ b/scripts/api/fetch_to_library_example.yml
@@ -0,0 +1,42 @@
+destination:
+ type: library
+ name: Training Material
+ description: Data for selected tutorials from https://training.galaxyproject.org.
+items:
+ - name: Quality Control
+ description: |
+ Data for sequence quality control tutorial at http://galaxyproject.github.io/training-material/topics/sequence-analysis/tutorials/quality-control/tutorial.html.
+
+ 10.5281/zenodo.61771
+ items:
+ - src: url
+ url: https://zenodo.org/record/61771/files/GSM461178_untreat_paired_subset_1.fastq
+ name: GSM461178_untreat_paired_subset_1
+ ext: fastqsanger
+ info: Untreated subseq of GSM461178 from 10.1186/s12864-017-3692-8
+ - src: url
+ url: https://zenodo.org/record/61771/files/GSM461182_untreat_single_subset.fastq
+ name: GSM461182_untreat_single_subset
+ ext: fastqsanger
+ info: Untreated subseq of GSM461182 from 10.1186/s12864-017-3692-8
+ - name: Small RNA-Seq
+ description: |
+ Data for small RNA-seq tutorial available at http://galaxyproject.github.io/training-material/topics/transcriptomics/tutorials/srna/tutorial.html
+
+ 10.5281/zenodo.826906
+ items:
+ - src: url
+ url: https://zenodo.org/record/826906/files/Symp_RNAi_sRNA-seq_rep1_downsampled.fastqsanger.gz
+ name: Symp RNAi sRNA Rep1
+ ext: fastqsanger.gz
+ info: Downsample rep1 from 10.1186/s12864-017-3692-8
+ - src: url
+ url: https://zenodo.org/record/826906/files/Symp_RNAi_sRNA-seq_rep2_downsampled.fastqsanger.gz
+ name: Symp RNAi sRNA Rep2
+ ext: fastqsanger.gz
+ info: Downsample rep2 from 10.1186/s12864-017-3692-8
+ - src: url
+ url: https://zenodo.org/record/826906/files/Symp_RNAi_sRNA-seq_rep3_downsampled.fastqsanger.gz
+ name: Symp RNAi sRNA Rep3
+ ext: fastqsanger.gz
+ info: Downsample rep3 from 10.1186/s12864-017-3692-8
diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh
index 6c460f4a69b..a123df666c1 100755
--- a/scripts/common_startup.sh
+++ b/scripts/common_startup.sh
@@ -197,10 +197,8 @@ if [ $SKIP_CLIENT_BUILD -eq 0 ]; then
if [ "$GIT_BRANCH" = "0" ]; then
SKIP_CLIENT_BUILD=1
else
- # Compare hash.
- githash=$(git rev-parse HEAD)
- statichash=$(cat static/client_build_hash.txt)
- if [ "$githash" = "$statichash" ]; then
+ # Check if anything has changed in client/ since the last build
+ if git diff --quiet $(cat static/client_build_hash.txt) -- client/; then
SKIP_CLIENT_BUILD=1
else
echo "The Galaxy client is out of date and will be built now."
diff --git a/scripts/microbes/BeautifulSoup.py b/scripts/microbes/BeautifulSoup.py
deleted file mode 100644
index a49381ae217..00000000000
--- a/scripts/microbes/BeautifulSoup.py
+++ /dev/null
@@ -1,1804 +0,0 @@
-"""Beautiful Soup
-Elixir and Tonic
-"The Screen-Scraper's Friend"
-https://www.crummy.com/software/BeautifulSoup/
-
-Beautiful Soup parses a (possibly invalid) XML or HTML document into a
-tree representation. It provides methods and Pythonic idioms that make
-it easy to navigate, search, and modify the tree.
-
-A well-structured XML/HTML document yields a well-behaved data
-structure. An ill-structured XML/HTML document yields a
-correspondingly ill-behaved data structure. If your document is only
-locally well-structured, you can use this library to find and process
-the well-structured part of it.
-
-Beautiful Soup works with Python 2.2 and up. It has no external
-dependencies, but you'll have more success at converting data to UTF-8
-if you also install these three packages:
-
-* chardet, for auto-detecting character encodings
- http://chardet.feedparser.org/
-* cjkcodecs and iconv_codec, which add more encodings to the ones supported
- by stock Python.
- http://cjkpython.i18n.org/
-
-Beautiful Soup defines classes for two main parsing strategies:
-
- * BeautifulStoneSoup, for parsing XML, SGML, or your domain-specific
- language that kind of looks like XML.
-
- * BeautifulSoup, for parsing run-of-the-mill HTML code, be it valid
- or invalid. This class has web browser-like heuristics for
- obtaining a sensible parse tree in the face of common HTML errors.
-
-Beautiful Soup also defines a class (UnicodeDammit) for autodetecting
-the encoding of an HTML or XML document, and converting it to
-Unicode. Much of this code is taken from Mark Pilgrim's Universal Feed
-Parser.
-
-For more than you ever wanted to know about Beautiful Soup, see the
-documentation:
-http://www.crummy.com/software/BeautifulSoup/documentation.html
-"""
-from __future__ import (
- generators,
- print_function
-)
-
-import codecs
-import re
-import sgmllib
-import string
-import sys
-from sgmllib import (
- SGMLParseError,
- SGMLParser
-)
-
-import six
-from six.moves.html_entities import name2codepoint
-
-__author__ = "Leonard Richardson (crummy.com)"
-__contributors__ = ["Sam Ruby (intertwingly.net)",
- "the unwitting Mark Pilgrim (diveintomark.org)",
- "http://www.crummy.com/software/BeautifulSoup/AUTHORS.html"]
-__version__ = "3.0.3"
-__copyright__ = "Copyright (c) 2004-2006 Leonard Richardson"
-__license__ = "PSF"
-
-# This RE makes Beautiful Soup able to parse XML with namespaces.
-sgmllib.tagfind = re.compile('[a-zA-Z][-_.:a-zA-Z0-9]*')
-
-# This RE makes Beautiful Soup capable of recognizing numeric character
-# references that use hexadecimal.
-sgmllib.charref = re.compile('&#(\d+|x[0-9a-fA-F]+);')
-
-DEFAULT_OUTPUT_ENCODING = "utf-8"
-
-
-# First, the classes that represent markup elements.
-class PageElement:
- """Contains the navigational information for some part of the page
- (either a tag or a piece of text)"""
-
- def setup(self, parent=None, previous=None):
- """Sets up the initial relations between this element and
- other elements."""
- self.parent = parent
- self.previous = previous
- self.next = None
- self.previousSibling = None
- self.nextSibling = None
- if self.parent and self.parent.contents:
- self.previousSibling = self.parent.contents[-1]
- self.previousSibling.nextSibling = self
-
- def replaceWith(self, replaceWith):
- oldParent = self.parent
- myIndex = self.parent.contents.index(self)
- if hasattr(replaceWith, 'parent') and replaceWith.parent == self.parent:
- # We're replacing this element with one of its siblings.
- index = self.parent.contents.index(replaceWith)
- if index and index < myIndex:
- # Furthermore, it comes before this element. That
- # means that when we extract it, the index of this
- # element will change.
- myIndex = myIndex - 1
- self.extract()
- oldParent.insert(myIndex, replaceWith)
-
- def extract(self):
- """Destructively rips this element out of the tree."""
- if self.parent:
- try:
- self.parent.contents.remove(self)
- except ValueError:
- pass
-
- # Find the two elements that would be next to each other if
- # this element (and any children) hadn't been parsed. Connect
- # the two.
- lastChild = self._lastRecursiveChild()
- nextElement = lastChild.next
-
- if self.previous:
- self.previous.next = nextElement
- if nextElement:
- nextElement.previous = self.previous
- self.previous = None
- lastChild.next = None
-
- self.parent = None
- if self.previousSibling:
- self.previousSibling.nextSibling = self.nextSibling
- if self.nextSibling:
- self.nextSibling.previousSibling = self.previousSibling
- self.previousSibling = self.nextSibling = None
-
- def _lastRecursiveChild(self):
- "Finds the last element beneath this object to be parsed."
- lastChild = self
- while hasattr(lastChild, 'contents') and lastChild.contents:
- lastChild = lastChild.contents[-1]
- return lastChild
-
- def insert(self, position, newChild):
- if isinstance(newChild, six.string_types) and \
- not isinstance(newChild, NavigableString):
- newChild = NavigableString(newChild)
-
- position = min(position, len(self.contents))
- if hasattr(newChild, 'parent') and newChild.parent is not None:
- # We're 'inserting' an element that's already one
- # of this object's children.
- if newChild.parent == self:
- index = self.find(newChild)
- if index and index < position:
- # Furthermore we're moving it further down the
- # list of this object's children. That means that
- # when we extract this element, our target index
- # will jump down one.
- position = position - 1
- newChild.extract()
-
- newChild.parent = self
- previousChild = None
- if position == 0:
- newChild.previousSibling = None
- newChild.previous = self
- else:
- previousChild = self.contents[position - 1]
- newChild.previousSibling = previousChild
- newChild.previousSibling.nextSibling = newChild
- newChild.previous = previousChild._lastRecursiveChild()
- if newChild.previous:
- newChild.previous.next = newChild
-
- newChildsLastElement = newChild._lastRecursiveChild()
-
- if position >= len(self.contents):
- newChild.nextSibling = None
-
- parent = self
- parentsNextSibling = None
- while not parentsNextSibling:
- parentsNextSibling = parent.nextSibling
- parent = parent.parent
- if not parent: # This is the last element in the document.
- break
- if parentsNextSibling:
- newChildsLastElement.next = parentsNextSibling
- else:
- newChildsLastElement.next = None
- else:
- nextChild = self.contents[position]
- newChild.nextSibling = nextChild
- if newChild.nextSibling:
- newChild.nextSibling.previousSibling = newChild
- newChildsLastElement.next = nextChild
-
- if newChildsLastElement.next:
- newChildsLastElement.next.previous = newChildsLastElement
- self.contents.insert(position, newChild)
-
- def findNext(self, name=None, attrs={}, text=None, **kwargs):
- """Returns the first item that matches the given criteria and
- appears after this Tag in the document."""
- return self._findOne(self.findAllNext, name, attrs, text, **kwargs)
-
- def findAllNext(self, name=None, attrs={}, text=None, limit=None,
- **kwargs):
- """Returns all items that match the given criteria and appear
- before after Tag in the document."""
- return self._findAll(name, attrs, text, limit, self.nextGenerator)
-
- def findNextSibling(self, name=None, attrs={}, text=None, **kwargs):
- """Returns the closest sibling to this Tag that matches the
- given criteria and appears after this Tag in the document."""
- return self._findOne(self.findNextSiblings, name, attrs, text,
- **kwargs)
-
- def findNextSiblings(self, name=None, attrs={}, text=None, limit=None,
- **kwargs):
- """Returns the siblings of this Tag that match the given
- criteria and appear after this Tag in the document."""
- return self._findAll(name, attrs, text, limit,
- self.nextSiblingGenerator, **kwargs)
- fetchNextSiblings = findNextSiblings # Compatibility with pre-3.x
-
- def findPrevious(self, name=None, attrs={}, text=None, **kwargs):
- """Returns the first item that matches the given criteria and
- appears before this Tag in the document."""
- return self._findOne(self.findAllPrevious, name, attrs, text, **kwargs)
-
- def findAllPrevious(self, name=None, attrs={}, text=None, limit=None,
- **kwargs):
- """Returns all items that match the given criteria and appear
- before this Tag in the document."""
- return self._findAll(name, attrs, text, limit, self.previousGenerator,
- **kwargs)
- fetchPrevious = findAllPrevious # Compatibility with pre-3.x
-
- def findPreviousSibling(self, name=None, attrs={}, text=None, **kwargs):
- """Returns the closest sibling to this Tag that matches the
- given criteria and appears before this Tag in the document."""
- return self._findOne(self.findPreviousSiblings, name, attrs, text,
- **kwargs)
-
- def findPreviousSiblings(self, name=None, attrs={}, text=None,
- limit=None, **kwargs):
- """Returns the siblings of this Tag that match the given
- criteria and appear before this Tag in the document."""
- return self._findAll(name, attrs, text, limit,
- self.previousSiblingGenerator, **kwargs)
- fetchPreviousSiblings = findPreviousSiblings # Compatibility with pre-3.x
-
- def findParent(self, name=None, attrs={}, **kwargs):
- """Returns the closest parent of this Tag that matches the given
- criteria."""
- # NOTE: We can't use _findOne because findParents takes a different
- # set of arguments.
- r = None
- l = self.findParents(name, attrs, 1)
- if l:
- r = l[0]
- return r
-
- def findParents(self, name=None, attrs={}, limit=None, **kwargs):
- """Returns the parents of this Tag that match the given
- criteria."""
-
- return self._findAll(name, attrs, None, limit, self.parentGenerator,
- **kwargs)
- fetchParents = findParents # Compatibility with pre-3.x
-
- # These methods do the real heavy lifting.
-
- def _findOne(self, method, name, attrs, text, **kwargs):
- r = None
- l = method(name, attrs, text, 1, **kwargs)
- if l:
- r = l[0]
- return r
-
- def _findAll(self, name, attrs, text, limit, generator, **kwargs):
- "Iterates over a generator looking for things that match."
-
- if isinstance(name, SoupStrainer):
- strainer = name
- else:
- # Build a SoupStrainer
- strainer = SoupStrainer(name, attrs, text, **kwargs)
- results = ResultSet(strainer)
- g = generator()
- while True:
- try:
- i = next(g)
- except StopIteration:
- break
- if i:
- found = strainer.search(i)
- if found:
- results.append(found)
- if limit and len(results) >= limit:
- break
- return results
-
- # These Generators can be used to navigate starting from both
- # NavigableStrings and Tags.
- def nextGenerator(self):
- i = self
- while i:
- i = i.next
- yield i
-
- def nextSiblingGenerator(self):
- i = self
- while i:
- i = i.nextSibling
- yield i
-
- def previousGenerator(self):
- i = self
- while i:
- i = i.previous
- yield i
-
- def previousSiblingGenerator(self):
- i = self
- while i:
- i = i.previousSibling
- yield i
-
- def parentGenerator(self):
- i = self
- while i:
- i = i.parent
- yield i
-
- # Utility methods
- def substituteEncoding(self, str, encoding=None):
- encoding = encoding or "utf-8"
- return str.replace("%SOUP-ENCODING%", encoding)
-
- def toEncoding(self, s, encoding=None):
- """Encodes an object to a string in some encoding, or to Unicode.
- ."""
- if not isinstance(s, six.string_types) and not isinstance(s, six.binary_type):
- s = str(s)
- if isinstance(s, six.text_type):
- if encoding:
- s = s.encode(encoding)
- else:
- if encoding:
- s = s.encode(encoding)
- else:
- s = six.text_type(s)
- return s
-
-
-class NavigableString(six.text_type, PageElement):
-
- def __getattr__(self, attr):
- """text.string gives you text. This is for backwards
- compatibility for Navigable*String, but for CData* it lets you
- get the string without the CData wrapper."""
- if attr == 'string':
- return self
- else:
- raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, attr))
-
- def __unicode__(self):
- return self.__str__()
-
- def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING):
- if encoding:
- return self.encode(encoding)
- else:
- return self
-
-
-class CData(NavigableString):
-
- def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING):
- return "" % NavigableString.__str__(self, encoding)
-
-
-class ProcessingInstruction(NavigableString):
- def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING):
- output = self
- if "%SOUP-ENCODING%" in output:
- output = self.substituteEncoding(output, encoding)
- return "" % self.toEncoding(output, encoding)
-
-
-class Comment(NavigableString):
- def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING):
- return "" % NavigableString.__str__(self, encoding)
-
-
-class Declaration(NavigableString):
- def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING):
- return "" % NavigableString.__str__(self, encoding)
-
-
-class Tag(PageElement):
- """Represents a found HTML tag with its attributes and contents."""
-
- XML_ENTITIES_TO_CHARS = {'apos': "'",
- "quot": '"',
- "amp": "&",
- "lt": "<",
- "gt": ">"}
- # An RE for finding ampersands that aren't the start of of a
- # numeric entity.
- BARE_AMPERSAND = re.compile("&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)")
-
- def __init__(self, parser, name, attrs=None, parent=None,
- previous=None):
- "Basic constructor."
-
- # We don't actually store the parser object: that lets extracted
- # chunks be garbage-collected
- self.parserClass = parser.__class__
- self.isSelfClosing = parser.isSelfClosingTag(name)
- self.convertHTMLEntities = parser.convertHTMLEntities
- self.name = name
- if attrs is None:
- attrs = []
- self.attrs = attrs
- self.contents = []
- self.setup(parent, previous)
- self.hidden = False
- self.containsSubstitutions = False
-
- def get(self, key, default=None):
- """Returns the value of the 'key' attribute for the tag, or
- the value given for 'default' if it doesn't have that
- attribute."""
- return self._getAttrMap().get(key, default)
-
- def has_key(self, key):
- return key in self._getAttrMap()
-
- def __getitem__(self, key):
- """tag[key] returns the value of the 'key' attribute for the tag,
- and throws an exception if it's not there."""
- return self._getAttrMap()[key]
-
- def __iter__(self):
- "Iterating over a tag iterates over its contents."
- return iter(self.contents)
-
- def __len__(self):
- "The length of a tag is the length of its list of contents."
- return len(self.contents)
-
- def __contains__(self, x):
- return x in self.contents
-
- def __bool__(self):
- "A tag is non-None even if it has no contents."
- return True
- __nonzero__ = __bool__
-
- def __setitem__(self, key, value):
- """Setting tag[key] sets the value of the 'key' attribute for the
- tag."""
- self._getAttrMap()
- self.attrMap[key] = value
- found = False
- for i in range(0, len(self.attrs)):
- if self.attrs[i][0] == key:
- self.attrs[i] = (key, value)
- found = True
- if not found:
- self.attrs.append((key, value))
- self._getAttrMap()[key] = value
-
- def __delitem__(self, key):
- "Deleting tag[key] deletes all 'key' attributes for the tag."
- for item in self.attrs:
- if item[0] == key:
- self.attrs.remove(item)
- # We don't break because bad HTML can define the same
- # attribute multiple times.
- self._getAttrMap()
- if key in self.attrMap:
- del self.attrMap[key]
-
- def __call__(self, *args, **kwargs):
- """Calling a tag like a function is the same as calling its
- findAll() method. Eg. tag('a') returns a list of all the A tags
- found within this tag."""
- return self.findAll(*args, **kwargs)
-
- def __getattr__(self, tag):
- if len(tag) > 3 and tag.rfind('Tag') == len(tag) - 3:
- return self.find(tag[:-3])
- elif tag.find('__') != 0:
- return self.find(tag)
-
- def __eq__(self, other):
- """Returns true iff this tag has the same name, the same attributes,
- and the same contents (recursively) as the given tag.
-
- NOTE: right now this will return false if two tags have the
- same attributes in a different order. Should this be fixed?"""
- if not hasattr(other, 'name') or not hasattr(other, 'attrs') or not hasattr(other, 'contents') or self.name != other.name or self.attrs != other.attrs or len(self) != len(other):
- return False
- for i in range(0, len(self.contents)):
- if self.contents[i] != other.contents[i]:
- return False
- return True
-
- def __ne__(self, other):
- """Returns true iff this tag is not identical to the other tag,
- as defined in __eq__."""
- return not self == other
-
- def __repr__(self, encoding=DEFAULT_OUTPUT_ENCODING):
- """Renders this tag as a string."""
- return self.__str__(encoding)
-
- def __unicode__(self):
- return self.__str__(None)
-
- def _convertEntities(self, match):
- x = match.group(1)
- if x in name2codepoint:
- return six.unichr(name2codepoint[x])
- elif "&" + x + ";" in self.XML_ENTITIES_TO_CHARS:
- return '&%s;' % x
- else:
- return '&%s;' % x
-
- def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING,
- prettyPrint=False, indentLevel=0):
- """Returns a string or Unicode representation of this tag and
- its contents. To get Unicode, pass None for encoding.
-
- NOTE: since Python's HTML parser consumes whitespace, this
- method is not certain to reproduce the whitespace present in
- the original string."""
-
- encodedName = self.toEncoding(self.name, encoding)
-
- attrs = []
- if self.attrs:
- for key, val in self.attrs:
- fmt = '%s="%s"'
- if isString(val):
- if self.containsSubstitutions and '%SOUP-ENCODING%' in val:
- val = self.substituteEncoding(val, encoding)
-
- # The attribute value either:
- #
- # * Contains no embedded double quotes or single quotes.
- # No problem: we enclose it in double quotes.
- # * Contains embedded single quotes. No problem:
- # double quotes work here too.
- # * Contains embedded double quotes. No problem:
- # we enclose it in single quotes.
- # * Embeds both single _and_ double quotes. This
- # can't happen naturally, but it can happen if
- # you modify an attribute value after parsing
- # the document. Now we have a bit of a
- # problem. We solve it by enclosing the
- # attribute in single quotes, and escaping any
- # embedded single quotes to XML entities.
- if '"' in val:
- # This can't happen naturally, but it can happen
- # if you modify an attribute value after parsing.
- if "'" in val:
- val = val.replace('"', """)
- else:
- fmt = "%s='%s'"
-
- # Optionally convert any HTML entities
- if self.convertHTMLEntities:
- val = re.sub("&(\w+);", self._convertEntities, val)
-
- # Now we're okay w/r/t quotes. But the attribute
- # value might also contain angle brackets, or
- # ampersands that aren't part of entities. We need
- # to escape those to XML entities too.
- val = val.replace("<", "<").replace(">", ">")
- val = self.BARE_AMPERSAND.sub("&", val)
-
- attrs.append(fmt % (self.toEncoding(key, encoding),
- self.toEncoding(val, encoding)))
- close = ''
- closeTag = ''
- if self.isSelfClosing:
- close = ' /'
- else:
- closeTag = '' % encodedName
-
- indentTag, indentContents = 0, 0
- if prettyPrint:
- indentTag = indentLevel
- space = (' ' * (indentTag - 1))
- indentContents = indentTag + 1
- contents = self.renderContents(encoding, prettyPrint, indentContents)
- if self.hidden:
- s = contents
- else:
- s = []
- attributeString = ''
- if attrs:
- attributeString = ' ' + ' '.join(attrs)
- if prettyPrint:
- s.append(space)
- s.append('<%s%s%s>' % (encodedName, attributeString, close))
- if prettyPrint:
- s.append("\n")
- s.append(contents)
- if prettyPrint and contents and contents[-1] != "\n":
- s.append("\n")
- if prettyPrint and closeTag:
- s.append(space)
- s.append(closeTag)
- if prettyPrint and closeTag and self.nextSibling:
- s.append("\n")
- s = ''.join(s)
- return s
-
- def prettify(self, encoding=DEFAULT_OUTPUT_ENCODING):
- return self.__str__(encoding, True)
-
- def renderContents(self, encoding=DEFAULT_OUTPUT_ENCODING,
- prettyPrint=False, indentLevel=0):
- """Renders the contents of this tag as a string in the given
- encoding. If encoding is None, returns a Unicode string.."""
- s = []
- for c in self:
- text = None
- if isinstance(c, NavigableString):
- text = c.__str__(encoding)
- elif isinstance(c, Tag):
- s.append(c.__str__(encoding, prettyPrint, indentLevel))
- if text and prettyPrint:
- text = text.strip()
- if text:
- if prettyPrint:
- s.append(" " * (indentLevel - 1))
- s.append(text)
- if prettyPrint:
- s.append("\n")
- return ''.join(s)
-
- # Soup methods
-
- def find(self, name=None, attrs={}, recursive=True, text=None,
- **kwargs):
- """Return only the first child of this Tag matching the given
- criteria."""
- r = None
- l = self.findAll(name, attrs, recursive, text, 1, **kwargs)
- if l:
- r = l[0]
- return r
- findChild = find
-
- def findAll(self, name=None, attrs={}, recursive=True, text=None,
- limit=None, **kwargs):
- """Extracts a list of Tag objects that match the given
- criteria. You can specify the name of the Tag and any
- attributes you want the Tag to have.
-
- The value of a key-value pair in the 'attrs' map can be a
- string, a list of strings, a regular expression object, or a
- callable that takes a string and returns whether or not the
- string matches for some custom definition of 'matches'. The
- same is true of the tag name."""
- generator = self.recursiveChildGenerator
- if not recursive:
- generator = self.childGenerator
- return self._findAll(name, attrs, text, limit, generator, **kwargs)
- findChildren = findAll
-
- # Pre-3.x compatibility methods
- first = find
- fetch = findAll
-
- def fetchText(self, text=None, recursive=True, limit=None):
- return self.findAll(text=text, recursive=recursive, limit=limit)
-
- def firstText(self, text=None, recursive=True):
- return self.find(text=text, recursive=recursive)
-
- # Utility methods
-
- def append(self, tag):
- """Appends the given tag to the contents of this tag."""
- self.contents.append(tag)
-
- # Private methods
-
- def _getAttrMap(self):
- """Initializes a map representation of this tag's attributes,
- if not already initialized."""
- if not getattr(self, 'attrMap'):
- self.attrMap = {}
- for (key, value) in self.attrs:
- self.attrMap[key] = value
- return self.attrMap
-
- # Generator methods
- def childGenerator(self):
- for i in range(0, len(self.contents)):
- yield self.contents[i]
- raise StopIteration
-
- def recursiveChildGenerator(self):
- stack = [(self, 0)]
- while stack:
- tag, start = stack.pop()
- if isinstance(tag, Tag):
- for i in range(start, len(tag.contents)):
- a = tag.contents[i]
- yield a
- if isinstance(a, Tag) and tag.contents:
- if i < len(tag.contents) - 1:
- stack.append((tag, i + 1))
- stack.append((a, 0))
- break
- raise StopIteration
-
-
-# Next, a couple classes to represent queries and their results.
-class SoupStrainer:
- """Encapsulates a number of ways of matching a markup element (tag or
- text)."""
-
- def __init__(self, name=None, attrs={}, text=None, **kwargs):
- self.name = name
- if isString(attrs):
- kwargs['class'] = attrs
- attrs = None
- if kwargs:
- if attrs:
- attrs = attrs.copy()
- attrs.update(kwargs)
- else:
- attrs = kwargs
- self.attrs = attrs
- self.text = text
-
- def __str__(self):
- if self.text:
- return self.text
- else:
- return "%s|%s" % (self.name, self.attrs)
-
- def searchTag(self, markupName=None, markupAttrs={}):
- found = None
- markup = None
- if isinstance(markupName, Tag):
- markup = markupName
- markupAttrs = markup
- callFunctionWithTagData = callable(self.name) and \
- not isinstance(markupName, Tag)
-
- if not self.name or callFunctionWithTagData or \
- (markup and self._matches(markup, self.name)) or \
- (not markup and self._matches(markupName, self.name)):
- if callFunctionWithTagData:
- match = self.name(markupName, markupAttrs)
- else:
- match = True
- markupAttrMap = None
- for attr, matchAgainst in self.attrs.items():
- if not markupAttrMap:
- if hasattr(markupAttrs, 'get'):
- markupAttrMap = markupAttrs
- else:
- markupAttrMap = {}
- for k, v in markupAttrs:
- markupAttrMap[k] = v
- attrValue = markupAttrMap.get(attr)
- if not self._matches(attrValue, matchAgainst):
- match = False
- break
- if match:
- if markup:
- found = markup
- else:
- found = markupName
- return found
-
- def search(self, markup):
- found = None
- # If given a list of items, scan it for a text element that
- # matches.
- if isList(markup) and not isinstance(markup, Tag):
- for element in markup:
- if isinstance(element, NavigableString) and \
- self.search(element):
- found = element
- break
- # If it's a Tag, make sure its name or attributes match.
- # Don't bother with Tags if we're searching for text.
- elif isinstance(markup, Tag):
- if not self.text:
- found = self.searchTag(markup)
- # If it's text, make sure the text matches.
- elif isinstance(markup, NavigableString) or isString(markup):
- if self._matches(markup, self.text):
- found = markup
- else:
- raise Exception("I don't know how to match against a %s" %
- markup.__class__)
- return found
-
- def _matches(self, markup, matchAgainst):
- result = False
- if matchAgainst is True:
- result = markup is not None
- elif callable(matchAgainst):
- result = matchAgainst(markup)
- else:
- # Custom match methods take the tag as an argument, but all
- # other ways of matching match the tag name as a string.
- if isinstance(markup, Tag):
- markup = markup.name
- if markup and not isString(markup):
- markup = six.text_type(markup)
- # Now we know that chunk is either a string, or None.
- if hasattr(matchAgainst, 'match'):
- # It's a regexp object.
- result = markup and matchAgainst.search(markup)
- elif isList(matchAgainst):
- result = markup in matchAgainst
- elif hasattr(matchAgainst, 'items'):
- result = matchAgainst in markup
- elif matchAgainst and isString(markup):
- if isinstance(markup, six.text_type):
- matchAgainst = six.text_type(matchAgainst)
- else:
- matchAgainst = str(matchAgainst)
-
- if not result:
- result = matchAgainst == markup
- return result
-
-
-class ResultSet(list):
- """A ResultSet is just a list that keeps track of the SoupStrainer
- that created it."""
-
- def __init__(self, source):
- list.__init__([])
- self.source = source
-
-# Now, some helper functions.
-
-
-def isList(l):
- """Convenience method to determine whether or not something is listlike."""
- return hasattr(l, '__iter__') or \
- type(l) in (list, tuple)
-
-
-def isString(s):
- """Convenience method to determine whether or not something is stringlike."""
- return isinstance(s, six.string_types)
-
-
-def buildTagMap(default, *args):
- """Turns a list of maps, lists, or scalars into a single map.
- Used to build the SELF_CLOSING_TAGS, NESTABLE_TAGS, and
- NESTING_RESET_TAGS maps out of lists and partial maps."""
- built = {}
- for portion in args:
- if hasattr(portion, 'items'):
- # It's a map. Merge it.
- for k, v in portion.items():
- built[k] = v
- elif isList(portion):
- # It's a list. Map each item to the default.
- for k in portion:
- built[k] = default
- else:
- # It's a scalar. Map it to the default.
- built[portion] = default
- return built
-
-# Now, the parser classes.
-
-
-class BeautifulStoneSoup(Tag, SGMLParser):
-
- """This class contains the basic parser and search code. It defines
- a parser that knows nothing about tag behavior except for the
- following:
-
- You can't close a tag without closing all the tags it encloses.
- That is, "" actually means
- "".
-
- [Another possible explanation is "", but since
- this class defines no SELF_CLOSING_TAGS, it will never use that
- explanation.]
-
- This class is useful for parsing XML or made-up markup languages,
- or when BeautifulSoup makes an assumption counter to what you were
- expecting."""
-
- SELF_CLOSING_TAGS = {}
- NESTABLE_TAGS = {}
- RESET_NESTING_TAGS = {}
- QUOTE_TAGS = {}
-
- MARKUP_MASSAGE = [(re.compile('(<[^<>]*)/>'),
- lambda x: x.group(1) + ' />'),
- (re.compile(']*)>'),
- lambda x: '')
- ]
-
- ROOT_TAG_NAME = u'[document]'
-
- HTML_ENTITIES = "html"
- XML_ENTITIES = "xml"
- ALL_ENTITIES = [HTML_ENTITIES, XML_ENTITIES]
-
- def __init__(self, markup="", parseOnlyThese=None, fromEncoding=None,
- markupMassage=True, smartQuotesTo=XML_ENTITIES,
- convertEntities=None, selfClosingTags=None):
- """The Soup object is initialized as the 'root tag', and the
- provided markup (which can be a string or a file-like object)
- is fed into the underlying parser.
-
- sgmllib will process most bad HTML, and the BeautifulSoup
- class has some tricks for dealing with some HTML that kills
- sgmllib, but Beautiful Soup can nonetheless choke or lose data
- if your data uses self-closing tags or declarations
- incorrectly.
-
- By default, Beautiful Soup uses regexes to sanitize input,
- avoiding the vast majority of these problems. If the problems
- don't apply to you, pass in False for markupMassage, and
- you'll get better performance.
-
- The default parser massage techniques fix the two most common
- instances of invalid HTML that choke sgmllib:
-
-
(No space between name of closing tag and tag close)
- (Extraneous whitespace in declaration)
-
- You can pass in a custom list of (RE object, replace method)
- tuples to get Beautiful Soup to scrub your input the way you
- want."""
-
- self.parseOnlyThese = parseOnlyThese
- self.fromEncoding = fromEncoding
- self.smartQuotesTo = smartQuotesTo
-
- if convertEntities:
- # It doesn't make sense to convert encoded characters to
- # entities even while you're converting entities to Unicode.
- # Just convert it all to Unicode.
- self.smartQuotesTo = None
-
- if isList(convertEntities):
- self.convertHTMLEntities = self.HTML_ENTITIES in convertEntities
- self.convertXMLEntities = self.XML_ENTITIES in convertEntities
- else:
- self.convertHTMLEntities = self.HTML_ENTITIES == convertEntities
- self.convertXMLEntities = self.XML_ENTITIES == convertEntities
-
- self.instanceSelfClosingTags = buildTagMap(None, selfClosingTags)
- SGMLParser.__init__(self)
-
- if hasattr(markup, 'read'): # It's a file-type object.
- markup = markup.read()
- self.markup = markup
- self.markupMassage = markupMassage
- try:
- self._feed()
- except StopParsing:
- pass
- self.markup = None # The markup can now be GCed
-
- def _feed(self, inDocumentEncoding=None):
- # Convert the document to Unicode.
- markup = self.markup
- if isinstance(markup, six.text_type):
- if not hasattr(self, 'originalEncoding'):
- self.originalEncoding = None
- else:
- dammit = UnicodeDammit(markup,
- [self.fromEncoding, inDocumentEncoding],
- smartQuotesTo=self.smartQuotesTo)
- markup = dammit.unicode
- self.originalEncoding = dammit.originalEncoding
- if markup:
- if self.markupMassage:
- if not isList(self.markupMassage):
- self.markupMassage = self.MARKUP_MASSAGE
- for fix, m in self.markupMassage:
- markup = fix.sub(m, markup)
- self.reset()
-
- SGMLParser.feed(self, markup or "")
- SGMLParser.close(self)
- # Close out any unfinished strings and close all the open tags.
- self.endData()
- while self.currentTag.name != self.ROOT_TAG_NAME:
- self.popTag()
-
- def __getattr__(self, methodName):
- """This method routes method call requests to either the SGMLParser
- superclass or the Tag superclass, depending on the method name."""
- if methodName.find('start_') == 0 or methodName.find('end_') == 0 or \
- methodName.find('do_') == 0:
- return SGMLParser.__getattr__(self, methodName)
- elif methodName.find('__') != 0:
- return Tag.__getattr__(self, methodName)
- else:
- raise AttributeError
-
- def isSelfClosingTag(self, name):
- """Returns true iff the given string is the name of a
- self-closing tag according to this parser."""
- return name in self.SELF_CLOSING_TAGS or \
- name in self.instanceSelfClosingTags
-
- def reset(self):
- Tag.__init__(self, self, self.ROOT_TAG_NAME)
- self.hidden = 1
- SGMLParser.reset(self)
- self.currentData = []
- self.currentTag = None
- self.tagStack = []
- self.quoteStack = []
- self.pushTag(self)
-
- def popTag(self):
- self.tagStack.pop()
- # Tags with just one string-owning child get the child as a
- # 'string' property, so that soup.tag.string is shorthand for
- # soup.tag.contents[0]
- if len(self.currentTag.contents) == 1 and \
- isinstance(self.currentTag.contents[0], NavigableString):
- self.currentTag.string = self.currentTag.contents[0]
-
- if self.tagStack:
- self.currentTag = self.tagStack[-1]
- return self.currentTag
-
- def pushTag(self, tag):
- if self.currentTag:
- self.currentTag.append(tag)
- self.tagStack.append(tag)
- self.currentTag = self.tagStack[-1]
-
- def endData(self, containerClass=NavigableString):
- if self.currentData:
- currentData = ''.join(self.currentData)
- if currentData.endswith('<') and self.convertHTMLEntities:
- currentData = currentData[:-1] + '<'
- if not currentData.strip():
- if '\n' in currentData:
- currentData = '\n'
- else:
- currentData = ' '
- self.currentData = []
- if self.parseOnlyThese and len(self.tagStack) <= 1 and \
- (not self.parseOnlyThese.text or
- not self.parseOnlyThese.search(currentData)):
- return
- o = containerClass(currentData)
- o.setup(self.currentTag, self.previous)
- if self.previous:
- self.previous.next = o
- self.previous = o
- self.currentTag.contents.append(o)
-
- def _popToTag(self, name, inclusivePop=True):
- """Pops the tag stack up to and including the most recent
- instance of the given tag. If inclusivePop is false, pops the tag
- stack up to but *not* including the most recent instqance of
- the given tag."""
- if name == self.ROOT_TAG_NAME:
- return
-
- numPops = 0
- mostRecentTag = None
- for i in range(len(self.tagStack) - 1, 0, -1):
- if name == self.tagStack[i].name:
- numPops = len(self.tagStack) - i
- break
- if not inclusivePop:
- numPops = numPops - 1
-
- for i in range(0, numPops):
- mostRecentTag = self.popTag()
- return mostRecentTag
-
- def _smartPop(self, name):
- """We need to pop up to the previous tag of this type, unless
- one of this tag's nesting reset triggers comes between this
- tag and the previous tag of this type, OR unless this tag is a
- generic nesting trigger and another generic nesting trigger
- comes between this tag and the previous tag of this type.
-
- Examples:
- FooBar
should pop to 'p', not 'b'.
-
Foo
Bar should pop to 'table', not 'p'.
-
Foo
Bar should pop to 'tr', not 'p'.
-
FooBar
should pop to 'p', not 'b'.
-
-
- *
- * should pop to 'ul', not the first 'li'.
-
*
* should pop to 'table', not the first 'tr'
- |
| * | * should pop to 'tr', not the first 'td'
- """
-
- nestingResetTriggers = self.NESTABLE_TAGS.get(name)
- isNestable = nestingResetTriggers is not None
- isResetNesting = name in self.RESET_NESTING_TAGS
- popTo = None
- inclusive = True
- for i in range(len(self.tagStack) - 1, 0, -1):
- p = self.tagStack[i]
- if (not p or p.name == name) and not isNestable:
- # Non-nestable tags get popped to the top or to their
- # last occurance.
- popTo = name
- break
- if (nestingResetTriggers is not None and
- p.name in nestingResetTriggers) or \
- (nestingResetTriggers is None and isResetNesting and
- p.name in self.RESET_NESTING_TAGS):
- # If we encounter one of the nesting reset triggers
- # peculiar to this tag, or we encounter another tag
- # that causes nesting to reset, pop up to but not
- # including that tag.
- popTo = p.name
- inclusive = False
- break
- p = p.parent
- if popTo:
- self._popToTag(popTo, inclusive)
-
- def unknown_starttag(self, name, attrs, selfClosing=0):
- if self.quoteStack:
- # This is not a real tag.
- attrs = ''.join(' %s="%s"' % (x, y) for x, y in attrs)
- self.currentData.append('<%s%s>' % (name, attrs))
- return
- self.endData()
-
- if not self.isSelfClosingTag(name) and not selfClosing:
- self._smartPop(name)
-
- if self.parseOnlyThese and len(self.tagStack) <= 1 and \
- (self.parseOnlyThese.text or not self.parseOnlyThese.searchTag(name, attrs)):
- return
-
- tag = Tag(self, name, attrs, self.currentTag, self.previous)
- if self.previous:
- self.previous.next = tag
- self.previous = tag
- self.pushTag(tag)
- if selfClosing or self.isSelfClosingTag(name):
- self.popTag()
- if name in self.QUOTE_TAGS:
- self.quoteStack.append(name)
- self.literal = 1
- return tag
-
- def unknown_endtag(self, name):
- if self.quoteStack and self.quoteStack[-1] != name:
- # This is not a real end tag.
- self.currentData.append('' % name)
- return
- self.endData()
- self._popToTag(name)
- if self.quoteStack and self.quoteStack[-1] == name:
- self.quoteStack.pop()
- self.literal = (len(self.quoteStack) > 0)
-
- def handle_data(self, data):
- if self.convertHTMLEntities:
- if data[0] == '&':
- data = self.BARE_AMPERSAND.sub("&", data)
- else:
- data = data.replace('&', '&') \
- .replace('<', '<') \
- .replace('>', '>')
- self.currentData.append(data)
-
- def _toStringSubclass(self, text, subclass):
- """Adds a certain piece of text to the tree as a NavigableString
- subclass."""
- self.endData()
- self.handle_data(text)
- self.endData(subclass)
-
- def handle_pi(self, text):
- """Handle a processing instruction as a ProcessingInstruction
- object, possibly one with a %SOUP-ENCODING% slot into which an
- encoding will be plugged later."""
- if text[:3] == "xml":
- text = "xml version='1.0' encoding='%SOUP-ENCODING%'"
- self._toStringSubclass(text, ProcessingInstruction)
-
- def handle_comment(self, text):
- "Handle comments as Comment objects."
- self._toStringSubclass(text, Comment)
-
- def handle_charref(self, ref):
- "Handle character references as data."
- if ref[0] == 'x':
- data = six.unichr(int(ref[1:], 16))
- else:
- data = six.unichr(int(ref))
-
- if u'\x80' <= data <= u'\x9F':
- data = UnicodeDammit.subMSChar(chr(ord(data)), self.smartQuotesTo)
- elif not self.convertHTMLEntities and not self.convertXMLEntities:
- data = '&#%s;' % ref
-
- self.handle_data(data)
-
- def handle_entityref(self, ref):
- """Handle entity references as data, possibly converting known
- HTML entity references to the corresponding Unicode
- characters."""
- replaceWithXMLEntity = self.convertXMLEntities and \
- ref in self.XML_ENTITIES_TO_CHARS
- if self.convertHTMLEntities or replaceWithXMLEntity:
- try:
- data = six.unichr(name2codepoint[ref])
- except KeyError:
- if replaceWithXMLEntity:
- data = self.XML_ENTITIES_TO_CHARS.get(ref)
- else:
- data = "&%s" % ref
- else:
- data = '&%s;' % ref
- self.handle_data(data)
-
- def handle_decl(self, data):
- "Handle DOCTYPEs and the like as Declaration objects."
- self._toStringSubclass(data, Declaration)
-
- def parse_declaration(self, i):
- """Treat a bogus SGML declaration as raw data. Treat a CDATA
- declaration as a CData object."""
- j = None
- if self.rawdata[i:i + 9] == '', i)
- if k == -1:
- k = len(self.rawdata)
- data = self.rawdata[i + 9:k]
- j = k + 3
- self._toStringSubclass(data, CData)
- else:
- try:
- j = SGMLParser.parse_declaration(self, i)
- except SGMLParseError:
- toHandle = self.rawdata[i:]
- self.handle_data(toHandle)
- j = i + len(toHandle)
- return j
-
-
-class BeautifulSoup(BeautifulStoneSoup):
-
- """This parser knows the following facts about HTML:
-
- * Some tags have no closing tag and should be interpreted as being
- closed as soon as they are encountered.
-
- * The text inside some tags (ie. 'script') may contain tags which
- are not really part of the document and which should be parsed
- as text, not tags. If you want to parse the text as tags, you can
- always fetch it and parse it explicitly.
-
- * Tag nesting rules:
-
- Most tags can't be nested at all. For instance, the occurance of
- a tag should implicitly close the previous tag.
-
- Para1 Para2
- should be transformed into:
- Para1 Para2
-
- Some tags can be nested arbitrarily. For instance, the occurance
- of a tag should _not_ implicitly close the previous
- tag.
-
- Alice said: Bob said: Blah
- should NOT be transformed into:
- Alice said: Bob said: Blah
-
- Some tags can be nested, but the nesting is reset by the
- interposition of other tags. For instance, a tag should
- implicitly close the previous tag within the same ,
- but not close a tag in another table.
-
- Blah Blah
- should be transformed into:
- Blah Blah
- but,
- BlahBlah
- should NOT be transformed into
- BlahBlah
-
- Differing assumptions about tag nesting rules are a major source
- of problems with the BeautifulSoup class. If BeautifulSoup is not
- treating as nestable a tag your page author treats as nestable,
- try ICantBelieveItsBeautifulSoup, MinimalSoup, or
- BeautifulStoneSoup before writing your own subclass."""
-
- def __init__(self, *args, **kwargs):
- if 'smartQuotesTo' not in kwargs:
- kwargs['smartQuotesTo'] = self.HTML_ENTITIES
- BeautifulStoneSoup.__init__(self, *args, **kwargs)
-
- SELF_CLOSING_TAGS = buildTagMap(None,
- ['br', 'hr', 'input', 'img', 'meta',
- 'spacer', 'link', 'frame', 'base'])
-
- QUOTE_TAGS = {'script': None}
-
- # According to the HTML standard, each of these inline tags can
- # contain another tag of the same type. Furthermore, it's common
- # to actually use these tags this way.
- NESTABLE_INLINE_TAGS = ['span', 'font', 'q', 'object', 'bdo', 'sub', 'sup',
- 'center']
-
- # According to the HTML standard, these block tags can contain
- # another tag of the same type. Furthermore, it's common
- # to actually use these tags this way.
- NESTABLE_BLOCK_TAGS = ['blockquote', 'div', 'fieldset', 'ins', 'del']
-
- # Lists can contain other lists, but there are restrictions.
- NESTABLE_LIST_TAGS = {'ol': [],
- 'ul': [],
- 'li': ['ul', 'ol'],
- 'dl': [],
- 'dd': ['dl'],
- 'dt': ['dl']}
-
- # Tables can contain other tables, but there are restrictions.
- NESTABLE_TABLE_TAGS = {'table': [],
- 'tr': ['table', 'tbody', 'tfoot', 'thead'],
- 'td': ['tr'],
- 'th': ['tr'],
- 'thead': ['table'],
- 'tbody': ['table'],
- 'tfoot': ['table'],
- }
-
- NON_NESTABLE_BLOCK_TAGS = ['address', 'form', 'p', 'pre']
-
- # If one of these tags is encountered, all tags up to the next tag of
- # this type are popped.
- RESET_NESTING_TAGS = buildTagMap(None, NESTABLE_BLOCK_TAGS, 'noscript',
- NON_NESTABLE_BLOCK_TAGS,
- NESTABLE_LIST_TAGS,
- NESTABLE_TABLE_TAGS)
-
- NESTABLE_TAGS = buildTagMap([], NESTABLE_INLINE_TAGS, NESTABLE_BLOCK_TAGS,
- NESTABLE_LIST_TAGS, NESTABLE_TABLE_TAGS)
-
- # Used to detect the charset in a META tag; see start_meta
- CHARSET_RE = re.compile("((^|;)\s*charset=)([^;]*)")
-
- def start_meta(self, attrs):
- """Beautiful Soup can detect a charset included in a META tag,
- try to convert the document to that charset, and re-parse the
- document from the beginning."""
- httpEquiv = None
- contentType = None
- contentTypeIndex = None
- tagNeedsEncodingSubstitution = False
-
- for i in range(0, len(attrs)):
- key, value = attrs[i]
- key = key.lower()
- if key == 'http-equiv':
- httpEquiv = value
- elif key == 'content':
- contentType = value
- contentTypeIndex = i
-
- if httpEquiv and contentType: # It's an interesting meta tag.
- match = self.CHARSET_RE.search(contentType)
- if match:
- if getattr(self, 'declaredHTMLEncoding') or \
- self.originalEncoding == self.fromEncoding:
- # This is our second pass through the document, or
- # else an encoding was specified explicitly and it
- # worked. Rewrite the meta tag.
- newAttr = self.CHARSET_RE.sub(
- lambda match: match.group(1) + "%SOUP-ENCODING%",
- value)
- attrs[contentTypeIndex] = (attrs[contentTypeIndex][0],
- newAttr)
- tagNeedsEncodingSubstitution = True
- else:
- # This is our first pass through the document.
- # Go through it again with the new information.
- newCharset = match.group(3)
- if newCharset and newCharset != self.originalEncoding:
- self.declaredHTMLEncoding = newCharset
- self._feed(self.declaredHTMLEncoding)
- raise StopParsing
- tag = self.unknown_starttag("meta", attrs)
- if tag and tagNeedsEncodingSubstitution:
- tag.containsSubstitutions = True
-
-
-class StopParsing(Exception):
- pass
-
-
-class ICantBelieveItsBeautifulSoup(BeautifulSoup):
-
- """The BeautifulSoup class is oriented towards skipping over
- common HTML errors like unclosed tags. However, sometimes it makes
- errors of its own. For instance, consider this fragment:
-
- FooBar
-
- This is perfectly valid (if bizarre) HTML. However, the
- BeautifulSoup class will implicitly close the first b tag when it
- encounters the second 'b'. It will think the author wrote
- "FooBar", and didn't close the first 'b' tag, because
- there's no real-world reason to bold something that's already
- bold. When it encounters '' it will close two more 'b'
- tags, for a grand total of three tags closed instead of two. This
- can throw off the rest of your document structure. The same is
- true of a number of other tags, listed below.
-
- It's much more common for someone to forget to close a 'b' tag
- than to actually use nested 'b' tags, and the BeautifulSoup class
- handles the common case. This class handles the not-co-common
- case: where you can't believe someone wrote what they did, but
- it's valid HTML and BeautifulSoup screwed up by assuming it
- wouldn't be."""
-
- I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS = ['em', 'big', 'i', 'small',
- 'tt', 'abbr', 'acronym', 'strong', 'cite', 'code', 'dfn', 'kbd', 'samp',
- 'strong', 'var', 'b', 'big']
-
- I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS = ['noscript']
-
- NESTABLE_TAGS = buildTagMap([], BeautifulSoup.NESTABLE_TAGS,
- I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS,
- I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS)
-
-
-class MinimalSoup(BeautifulSoup):
- """The MinimalSoup class is for parsing HTML that contains
- pathologically bad markup. It makes no assumptions about tag
- nesting, but it does know which tags are self-closing, that
-
|