opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'admin_grid_roles' into admin_grid_groups

Browse Source
This commit is contained in:
guerler
2017-08-10 12:56:50 -04:00
parent 4a77956dc8 be19b36eca
commit b38ccbc105
89 changed files with 1435 additions and 402 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.ci/flake8_lint_include_list.txt
+1 -3
View File
@@ -79,9 +79,7 @@ lib/galaxy/visualization/__init__.py
lib/galaxy/visualization/plugins/__init__.py
lib/galaxy/visualization/plugins/utils.py
lib/galaxy/visualization/tracks/
lib/galaxy/web/base/controllers/__init__.py
lib/galaxy/web/base/__init__.py
lib/galaxy/web/base/interactive_environments.py
lib/galaxy/web/base/
lib/galaxy/web/buildapp.py
lib/galaxy/web/formatting.py
lib/galaxy/web/framework/base.py
.ci/py3_sources.txt
+1
View File
@@ -27,6 +27,7 @@ lib/galaxy/tools/
lib/galaxy/tours/
lib/galaxy/util/
lib/galaxy/visualization/
lib/galaxy/web/base/
lib/galaxy/web/buildapp.py
lib/galaxy/web/framework/base.py
lib/galaxy/web/framework/decorators.py
client/galaxy/scripts/apps/analysis.js
+7 -1
View File
@@ -17,6 +17,7 @@ var jQuery = require( 'jquery' ),
ToolFormComposite = require( 'mvc/tool/tool-form-composite' ),
Utils = require( 'utils/utils' ),
Ui = require( 'mvc/ui/ui-misc' ),
DatasetError = require( 'mvc/dataset/dataset-error' ),
DatasetEditAttributes = require('mvc/dataset/dataset-edit-attributes');
/** define the 'Analyze Data'/analysis/main/home page for Galaxy
@@ -51,7 +52,8 @@ window.app = function app( options, bootstrapped ){
'(/)datasets(/)list(/)' : 'show_datasets',
'(/)workflow/import_workflow' : 'show_import_workflow',
'(/)custom_builds' : 'show_custom_builds',
'(/)datasets/edit': 'show_dataset_edit_attributes'
'(/)datasets/edit': 'show_dataset_edit_attributes',
'(/)datasets/error': 'show_dataset_error'
},
require_login: [
@@ -127,6 +129,10 @@ window.app = function app( options, bootstrapped ){
this.page.display( new DatasetEditAttributes.View() );
},
show_dataset_error : function() {
this.page.display( new DatasetError.View() );
},
/** */
home : function( params ){
// TODO: to router, remove Globals
client/galaxy/scripts/mvc/dataset/dataset-error.js
+176
View File
@@ -0,0 +1,176 @@
define( [ 'utils/utils', 'mvc/ui/ui-misc', 'mvc/form/form-view' ], function( Utils, Ui, Form ) {
/** Dataset edit attributes view */
var View = Backbone.View.extend({
initialize: function() {
this.setElement( '<div/>' );
this.model = new Backbone.Model( { 'dataset_id': Galaxy.params.dataset_id } );
this.render();
},
// Fetch data for the selected dataset and
render: function() {
var data_url = Galaxy.root + 'api/datasets/' + this.model.get( 'dataset_id' ),
self = this;
Utils.get({
url : data_url,
success : function( dataset ) {
var job_url = Galaxy.root + 'api/jobs/' + dataset.creating_job + '?full=True';
Utils.get({
url : job_url,
success : function( job ) {
var job_url = Galaxy.root + 'api/jobs/' + dataset.creating_job + '?full=True';
self.render_error_page( self, dataset, job );
},
error : function( response ) {
var error_response = {
'status': 'error',
'message': 'Error occured while loading the job.',
'persistent': true,
'cls': 'errormessage'
};
self.display_message( error_response, self.$( '.response-message' ) );
}
});
},
error : function( response ) {
var error_response = {
'status': 'error',
'message': 'Error occured while loading the dataset.',
'persistent': true,
'cls': 'errormessage'
};
self.display_message( error_response, self.$( '.response-message' ) );
}
});
},
/** Render the view */
render_error_page: function( self, dataset, job ) {
self.$el.empty().append( self._templateHeader() );
self.$el.append('<h2>Dataset Error</h2>');
self.$el.append('<p>An error occured while running the tool <b>' + job.tool_id + '</b>.</p>');
self.$el.append('<p>Tool execution generated the following messages:</p>');
self.$el.append('<pre class="code">' + job.stderr + '</pre>');
self.$el.append('<h2>Report This Error</pre>');
self.$el.append('<p>Usually the local Galaxy administrators regularly review errors that occur on the server. However, if you would like to provide additional information (such as what you were trying to do when the error occurred) and a contact e-mail address, we will be better able to investigate your problem and get back to you.</p>');
self.$el.append(self._getBugFormTemplate(dataset, job));
},
/** Display actions messages */
display_message: function( response, $el, doNotClear, safe ) {
if(!safe){
if(doNotClear) {
$el.append( new Ui.Message( response ).$el );
} else {
$el.empty().html( new Ui.Message( response ).$el );
}
} else {
if(doNotClear) {
$el.append( new Ui.UnescapedMessage( response ).$el );
} else {
$el.empty().html( new Ui.UnescapedMessage( response ).$el );
}
}
},
/** Main template */
_templateHeader: function() {
return '<div class="page-container edit-attr">' +
'<div class="response-message"></div>' +
'</div>';
},
/** Convert tab template */
_getBugFormTemplate: function(dataset, job) {
var self = this;
var inputs = [
{
"help": "Your email address",
"options": [],
"type": "text",
"name": "email",
"label": "Your email",
"value": Galaxy.user.get('email')
},
{
"help": "Any additional comments you can provide regarding what you were doing at the time of the bug.",
"options": [],
"type": "text",
"area": true,
"name": "message",
"label": "Message"
}
];
// TODO
if(false && response.any_public){
inputs.push({
"name": "public_consent",
"label": "Public Disclosure Consent",
"help": "This Galaxy is configured to report to one or more error reporting backends that public to the world. By selecting 'yes', you acknowledge that this bug report will be made public.",
"value": String( Boolean( false ) ),
"options": [],
"type": "boolean",
})
}
var form = new Form({
title : 'Error Report',
inputs : inputs,
buttons : {
save : new Ui.Button({
icon : 'fa-bug',
title : 'Report',
cls : 'ui-button btn btn-primary',
floating : 'clear',
onclick : function() {
var form_data = form.data.create();
var url = Galaxy.root + 'api/jobs/' + job.id + '/error'
form_data.dataset_id = dataset.id;
self.submit(form_data, url)
}
})
},
});
return form.$el;
},
/** Make ajax request */
submit : function(form_data, url){
var self = this;
// Some required metadata
$.ajax({
type: "POST",
url: url,
data: form_data,
success: function( response ) {
// Clear out the div
self.$el.empty().append( self._templateHeader() );
// And display the messages.
response.messages.forEach(function(message){
self.display_message( {
'status': message[1],
'message': message[0],
'persistent': true,
}, self.$( '.response-message' ), true, true );
});
},
error : function( response ) {
var error_response = {
'status': 'error',
'message': 'Error occured while saving. Please fill all the required fields and try again.',
'persistent': true,
'cls': 'errormessage'
};
self.display_message( error_response, self.$( '.response-message' ) );
}
});
}
});
return {
View : View
};
});
client/galaxy/scripts/mvc/dataset/dataset-li-edit.js
+1 -2
View File
@@ -197,9 +197,8 @@ var DatasetListItemEdit = _super.extend(
_renderErrButton : function(){
return faIconButton({
title : _l( 'View or report this error' ),
href : this.model.urls.report_error,
href : Galaxy.root + 'datasets/error?dataset_id=' + this.model.attributes.id,
classes : 'report-error-btn',
target : this.linkTarget,
faIcon : 'fa-bug'
});
},
client/galaxy/scripts/mvc/job/job-params.js
+63
View File
@@ -0,0 +1,63 @@
/** Workflow view */
define( [ 'utils/utils' ], function( Utils ) {
/** Build messages after user action */
function build_messages( self ) {
var $el_message = self.$el.find( '.response-message' ),
status = Utils.getQueryString( 'status' ),
message = Utils.getQueryString( 'message' );
if( message && message !== null && message !== "" ) {
$el_message.addClass( status + 'message' );
$el_message.html( '<p>' + _.escape( message ) + '</p>' );
}
else {
$el_message.html("");
}
}
/** View of the main workflow list page */
var View = Backbone.View.extend({
initialize: function( options ) {
var self = this;
this.options = options;
this.setElement( '<div/>' );
this.render();
},
render: function() {
console.log('HI');
var self = this;
self.$el.empty().append( '<h1>Testing</h1>' );
//var self = this,
//min_query_length = 3;
//$.getJSON( Galaxy.root + 'api/workflows/', function( workflows ) {
//var $el_workflow = null;
//// Add workflow header
//// Add user actions message if any
//build_messages( self );
//$el_workflow = self.$el.find( '.user-workflows' );
//// Add the actions buttons
//$el_workflow.append( self._templateActionButtons() );
//if( workflows.length > 0) {
//$el_workflow.append( self._templateWorkflowTable( self, workflows) );
//self.adjust_actiondropdown( $el_workflow );
//// Register delete and run workflow events
//_.each( workflows, function( wf ) {
//self.confirm_delete( self, wf );
//});
//// Register search workflow event
//self.search_workflow( self, self.$el.find( '.search-wf' ), self.$el.find( '.workflow-search tr' ), min_query_length );
//}
//else {
//$el_workflow.append( self._templateNoWorkflow() );
//}
//});
}
});
return {
View : View
};
});
client/galaxy/scripts/mvc/library/library-dataset-view.js
+19
View File
@@ -721,6 +721,12 @@ var LibraryDatasetView = Backbone.View.extend({
'<td scope="row"><%= _.escape(item.get("misc_info")) %></td>',
'</tr>',
'<% } %>',
'<% if (item.get("tags")) { %>',
'<tr>',
'<th scope="row">Tags</th>',
'<td scope="row"><%= _.escape(item.get("tags")) %></td>',
'</tr>',
'<% } %>',
'</table>',
'<div>',
@@ -850,6 +856,12 @@ var LibraryDatasetView = Backbone.View.extend({
'<td scope="row"><%= _.escape(ldda.get("misc_info")) %></td>',
'</tr>',
'<% } %>',
'<% if (item.get("tags")) { %>',
'<tr>',
'<th scope="row">Tags</th>',
'<td scope="row"><%= _.escape(item.get("tags")) %></td>',
'</tr>',
'<% } %>',
'</table>',
'<div>',
'<pre class="peek">',
@@ -951,6 +963,13 @@ var LibraryDatasetView = Backbone.View.extend({
'<th scope="row">Miscellaneous blurb</th>',
'<td scope="row"><%= _.escape(item.get("misc_blurb")) %></td>',
'</tr>',
//TODO: add functionality to modify tags here
'<% if (item.get("tags")) { %>',
'<tr>',
'<th scope="row">Tags</th>',
'<td scope="row"><%= _.escape(item.get("tags")) %></td>',
'</tr>',
'<% } %>',
'</table>',
'<div>',
'<pre class="peek">',
client/galaxy/scripts/mvc/library/library-foldertoolbar-view.js
+25 -12
View File
@@ -675,6 +675,7 @@ var FolderToolbarView = Backbone.View.extend({
var link_data = this.modal.$el.find( '.link-checkbox' ).is( ':checked' );
var file_type = this.select_extension.value();
var dbkey = this.select_genome.value();
var tag_using_filenames = this.modal.$el.find( '.tag-files' ).is( ':checked' );
var selection_type = selected_nodes[0].type;
var paths = [];
if ( selected_nodes.length < 1 ){
@@ -694,13 +695,15 @@ var FolderToolbarView = Backbone.View.extend({
link_data: link_data,
source: full_source,
file_type: file_type,
dbkey: dbkey } );
dbkey: dbkey,
tag_using_filenames: tag_using_filenames } );
} else if ( selection_type === 'file' ){
var full_source = options.source + '_file';
this.chainCallImportingUserdirFiles( { paths : paths,
file_type: file_type,
dbkey: dbkey,
source: full_source } );
source: full_source,
tag_using_filenames: tag_using_filenames } );
}
}
},
@@ -794,9 +797,10 @@ var FolderToolbarView = Backbone.View.extend({
},
/**
* Take the array of paths and createa request for each of them
* Take the array of paths and create a request for each of them
* calling them in chain. Update the progress bar in between each.
* @param {array} paths paths relative to user folder on Galaxy
* @param {array} paths paths relative to user folder on Galaxy
* @param {boolean} tag_using_filenames add tags to datasets using names of files
*/
chainCallImportingUserdirFiles: function( options ){
@@ -815,7 +819,8 @@ var FolderToolbarView = Backbone.View.extend({
'&source=' + options.source +
'&path=' + popped_item +
'&file_type=' + options.file_type +
'&dbkey=' + options.dbkey ) )
'&dbkey=' + options.dbkey +
'&tag_using_filenames=' + options.tag_using_filenames ) )
promise.done( function( response ){
that.updateProgress();
that.chainCallImportingUserdirFiles( options );
@@ -830,11 +835,12 @@ var FolderToolbarView = Backbone.View.extend({
/**
* Take the array of paths and createa request for each of them
* calling them in chain. Update the progress bar in between each.
* @param {array} paths paths relative to Galaxy root folder
* @param {boolean} preserve_dirs indicates whether to preserve folder structure
* @param {boolean} link_data copy files to Galaxy or link instead
* @param {str} source string representing what type of folder
* is the source of import
* @param {array} paths paths relative to Galaxy root folder
* @param {boolean} preserve_dirs indicates whether to preserve folder structure
* @param {boolean} link_data copy files to Galaxy or link instead
* @param {str} source string representing what type of folder
* is the source of import
* @param {boolean} tag_using_filenames add tags to datasets using names of files
*/
chainCallImportingFolders: function( options ){
// TODO need to check which paths to call
@@ -856,7 +862,8 @@ var FolderToolbarView = Backbone.View.extend({
'&preserve_dirs=' + options.preserve_dirs +
'&link_data=' + options.link_data +
'&file_type=' + options.file_type +
'&dbkey=' + options.dbkey ) )
'&dbkey=' + options.dbkey +
'&tag_using_filenames=' + options.tag_using_filenames ) )
promise.done(function(response){
that.updateProgress();
that.chainCallImportingFolders( options );
@@ -1341,7 +1348,13 @@ var FolderToolbarView = Backbone.View.extend({
'<div>',
'Type: <span id="library_extension_select" class="library-extension-select" />',
'Genome: <span id="library_genome_select" class="library-genome-select" />',
'</div>',
'</div>',
'<div>',
'<label class="checkbox-inline tag-files">',
'Tag datasets based on file names.',
'<input class="tag-files" type="checkbox" value="tag_using_filenames" checked="checked">',
'</label>',
'</div>',
'</div>'
].join(''));
},
client/galaxy/style/less/galaxy_bootstrap/overrides.less
+8
View File
@@ -94,3 +94,11 @@ div.ui-form-help {
}
}
}
// pre styling
pre.code {
white-space: pre-wrap;
background: #1d1f21;
color: white;
padding: 1em;
}
config/datatypes_conf.xml.sample
+3 -1
View File
@@ -242,6 +242,8 @@
<datatype extension="kronik" type="galaxy.datatypes.tabular:Tabular" subclass="true" />
<datatype extension="imzml" type="galaxy.datatypes.proteomics:ImzML" mimetype="application/xml" display_in_upload="true"/>
<!-- End Proteomics Datatypes -->
<datatype extension="deeptools_compute_matrix_archive" type="galaxy.datatypes.binary:CompressedArchive" subclass="true" display_in_upload="true"/>
<datatype extension="deeptools_coverage_matrix" type="galaxy.datatypes.binary:CompressedArchive" subclass="true" display_in_upload="true"/>
<datatype extension="netcdf" type="galaxy.datatypes.binary:NetCDF" mimetype="application/octet-stream" display_in_upload="true" description="Format used by netCDF software library for writing and reading chromatography-MS data files." />
<datatype extension="eps" type="galaxy.datatypes.images:Eps" mimetype="image/eps"/>
<datatype extension="rast" type="galaxy.datatypes.images:Rast" mimetype="image/rast"/>
@@ -323,7 +325,7 @@
<datatype extension="interval_index" type="galaxy.datatypes.binary:Binary" subclass="true" />
<datatype extension="tabix" type="galaxy.datatypes.binary:Binary" subclass="true" />
<datatype extension="bgzip" type="galaxy.datatypes.binary:Binary" subclass="true" />
<datatype extension="vcf_bgzip" type_extension="bgzip" subclass="true" >
<datatype extension="vcf_bgzip" type="galaxy.datatypes.tabular:VcfGz" type_extension="bgzip" subclass="true" display_in_upload="true">
<display file="igv/vcf.xml" />
<converter file="vcf_bgzip_to_tabix_converter.xml" target_datatype="tabix"/>
</datatype>
config/error_report.yml.sample
+42
View File
@@ -0,0 +1,42 @@
# Each element in this file corresponds to a destination for an error
# report.
# If you supply the parameter ``user_submission``, and set it to True,
# this error plugin will only be activated when the user uses the bug
# report submission interface. Otherwise, it will be triggered without
# any user interaction (e.g. historical behaviour of sentry plugin)
# If your plugin is ``user_submission=True``, then supplying
# ``verbose=True`` will cause the plugin to display a message to the end
# user. E.g. the email plugin simply states "Your error report has been
# sent", or the JSON plugin informs the user "Wrote error report to ..."
# The plugins below are listed with their default values of
# verbose/user_submission, but those are not necessary to provide.
# The default Email bug reporter. By default, the standard
# configuration is taken from your galaxy.ini
- type: email
verbose: true
user_submission: true
# Example JSON bug reporter implementation. This just writes the bug
# report as a JSON file to a specified directory and serves as an
# example for others to extend.
# - type: json
# verbose: true
# user_submission: true
# directory: /tmp/reports/
# Submit error reports to sentry. If a sentry_dsn is configured in your
# galaxy.ini, then Galaxy will submit the job error to Sentry.
- type: sentry
user_submission: false
# Allow users to submit error reports to biostars. This requires that the
# biostars integration is configured. This *only* makes sense when
# user_submission is true, as it only generates the link for the user to click
# on and submit the bug report, it does not actually submit the bug report on
# their behalf.
# - type: biostars
# user_submission: true
config/tool_conf.xml.sample
+1
View File
@@ -38,6 +38,7 @@
<tool file="${model_tools_path}/merge_collection.xml" />
<tool file="${model_tools_path}/relabel_from_file.xml" />
<tool file="${model_tools_path}/filter_from_file.xml" />
<tool file="${model_tools_path}/sort_collection_list.xml" />
</section>
<section id="liftOver" name="Lift-Over">
<tool file="extract/liftOver_wrapper.xml" />
config/user_preferences_extra_conf.yml.sample
+41
View File
@@ -0,0 +1,41 @@
preferences:
# the key you can refer to
apollo_url_01:
# description that is displayed to the user
description: The URL to your personal Apollo instance
inputs:
# the content can be accessed by apollo_url_01.apollo_url
- name: apollo_url
label: Apollo URL
# type of input field that will be displayed to the user
# can be string or password
type: text
# by defaul all inputs are required
required: True
- name: apollo_text
label: Apollo Text
# type of input field that will be displayed to the user
# can be string or password
type: text
# by defaul all inputs are required
required: True
openstack_account:
description: Your own Open Stack account
inputs:
- name: username
label: Username
type: text
required: False
- name: url
label: Open Stack URL
type: text
required: True
webhook_overlay_search:
description: Configuration option for your search results
inputs:
- name: max_search_results
label: Maximum number of search results
type: text
required: False
doc/source/admin/special_topics/bug_reports.rst
+72
View File
@@ -0,0 +1,72 @@
Bug Reports
===========
In 17.09, bug reports were refactored to a plugin-type infrastructure. This
gives the administrator more control over how and when bug reports are
generated. In the past, email reports would be generated when the user clicks
the button and only then. Sentry reports would be generated for every failing
tool, as soon as the tool failed. This disparate behaviour was unified under a
single configuration file, ``config/error_report.xml.sample``.
Let's look at that briefly:
.. code-block:: xml
<?xml version="1.0"?>
<bug_reports>
<email verbose="true" user_submission="true" />
<json user_submission="false" directory="/tmp/reports/" />
<sentry user_submission="false"/>
</bug_reports>
The actual configuration file contains more documentation which has been
stripped out here for the sake of brevity. As you can see, there are a couple of
different plugins that already exist. Additionally, there are some options that
are common to all plugins:
``user_submission``
When true, this action is triggered only when the user is on the job error
page and clicks "submit bug report".
When false, this action is triggered *any* time a job errors, without user
involvement
``verbose``
When ``user_submission=True``, and ``verbose=True``, this indicates that a
message is to be displayed to the end user. For example, the email plugin will
inform the user that a mail has been sent. Or the JSON plugin will inform the
user that a file has been written to a specific directory.
Email
-----
This is the classic bug-report mechanism that we are all familiar with. It
generates an email to the admin and the submitting user containing detailed
information about the job and links to the precise locations within Galaxy.
As a plugin, nothing has changed to this functionality, though future updates
may add features. It currently takes all of its configuration from your
``$GALAXY/config/galaxy.ini``, e.g. the variables ``error_email_to`` and
``email_from``.
JSON
----
This is a demo-plugin that writes the contents of the error report to a file in
your temp directory. This just serves as a full-featured reference
implementation that others can use to build their own bug reporting plugins.
When ``verbose="true" user_submission="true"``, the plugin will inform the user
that a report has been written to ``/tmp/<number>``.
Sentry
------
This refactors the existing on-failure submit-to-sentry behaviour into a bug
reporting plugin. Now, for example, you are able to disable the automatic
submission to sentry and only run that whenever the user reports it.
When ``verbose="true" user_submission="true"``, the plugin will inform the user
that ``Submitted bug report to Sentry. Your guru meditation number is
dc907d44ce294f78b267a56f68e5cd1a``, using the same phrasing that is common to
users from Galaxy internal server errors.
doc/source/admin/special_topics/index.rst
+1
View File
@@ -11,3 +11,4 @@ Special Topics
chat
webhooks
performance_tracking
bug_reports
lib/galaxy/app.py
+4
View File
@@ -27,6 +27,7 @@ from galaxy.tools.cache import (
ToolShedRepositoryCache
)
from galaxy.jobs import metrics as job_metrics
from galaxy.tools.error_reports import ErrorReports
from galaxy.web.proxy import ProxyManager
from galaxy.web.stack import application_stack_instance
from galaxy.queue_worker import GalaxyQueueWorker
@@ -102,6 +103,9 @@ class UniverseApplication( object, config.ConfiguresGalaxyMixin ):
# config so per-destination modifications can be made.
self.job_metrics = job_metrics.JobMetrics( self.config.job_metrics_config_file, app=self )
# Initialize error report plugins.
self.error_reports = ErrorReports( self.config.error_report_file, app=self )
# Initialize the job management configuration
self.job_config = jobs.JobConfiguration(self)
lib/galaxy/config.py
+2
View File
@@ -43,6 +43,7 @@ PATH_DEFAULTS = dict(
job_config_file=['config/job_conf.xml', 'job_conf.xml'],
tool_destinations_config_file=['config/tool_destinations.yml', 'config/tool_destinations.yml.sample'],
job_metrics_config_file=['config/job_metrics_conf.xml', 'job_metrics_conf.xml', 'config/job_metrics_conf.xml.sample'],
error_report_file=['config/error_report.yml', 'config/error_report.yml.sample'],
dependency_resolvers_config_file=['config/dependency_resolvers_conf.xml', 'dependency_resolvers_conf.xml'],
job_resource_params_file=['config/job_resource_params_conf.xml', 'job_resource_params_conf.xml'],
migrated_tools_config=['migrated_tools_conf.xml', 'config/migrated_tools_conf.xml'],
@@ -54,6 +55,7 @@ PATH_DEFAULTS = dict(
workflow_schedulers_config_file=['config/workflow_schedulers_conf.xml', 'config/workflow_schedulers_conf.xml.sample'],
modules_mapping_files=['config/environment_modules_mapping.yml', 'config/environment_modules_mapping.yml.sample'],
local_conda_mapping_file=['config/local_conda_mapping.yml', 'config/local_conda_mapping.yml.sample'],
user_preferences_extra_config_file=[ 'config/user_preferences_extra_conf.yml' ],
containers_config_file=['config/containers_conf.yml'],
)
lib/galaxy/datatypes/interval.py
+19 -16
View File
@@ -13,7 +13,10 @@ from six.moves.urllib.parse import quote_plus
from galaxy import util
from galaxy.datatypes import metadata
from galaxy.datatypes.metadata import MetadataElement
from galaxy.datatypes.sniff import get_headers
from galaxy.datatypes.sniff import (
get_headers,
iter_headers
)
from galaxy.datatypes.tabular import Tabular
from galaxy.datatypes.util.gff_util import parse_gff3_attributes, parse_gff_attributes
from galaxy.web import url_for
@@ -313,12 +316,12 @@ class Interval( Tabular ):
>>> Interval().sniff( fname )
True
"""
headers = get_headers( filename, '\t', comment_designator='#' )
try:
"""
If we got here, we already know the file is_column_based and is not bed,
so we'll just look for some valid data.
"""
headers = iter_headers( filename, '\t', comment_designator='#' )
for hdr in headers:
if hdr:
if len(hdr) < 3:
@@ -489,10 +492,10 @@ class Bed( Interval ):
>>> Bed().sniff( fname )
True
"""
headers = get_headers( filename, '\t', comment_designator='#' )
if not get_headers( filename, '\t', comment_designator='#', count=1 ):
return False
try:
if not headers:
return False
headers = iter_headers( filename, '\t', comment_designator='#' )
for hdr in headers:
if hdr[0] == '':
continue
@@ -832,10 +835,10 @@ class Gff( Tabular, _RemoteCallMixin ):
>>> Gff().sniff( fname )
True
"""
headers = get_headers( filename, '\t' )
if len(get_headers( filename, '\t', count=2 )) < 2:
return False
try:
if len(headers) < 2:
return False
headers = iter_headers( filename, '\t' )
for hdr in headers:
if hdr and hdr[0].startswith( '##gff-version' ) and hdr[0].find( '2' ) < 0:
return False
@@ -964,10 +967,10 @@ class Gff3( Gff ):
>>> Gff3().sniff( fname )
True
"""
headers = get_headers( filename, '\t' )
if len(get_headers( filename, '\t', count=2 )) < 2:
return False
try:
if len(headers) < 2:
return False
headers = iter_headers( filename, '\t' )
for hdr in headers:
if hdr and hdr[0].startswith( '##gff-version' ) and hdr[0].find( '3' ) >= 0:
return True
@@ -1039,10 +1042,10 @@ class Gtf( Gff ):
>>> Gtf().sniff( fname )
True
"""
headers = get_headers( filename, '\t' )
if len(get_headers( filename, '\t', count=2 )) < 2:
return False
try:
if len(headers) < 2:
return False
headers = iter_headers( filename, '\t' )
for hdr in headers:
if hdr and hdr[0].startswith( '##gff-version' ) and hdr[0].find( '2' ) < 0:
return False
@@ -1235,8 +1238,8 @@ class Wiggle( Tabular, _RemoteCallMixin ):
>>> Wiggle().sniff( fname )
True
"""
headers = get_headers( filename, None )
try:
headers = iter_headers( filename, None )
for hdr in headers:
if len(hdr) > 1 and hdr[0] == 'track' and hdr[1].startswith('type=wiggle'):
return True
@@ -1371,7 +1374,7 @@ class CustomTrack ( Tabular ):
>>> CustomTrack().sniff( fname )
True
"""
headers = get_headers( filename, None )
headers = iter_headers( filename, None )
first_line = True
for hdr in headers:
if first_line:
lib/galaxy/datatypes/molecules.py
+7 -4
View File
@@ -10,7 +10,10 @@ from galaxy.datatypes import (
from galaxy.datatypes.binary import Binary
from galaxy.datatypes.data import get_file_peek
from galaxy.datatypes.metadata import MetadataElement
from galaxy.datatypes.sniff import get_headers
from galaxy.datatypes.sniff import (
get_headers,
iter_headers
)
from galaxy.datatypes.tabular import Tabular
from galaxy.datatypes.xml import GenericXml
@@ -461,7 +464,7 @@ class PDB(GenericMolFile):
>>> PDB().sniff(fname)
False
"""
headers = get_headers(filename, sep=' ', count=300)
headers = iter_headers(filename, sep=' ', count=300)
h = t = c = s = k = e = False
for line in headers:
section_name = line[0].strip()
@@ -514,7 +517,7 @@ class PDBQT(GenericMolFile):
>>> PDBQT().sniff(fname)
False
"""
headers = get_headers(filename, sep=' ', count=300)
headers = iter_headers(filename, sep=' ', count=300)
h = t = c = s = k = False
for line in headers:
section_name = line[0].strip()
@@ -607,7 +610,7 @@ class InChI(Tabular):
>>> InChI().sniff(fname)
False
"""
inchi_lines = get_headers(filename, sep=' ', count=10)
inchi_lines = iter_headers(filename, sep=' ', count=10)
for inchi in inchi_lines:
if not inchi[0].startswith('InChI='):
return False
lib/galaxy/datatypes/mothur.py
+24 -20
View File
@@ -7,7 +7,10 @@ import sys
from galaxy.datatypes.data import Text
from galaxy.datatypes.metadata import MetadataElement
from galaxy.datatypes.sniff import get_headers
from galaxy.datatypes.sniff import (
get_headers,
iter_headers
)
from galaxy.datatypes.tabular import Tabular
log = logging.getLogger(__name__)
@@ -32,10 +35,11 @@ class Otu(Text):
data_lines = 0
comment_lines = 0
headers = get_headers(dataset.file_name, sep='\t', count=-1)
headers = iter_headers(dataset.file_name, sep='\t', count=-1)
first_line = get_headers(dataset.file_name, sep='\t', count=1)
# set otulabels
if len(headers[0]) > 2:
otulabel_names = headers[0][2:]
if len(first_line) > 2:
otulabel_names = first_line[2:]
# set label names and number of lines
for line in headers:
if len(line) >= 2 and not line[0].startswith('@'):
@@ -64,7 +68,7 @@ class Otu(Text):
>>> Otu().sniff( fname )
False
"""
headers = get_headers(filename, sep='\t')
headers = iter_headers(filename, sep='\t')
count = 0
for line in headers:
if not line[0].startswith('@'):
@@ -109,7 +113,7 @@ class Sabund(Otu):
>>> Sabund().sniff( fname )
False
"""
headers = get_headers(filename, sep='\t')
headers = iter_headers(filename, sep='\t')
count = 0
for line in headers:
if not line[0].startswith('@'):
@@ -151,7 +155,7 @@ class GroupAbund(Otu):
comment_lines = 0
ncols = 0
headers = get_headers(dataset.file_name, sep='\t', count=-1)
headers = iter_headers(dataset.file_name, sep='\t', count=-1)
for line in headers:
if line[0] == 'label' and line[1] == 'Group':
skip = 1
@@ -187,7 +191,7 @@ class GroupAbund(Otu):
>>> GroupAbund().sniff( fname )
False
"""
headers = get_headers(filename, sep='\t')
headers = iter_headers(filename, sep='\t')
count = 0
for line in headers:
if not line[0].startswith('@'):
@@ -234,7 +238,7 @@ class SecondaryStructureMap(Tabular):
>>> SecondaryStructureMap().sniff( fname )
False
"""
headers = get_headers(filename, sep='\t')
headers = iter_headers(filename, sep='\t')
line_num = 0
rowidxmap = {}
for line in headers:
@@ -302,7 +306,7 @@ class DistanceMatrix(Text):
def set_meta(self, dataset, overwrite=True, skip=0, **kwd):
super(DistanceMatrix, self).set_meta(dataset, overwrite=overwrite, skip=skip, **kwd)
headers = get_headers(dataset.file_name, sep='\t')
headers = iter_headers(dataset.file_name, sep='\t')
for line in headers:
if not line[0].startswith('@'):
try:
@@ -344,7 +348,7 @@ class LowerTriangleDistanceMatrix(DistanceMatrix):
False
"""
numlines = 300
headers = get_headers(filename, sep='\t', count=numlines)
headers = iter_headers(filename, sep='\t', count=numlines)
line_num = 0
for line in headers:
if not line[0].startswith('@'):
@@ -405,7 +409,7 @@ class SquareDistanceMatrix(DistanceMatrix):
False
"""
numlines = 300
headers = get_headers(filename, sep='\t', count=numlines)
headers = iter_headers(filename, sep='\t', count=numlines)
line_num = 0
for line in headers:
if not line[0].startswith('@'):
@@ -461,7 +465,7 @@ class PairwiseDistanceMatrix(DistanceMatrix, Tabular):
>>> PairwiseDistanceMatrix().sniff( fname )
False
"""
headers = get_headers(filename, sep='\t')
headers = iter_headers(filename, sep='\t')
count = 0
for line in headers:
if not line[0].startswith('@'):
@@ -525,7 +529,7 @@ class Group(Tabular):
super(Group, self).set_meta(dataset, overwrite, skip, max_data_lines)
group_names = set()
headers = get_headers(dataset.file_name, sep='\t', count=-1)
headers = iter_headers(dataset.file_name, sep='\t', count=-1)
for line in headers:
if len(line) > 1:
group_names.add(line[1])
@@ -558,7 +562,7 @@ class Oligos(Text):
>>> Oligos().sniff( fname )
False
"""
headers = get_headers(filename, sep='\t')
headers = iter_headers(filename, sep='\t')
count = 0
for line in headers:
if not line[0].startswith('@') and not line[0].startswith('#'):
@@ -602,7 +606,7 @@ class Frequency(Tabular):
>>> Frequency().sniff( fname )
False
"""
headers = get_headers(filename, sep='\t')
headers = iter_headers(filename, sep='\t')
count = 0
for line in headers:
if not line[0].startswith('@'):
@@ -653,7 +657,7 @@ class Quantile(Tabular):
>>> Quantile().sniff( fname )
False
"""
headers = get_headers(filename, sep='\t')
headers = iter_headers(filename, sep='\t')
count = 0
for line in headers:
if not line[0].startswith('@') and not line[0].startswith('#'):
@@ -691,7 +695,7 @@ class LaneMask(Text):
>>> LaneMask().sniff( fname )
False
"""
headers = get_headers(filename, sep='\t')
headers = get_headers(filename, sep='\t', count=2)
if len(headers) != 1 or len(headers[0]) != 1:
return False
@@ -774,7 +778,7 @@ class RefTaxonomy(Tabular):
>>> RefTaxonomy().sniff( fname )
False
"""
headers = get_headers(filename, sep='\t', count=300)
headers = iter_headers(filename, sep='\t', count=300)
count = 0
pat_prog = re.compile('^([^ \t\n\r\x0c\x0b;]+([(]\\d+[)])?(;[^ \t\n\r\x0c\x0b;]+([(]\\d+[)])?)*(;)?)$')
found_semicolons = False
@@ -849,7 +853,7 @@ class Axes(Tabular):
>>> Axes().sniff( fname )
False
"""
headers = get_headers(filename, sep='\t')
headers = iter_headers(filename, sep='\t')
count = 0
col_cnt = None
all_integers = True
lib/galaxy/datatypes/sequence.py
+9 -4
View File
@@ -9,6 +9,7 @@ import re
import string
import sys
from cgi import escape
from itertools import islice
import bx.align.maf
@@ -16,7 +17,10 @@ from galaxy import util
from galaxy.datatypes import metadata
from galaxy.datatypes.binary import Binary
from galaxy.datatypes.metadata import MetadataElement
from galaxy.datatypes.sniff import get_headers
from galaxy.datatypes.sniff import (
get_headers,
iter_headers
)
from galaxy.util import (
compression_utils,
nice_size
@@ -611,7 +615,7 @@ class BaseFastq ( Sequence ):
compressed = is_gzip(filename) or is_bz2(filename)
if compressed and not isinstance(self, Binary):
return False
headers = get_headers( filename, None, count=1000 )
headers = iter_headers( filename, None, count=1000 )
# If this is a FastqSanger-derived class, then check to see if the base qualities match
if isinstance(self, FastqSanger) or isinstance(self, FastqSangerGz) or isinstance(self, FastqSangerBz2):
@@ -621,7 +625,8 @@ class BaseFastq ( Sequence ):
bases_regexp = re.compile( "^[NGTAC]*" )
# check that first block looks like a fastq block
try:
if len( headers ) >= 4 and headers[0][0] and headers[0][0][0] == "@" and headers[2][0] and headers[2][0][0] == "+" and headers[1][0]:
headers = get_headers( filename, None, count=4 )
if len( headers ) == 4 and headers[0][0] and headers[0][0][0] == "@" and headers[2][0] and headers[2][0][0] == "+" and headers[1][0]:
# Check the sequence line, make sure it contains only G/C/A/T/N
if not bases_regexp.match( headers[1][0] ):
return False
@@ -695,7 +700,7 @@ class BaseFastq ( Sequence ):
@staticmethod
def sangerQualities( lines ):
"""Presuming lines are lines from a fastq file, return True if the qualities are compatible with sanger encoding"""
for line in lines[3::4]:
for line in islice(lines, 3, None, 4):
if not all(_ >= '!' and _ <= 'M' for _ in line[0]):
return False
return True
lib/galaxy/datatypes/sniff.py
+17 -15
View File
@@ -200,19 +200,7 @@ def convert_newlines_sep2tabs( fname, in_place=True, patt="\\s+", tmp_dir=None,
return ( i + 1, temp_name )
def get_headers( fname, sep, count=60, is_multi_byte=False, comment_designator=None ):
"""
Returns a list with the first 'count' lines split by 'sep', ignoring lines
starting with 'comment_designator'
>>> fname = get_test_fname('complete.bed')
>>> get_headers(fname,'\\t')
[['chr7', '127475281', '127491632', 'NM_000230', '0', '+', '127486022', '127488767', '0', '3', '29,172,3225,', '0,10713,13126,'], ['chr7', '127486011', '127488900', 'D49487', '0', '+', '127486022', '127488767', '0', '2', '155,490,', '0,2399']]
>>> fname = get_test_fname('test.gff')
>>> get_headers(fname, '\\t', count=5, comment_designator='#')
[[''], ['chr7', 'bed2gff', 'AR', '26731313', '26731437', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731491', '26731536', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731541', '26731649', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731659', '26731841', '.', '+', '.', 'score']]
"""
headers = []
def iter_headers( fname, sep, count=60, is_multi_byte=False, comment_designator=None ):
with compression_utils.get_fileobj(fname) as in_file:
idx = 0
for line in in_file:
@@ -225,11 +213,25 @@ def get_headers( fname, sep, count=60, is_multi_byte=False, comment_designator=N
comment_designator = comment_designator.encode( 'utf-8' )
if comment_designator is not None and comment_designator != '' and line.startswith( comment_designator ):
continue
headers.append( line.split(sep) )
yield line.split(sep)
idx += 1
if idx == count:
break
return headers
def get_headers( fname, sep, count=60, is_multi_byte=False, comment_designator=None ):
"""
Returns a list with the first 'count' lines split by 'sep', ignoring lines
starting with 'comment_designator'
>>> fname = get_test_fname('complete.bed')
>>> get_headers(fname,'\\t')
[['chr7', '127475281', '127491632', 'NM_000230', '0', '+', '127486022', '127488767', '0', '3', '29,172,3225,', '0,10713,13126,'], ['chr7', '127486011', '127488900', 'D49487', '0', '+', '127486022', '127488767', '0', '2', '155,490,', '0,2399']]
>>> fname = get_test_fname('test.gff')
>>> get_headers(fname, '\\t', count=5, comment_designator='#')
[[''], ['chr7', 'bed2gff', 'AR', '26731313', '26731437', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731491', '26731536', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731541', '26731649', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731659', '26731841', '.', '+', '.', 'score']]
"""
return list(iter_headers(fname=fname, sep=sep, count=count, is_multi_byte=is_multi_byte, comment_designator=comment_designator))
def is_column_based( fname, sep='\t', skip=0, is_multi_byte=False ):
lib/galaxy/datatypes/tabular.py
+92 -49
View File
@@ -8,6 +8,7 @@ import csv
import logging
import os
import re
import shutil
import subprocess
import sys
import tempfile
@@ -15,9 +16,12 @@ from cgi import escape
from json import dumps
from galaxy import util
from galaxy.datatypes import data, metadata
from galaxy.datatypes import binary, data, metadata
from galaxy.datatypes.metadata import MetadataElement
from galaxy.datatypes.sniff import get_headers
from galaxy.datatypes.sniff import (
get_headers,
iter_headers
)
from galaxy.util import compression_utils
from . import dataproviders
@@ -62,7 +66,7 @@ class TabularData( data.Text ):
return False
def get_chunk(self, trans, dataset, offset=0, ck_size=None):
with open(dataset.file_name) as f:
with compression_utils.get_fileobj(dataset.file_name) as f:
f.seek(offset)
ck_data = f.read(ck_size or trans.app.config.display_chunk_size)
if ck_data and ck_data[-1] != '\n':
@@ -325,49 +329,48 @@ class Tabular( TabularData ):
first_line_column_types = [default_column_type] # default value is one column of type str
if dataset.has_data():
# NOTE: if skip > num_check_lines, we won't detect any metadata, and will use default
dataset_fh = open( dataset.file_name )
i = 0
while True:
line = dataset_fh.readline()
if not line:
break
line = line.rstrip( '\r\n' )
if i < skip or not line or line.startswith( '#' ):
# We'll call blank lines comments
comment_lines += 1
else:
data_lines += 1
if max_guess_type_data_lines is None or data_lines <= max_guess_type_data_lines:
fields = line.split( '\t' )
for field_count, field in enumerate( fields ):
if field_count >= len( column_types ): # found a previously unknown column, we append None
column_types.append( None )
column_type = guess_column_type( field )
if type_overrules_type( column_type, column_types[field_count] ):
column_types[field_count] = column_type
if i == 0 and requested_skip is None:
# This is our first line, people seem to like to upload files that have a header line, but do not
# start with '#' (i.e. all column types would then most likely be detected as str). We will assume
# that the first line is always a header (this was previous behavior - it was always skipped). When
# the requested skip is None, we only use the data from the first line if we have no other data for
# a column. This is far from perfect, as
# 1,2,3 1.1 2.2 qwerty
# 0 0 1,2,3
# will be detected as
# "column_types": ["int", "int", "float", "list"]
# instead of
# "column_types": ["list", "float", "float", "str"] *** would seem to be the 'Truth' by manual
# observation that the first line should be included as data. The old method would have detected as
# "column_types": ["int", "int", "str", "list"]
first_line_column_types = column_types
column_types = [ None for col in first_line_column_types ]
if max_data_lines is not None and data_lines >= max_data_lines:
if dataset_fh.tell() != dataset.get_size():
data_lines = None # Clear optional data_lines metadata value
comment_lines = None # Clear optional comment_lines metadata value; additional comment lines could appear below this point
break
i += 1
dataset_fh.close()
with compression_utils.get_fileobj(dataset.file_name) as dataset_fh:
i = 0
while True:
line = dataset_fh.readline()
if not line:
break
line = line.rstrip( '\r\n' )
if i < skip or not line or line.startswith( '#' ):
# We'll call blank lines comments
comment_lines += 1
else:
data_lines += 1
if max_guess_type_data_lines is None or data_lines <= max_guess_type_data_lines:
fields = line.split( '\t' )
for field_count, field in enumerate( fields ):
if field_count >= len( column_types ): # found a previously unknown column, we append None
column_types.append( None )
column_type = guess_column_type( field )
if type_overrules_type( column_type, column_types[field_count] ):
column_types[field_count] = column_type
if i == 0 and requested_skip is None:
# This is our first line, people seem to like to upload files that have a header line, but do not
# start with '#' (i.e. all column types would then most likely be detected as str). We will assume
# that the first line is always a header (this was previous behavior - it was always skipped). When
# the requested skip is None, we only use the data from the first line if we have no other data for
# a column. This is far from perfect, as
# 1,2,3 1.1 2.2 qwerty
# 0 0 1,2,3
# will be detected as
# "column_types": ["int", "int", "float", "list"]
# instead of
# "column_types": ["list", "float", "float", "str"] *** would seem to be the 'Truth' by manual
# observation that the first line should be included as data. The old method would have detected as
# "column_types": ["int", "int", "str", "list"]
first_line_column_types = column_types
column_types = [ None for col in first_line_column_types ]
if max_data_lines is not None and data_lines >= max_data_lines:
if dataset_fh.tell() != dataset.get_size():
data_lines = None # Clear optional data_lines metadata value
comment_lines = None # Clear optional comment_lines metadata value; additional comment lines could appear below this point
break
i += 1
# we error on the larger number of columns
# first we pad our column_types by using data from first line
@@ -638,7 +641,7 @@ class Pileup( Tabular ):
>>> Pileup().sniff( fname )
True
"""
headers = get_headers( filename, '\t' )
headers = iter_headers( filename, '\t' )
try:
for hdr in headers:
if hdr and not hdr[0].startswith( '#' ):
@@ -670,7 +673,7 @@ class Pileup( Tabular ):
@dataproviders.decorators.has_dataproviders
class Vcf( Tabular ):
class BaseVcf( Tabular ):
""" Variant Call Format for describing SNPs and other simple genome variations. """
edam_format = "format_3016"
track_type = "VariantTrack"
@@ -693,7 +696,7 @@ class Vcf( Tabular ):
return self.make_html_table( dataset, column_names=self.column_names )
def set_meta( self, dataset, **kwd ):
super( Vcf, self ).set_meta( dataset, **kwd )
super( BaseVcf, self ).set_meta( dataset, **kwd )
source = open( dataset.file_name )
# Skip comments.
@@ -732,6 +735,46 @@ class Vcf( Tabular ):
return self.genomic_region_dataprovider( dataset, **settings )
class Vcf ( BaseVcf ):
extension = 'vcf'
class VcfGz( BaseVcf, binary.Binary):
extension = 'vcf.gz'
compressed = True
MetadataElement( name="tabix_index", desc="Vcf Index File", param=metadata.FileParameter, file_ext="tbi", readonly=True, no_value=None, visible=False, optional=True )
def set_meta( self, dataset, **kwd ):
super(BaseVcf, self).set_meta(dataset, **kwd)
""" Creates the index for the VCF file. """
# These metadata values are not accessible by users, always overwrite
index_file = dataset.metadata.bcf_index
if not index_file:
index_file = dataset.metadata.spec['tabix_index'].param.new_file( dataset=dataset )
# Create the bcf index
# $ bcftools index
# Usage: bcftools index <in.bcf>
dataset_symlink = os.path.join( os.path.dirname( index_file.file_name ),
'__dataset_%d_%s' % ( dataset.id, os.path.basename( index_file.file_name ) ) ) + ".vcf.gz"
os.symlink( dataset.file_name, dataset_symlink )
stderr_name = tempfile.NamedTemporaryFile( prefix="bcf_index_stderr" ).name
command = [ 'bcftools', 'index', '-t', dataset_symlink ]
try:
subprocess.check_call( args=command, stderr=open( stderr_name, 'wb' ) )
shutil.move( dataset_symlink + '.tbi', index_file.file_name ) # this will fail if bcftools < 1.0 is used, because it creates a .bci index file instead of .csi
except Exception as e:
stderr = open( stderr_name ).read().strip()
raise Exception('Error setting BCF metadata: %s' % (stderr or str(e)))
finally:
# Remove temp file and symlink
os.remove( stderr_name )
os.remove( dataset_symlink )
dataset.metadata.tabix_index = index_file
class Eland( Tabular ):
"""Support for the export.txt.gz file used by Illumina's ELANDv2e aligner"""
file_ext = '_export.txt.gz'
lib/galaxy/datatypes/text.py
+2 -2
View File
@@ -12,7 +12,7 @@ import tempfile
from galaxy.datatypes.data import get_file_peek, Text
from galaxy.datatypes.metadata import MetadataElement, MetadataParameter
from galaxy.datatypes.sniff import get_headers
from galaxy.datatypes.sniff import iter_headers
from galaxy.util import nice_size, string_as_bool
log = logging.getLogger(__name__)
@@ -47,7 +47,7 @@ class Html( Text ):
>>> Html().sniff( fname )
True
"""
headers = get_headers( filename, None )
headers = iter_headers( filename, None )
try:
for i, hdr in enumerate(headers):
if hdr and hdr[0].lower().find( '<html>' ) >= 0:
lib/galaxy/jobs/__init__.py
+5 -21
View File
@@ -1028,7 +1028,7 @@ class JobWrapper( object, HasResourceParameters ):
# the partial files to the object store regardless of whether job.state == DELETED
self.__update_output(job, dataset, clean_only=True)
self._report_error_to_sentry()
self._report_error()
# Perform email action even on failure.
for pja in [pjaa.post_job_action for pjaa in job.post_job_actions if pjaa.post_job_action.action_type == "EmailAction"]:
ActionBox.execute(self.app, self.sa_session, pja, job)
@@ -1437,7 +1437,7 @@ class JobWrapper( object, HasResourceParameters ):
self.sa_session.flush()
log.debug( 'job %d ended (finish() executed in %s)' % (self.job_id, finish_timer) )
if job.state == job.states.ERROR:
self._report_error_to_sentry()
self._report_error()
cleanup_job = self.cleanup_job
delete_files = cleanup_job == 'always' or ( job.state == job.states.OK and cleanup_job == 'onsuccess' )
self.cleanup( delete_files=delete_files )
@@ -1809,27 +1809,11 @@ class JobWrapper( object, HasResourceParameters ):
return self.tool.requires_setting_metadata
return False
def _report_error_to_sentry( self ):
def _report_error( self ):
job = self.get_job()
tool = self.app.toolbox.get_tool(job.tool_id, tool_version=job.tool_version) or None
if self.app.sentry_client and job.state == job.states.ERROR:
self.app.sentry_client.capture(
'raven.events.Message',
message="Galaxy Job Error: %s v.%s" % (job.tool_id, job.tool_version),
extra={
'info' : job.info,
'id' : job.id,
'command_line' : job.command_line,
'stderr' : job.stderr,
'traceback': job.traceback,
'exit_code': job.exit_code,
'stdout': job.stdout,
'handler': job.handler,
'user': self.user,
'tool_version': job.tool_version,
'tool_xml': tool.config_file if tool else None
}
)
for dataset in job.output_datasets:
self.app.error_reports.default_error_plugin.submit_report(dataset, job, tool, user_submission=False)
class TaskWrapper(JobWrapper):
lib/galaxy/jobs/command_factory.py
+8 -1
View File
@@ -66,8 +66,13 @@ def build_command(
if not commands_builder.commands:
return None
# Version, dependency resolution, and task splitting are prepended to the
# command - so they need to appear in the following order to ensure that
# the underlying application used by version command is available in the
# environment after dependency resolution, but the task splitting command
# is still executed in Galaxy's Python environment.
__handle_version_command(commands_builder, job_wrapper)
__handle_task_splitting(commands_builder, job_wrapper)
# One could imagine also allowing dependencies inside of the container but
# that is too sophisticated for a first crack at this - build your
@@ -75,6 +80,8 @@ def build_command(
if not container or container.resolve_dependencies:
__handle_dependency_resolution(commands_builder, job_wrapper, remote_command_params)
__handle_task_splitting(commands_builder, job_wrapper)
if (container and modify_command_for_container) or job_wrapper.commands_in_new_shell:
if container and modify_command_for_container:
# Many Docker containers do not have /bin/bash.
lib/galaxy/managers/tags.py
+4
View File
@@ -324,6 +324,10 @@ class GalaxyTagManager( TagManager ):
ItemTagAssocInfo( model.HistoryDatasetCollectionAssociation,
model.HistoryDatasetCollectionTagAssociation,
model.HistoryDatasetCollectionTagAssociation.table.c.history_dataset_collection_id )
self.item_tag_assoc_info["LibraryDatasetDatasetAssociation"] = \
ItemTagAssocInfo( model.LibraryDatasetDatasetAssociation,
model.LibraryDatasetDatasetAssociationTagAssociation,
model.LibraryDatasetDatasetAssociationTagAssociation.table.c.library_dataset_dataset_association_id )
self.item_tag_assoc_info["Page"] = ItemTagAssocInfo( model.Page,
model.PageTagAssociation,
model.PageTagAssociation.table.c.page_id )
lib/galaxy/managers/workflows.py
+1 -1
View File
@@ -373,7 +373,7 @@ class WorkflowContentsManager(UsesAnnotations):
step_model = None
if step.type == 'tool':
incoming = {}
tool = trans.app.toolbox.get_tool( step.tool_id )
tool = trans.app.toolbox.get_tool( step.tool_id, tool_version=step.tool_version )
params_to_incoming( incoming, tool.inputs, step.state.inputs, trans.app )
step_model = tool.to_json( trans, incoming, workflow_building_mode=workflow_building_modes.USE_HISTORY )
step_model[ 'post_job_actions' ] = [{
lib/galaxy/model/__init__.py
+18
View File
@@ -28,6 +28,8 @@ import galaxy.model.metadata
import galaxy.model.orm.now
import galaxy.security.passwords
import galaxy.util
from galaxy.managers import tags
from galaxy.model.item_attrs import UsesAnnotations
from galaxy.model.util import pgcalc
from galaxy.security import get_permitted_actions
@@ -2470,6 +2472,7 @@ class HistoryDatasetAssociation( DatasetInstance, HasTags, Dictifiable, UsesAnno
trans.sa_session.flush()
# Must set metadata after ldda flushed, as MetadataFiles require ldda.id
ldda.metadata = self.metadata
# TODO: copy #tags from history
if ldda_message:
ldda.message = ldda_message
if not replace_dataset:
@@ -2934,6 +2937,11 @@ class LibraryDatasetDatasetAssociation( DatasetInstance, HasName ):
parent_id=parent_id,
copied_from_library_dataset_dataset_association=self,
history=target_history )
tag_manager = tags.GalaxyTagManager( sa_session )
src_ldda_tags = tag_manager.get_tags_str(self.tags)
tag_manager.apply_item_tags( user=self.user, item=hda, tags_str=src_ldda_tags )
sa_session.add( hda )
sa_session.flush()
hda.metadata = self.metadata # need to set after flushed, as MetadataFiles require dataset.id
@@ -2961,6 +2969,11 @@ class LibraryDatasetDatasetAssociation( DatasetInstance, HasName ):
parent_id=parent_id,
copied_from_library_dataset_dataset_association=self,
folder=target_folder )
tag_manager = tags.GalaxyTagManager( sa_session )
src_ldda_tags = tag_manager.get_tags_str(self.tags)
tag_manager.apply_item_tags( user=self.user, item=ldda, tags_str=src_ldda_tags )
sa_session.add( ldda )
sa_session.flush()
# Need to set after flushed, as MetadataFiles require dataset.id
@@ -3009,6 +3022,7 @@ class LibraryDatasetDatasetAssociation( DatasetInstance, HasName ):
except OSError:
file_size = 0
# TODO: render tags here
rval = dict( id=ldda.id,
hda_ldda='ldda',
model_class=self.__class__.__name__,
@@ -5143,6 +5157,10 @@ class HistoryDatasetAssociationTagAssociation ( ItemTagAssociation ):
pass
class LibraryDatasetDatasetAssociationTagAssociation ( ItemTagAssociation ):
pass
class PageTagAssociation ( ItemTagAssociation ):
pass
lib/galaxy/model/mapping.py
+14
View File
@@ -1213,6 +1213,16 @@ model.HistoryDatasetAssociationTagAssociation.table = Table(
Column( "value", TrimmedString( 255 ), index=True ),
Column( "user_value", TrimmedString( 255 ), index=True ) )
model.LibraryDatasetDatasetAssociationTagAssociation.table = Table(
"library_dataset_dataset_association_tag_association", metadata,
Column( "id", Integer, primary_key=True ),
Column( "library_dataset_dataset_association_id", Integer, ForeignKey( "library_dataset_dataset_association.id" ), index=True ),
Column( "tag_id", Integer, ForeignKey( "tag.id" ), index=True ),
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
Column( "user_tname", TrimmedString(255), index=True ),
Column( "value", TrimmedString(255), index=True ),
Column( "user_value", TrimmedString(255), index=True ) )
model.StoredWorkflowTagAssociation.table = Table(
"stored_workflow_tag_association", metadata,
Column( "id", Integer, primary_key=True ),
@@ -1990,6 +2000,9 @@ mapper( model.LibraryDatasetDatasetAssociation, model.LibraryDatasetDatasetAssoc
( model.LibraryDatasetDatasetAssociation.table.c.visible == true() )
),
remote_side=[model.LibraryDatasetDatasetAssociation.table.c.id] ),
tags=relation(model.LibraryDatasetDatasetAssociationTagAssociation,
order_by=model.LibraryDatasetDatasetAssociationTagAssociation.table.c.id,
backref='history_tag_associations'),
extended_metadata=relation( model.ExtendedMetadata,
primaryjoin=( ( model.LibraryDatasetDatasetAssociation.table.c.extended_metadata_id == model.ExtendedMetadata.table.c.id ) )
),
@@ -2448,6 +2461,7 @@ def tag_mapping( tag_association_class, backref_name ):
tag_mapping( model.HistoryTagAssociation, "tagged_histories" )
tag_mapping( model.DatasetTagAssociation, "tagged_datasets" )
tag_mapping( model.HistoryDatasetAssociationTagAssociation, "tagged_history_dataset_associations" )
tag_mapping( model.LibraryDatasetDatasetAssociationTagAssociation, "tagged_library_dataset_dataset_associations" )
tag_mapping( model.PageTagAssociation, "tagged_pages" )
tag_mapping( model.StoredWorkflowTagAssociation, "tagged_workflows" )
tag_mapping( model.WorkflowStepTagAssociation, "tagged_workflow_steps" )
lib/galaxy/model/migrate/versions/0135_add_library_tags.py
+47
View File
@@ -0,0 +1,47 @@
"""
This migration script adds support for storing tags in the context of a dataset in a library
"""
from __future__ import print_function
import logging
from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import TrimmedString
log = logging.getLogger( __name__ )
metadata = MetaData()
LibraryDatasetDatasetAssociationTagAssociation_table = Table(
"library_dataset_dataset_association_tag_association", metadata,
Column("id", Integer, primary_key=True),
Column( "library_dataset_dataset_association_id", Integer, ForeignKey( "library_dataset_dataset_association.id" ), index=True ),
Column( "tag_id", Integer, ForeignKey( "tag.id" ), index=True ),
Column( "user_tname", TrimmedString(255), index=True),
Column( "value", TrimmedString(255), index=True),
Column( "user_value", TrimmedString(255), index=True),
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True )
)
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print(__doc__)
metadata.reflect()
try:
LibraryDatasetDatasetAssociationTagAssociation_table.create()
except Exception:
log.exception("Creating library_dataset_association_tag_association table failed.")
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
try:
LibraryDatasetDatasetAssociationTagAssociation_table.drop()
except Exception:
log.exception("Dropping library_dataset_association_tag_association table failed.")
lib/galaxy/tools/__init__.py
+24
View File
@@ -2510,6 +2510,30 @@ class FlattenTool( DatabaseOperationTool ):
)
class SortTool( DatabaseOperationTool ):
tool_type = 'sort_collection'
def produce_outputs( self, trans, out_data, output_collections, incoming, history ):
hdca = incoming[ "input" ]
sorttype = incoming["sort_type"]
new_elements = odict()
elements = hdca.collection.elements
if sorttype == 'alpha':
presort_elements = [(dce.element_identifier, dce) for dce in elements]
elif sorttype == 'numeric':
presort_elements = [(int(re.sub('[^0-9]', '', dce.element_identifier)), dce) for dce in elements]
sorted_elements = [x[1] for x in sorted(presort_elements, key=lambda x: x[0])]
for dce in sorted_elements:
dce_object = dce.element_object
copied_dataset = dce_object.copy()
history.add_dataset(copied_dataset, set_hid=False)
new_elements[dce.element_identifier] = copied_dataset
output_collections.create_collection(
next(iter(self.outputs.values())), "output", elements=new_elements
)
class RelabelFromFileTool(DatabaseOperationTool):
tool_type = 'relabel_from_file'
lib/galaxy/tools/actions/upload_common.py
+6
View File
@@ -11,6 +11,7 @@ from sqlalchemy.orm import eagerload_all
from galaxy import datatypes, util
from galaxy.exceptions import ObjectInvalid
from galaxy.managers import tags
from galaxy.util.odict import odict
log = logging.getLogger( __name__ )
@@ -178,6 +179,11 @@ def __new_library_upload( trans, cntrller, uploaded_dataset, library_bunch, stat
user=trans.user,
create_dataset=True,
sa_session=trans.sa_session )
if uploaded_dataset.get( 'tag_using_filenames', False ):
tag_from_filename = os.path.splitext( os.path.basename( uploaded_dataset.name ))[0]
tag_manager = tags.GalaxyTagManager( trans.sa_session )
tag_manager.apply_item_tag( item=ldda, user=trans.user, name='name', value=tag_from_filename )
trans.sa_session.add( ldda )
if state:
ldda.state = state
lib/galaxy/tools/error_reports/__init__.py
+74
View File
@@ -0,0 +1,74 @@
"""This module defines the error reporting framework for Galaxy jobs.
"""
import collections
import logging
import os
from galaxy.util import plugin_config
log = logging.getLogger(__name__)
class ErrorReports(object):
"""Load and store a collection of :class:`ErrorPlugin` objects."""
def __init__(self, conf_file=None, **kwargs):
"""Load :class:`ErrorPlugin` objects from specified configuration file."""
self.plugin_classes = self.__plugins_dict()
self.default_error_plugin = ErrorPlugin.from_file(self.plugin_classes, conf_file, **kwargs)
self.error_plugin = collections.defaultdict(lambda: self.default_error_plugin)
def __plugins_dict(self):
import galaxy.tools.error_reports.plugins
return plugin_config.plugins_dict(galaxy.tools.error_reports.plugins, 'plugin_type')
class NullErrorPlugin(object):
def submit_report(self, dataset, job, tool, **kwargs):
return "Submitted Bug Report"
NULL_ERROR_PLUGIN = NullErrorPlugin()
class ErrorPlugin(object):
def __init__(self, plugin_classes, plugins_source, **kwargs):
self.extra_kwargs = kwargs
self.app = kwargs['app']
self.plugin_classes = plugin_classes
self.plugins = self.__plugins_from_source(plugins_source)
def _can_access_dataset(self, dataset, user):
if user:
roles = user.all_roles()
else:
roles = []
return self.app.security_agent.can_access_dataset(roles, dataset.dataset)
def submit_report(self, dataset, job, tool, user=None, user_submission=False, **kwargs):
if user_submission:
assert self._can_access_dataset(dataset, user), Exception("You are not allowed to access this dataset.")
responses = []
for plugin in self.plugins:
if user_submission == plugin.user_submission:
try:
response = plugin.submit_report(dataset, job, tool, **kwargs)
log.debug("Bug report plugin %s generated response %s", plugin, response)
if plugin.verbose and response:
responses.append(response)
except Exception:
log.exception("Failed to generate submit_report commands for plugin %s", plugin)
return responses
def __plugins_from_source(self, plugins_source):
return plugin_config.load_plugins(self.plugin_classes, plugins_source, self.extra_kwargs)
@staticmethod
def from_file(plugin_classes, conf_file, **kwargs):
if not conf_file or not os.path.exists(conf_file):
return NULL_ERROR_PLUGIN
plugins_source = plugin_config.plugin_source_from_path(conf_file)
return ErrorPlugin(plugin_classes, plugins_source, **kwargs)
lib/galaxy/tools/error_reports/plugins/__init__.py
+25
View File
@@ -0,0 +1,25 @@
"""This module describes the abstract interface for :class:`InstrumentPlugin`.
These are responsible for collecting and formatting a coherent set of metrics.
"""
from abc import (
ABCMeta,
abstractmethod
)
import six
@six.add_metaclass(ABCMeta)
class ErrorPlugin( object ):
"""Describes how to send bug reports to various locations."""
@property
@abstractmethod
def plugin_type( self ):
"""Short string providing labelling this plugin """
def submit_report( self, dataset, job, tool, user_submission=False, **kwargs ):
"""Submit the bug report and render a string to be displayed to the user.
"""
return None
lib/galaxy/tools/error_reports/plugins/biostars.py
+42
View File
@@ -0,0 +1,42 @@
"""The module describes the ``biostars`` error plugin."""
from __future__ import absolute_import
import logging
from galaxy.util import biostar
from galaxy.util import string_as_bool
from galaxy.web.base.controller import url_for
from ..plugins import ErrorPlugin
log = logging.getLogger( __name__ )
class BiostarsPlugin( ErrorPlugin ):
"""Send error report as an issue on Biostars
"""
plugin_type = "biostars"
def __init__( self, **kwargs ):
self.app = kwargs['app']
self.verbose = string_as_bool(kwargs.get('verbose', True))
self.user_submission = string_as_bool(kwargs.get('user_submission', True))
def submit_report( self, dataset, job, tool, **kwargs ):
"""Doesn't do anything, just shows a link to submit on biostars.
"""
try:
assert biostar.biostar_enabled( self.app ), ValueError( "Biostar is not configured for this galaxy instance" )
assert self.app.config.biostar_enable_bug_reports, ValueError( "Biostar is not configured to allow bug reporting for this galaxy instance" )
print(kwargs)
url = url_for(controller='biostar',
action='biostar_tool_bug_report',
hda=self.app.security.encode_id(dataset.id),
email=kwargs['email'], message=kwargs['message'])
return ( 'Click <a href="%s">here</a> to submit to BioStars' % url, 'success' )
except Exception as e:
return ( "An error occurred submitting the report to biostars: %s" % str( e ), "danger" )
__all__ = ( 'BiostarsPlugin', )
lib/galaxy/tools/error_reports/plugins/email.py
+35
View File
@@ -0,0 +1,35 @@
"""The module describes the ``email`` error plugin."""
from __future__ import absolute_import
import logging
from galaxy.tools.errors import EmailErrorReporter
from galaxy.util import string_as_bool
from ..plugins import ErrorPlugin
log = logging.getLogger( __name__ )
class EmailPlugin( ErrorPlugin ):
"""Send error report as an email
"""
plugin_type = "email"
def __init__( self, **kwargs ):
self.app = kwargs['app']
self.verbose = string_as_bool(kwargs.get('verbose', True))
self.user_submission = string_as_bool(kwargs.get('user_submission', True))
def submit_report( self, dataset, job, tool, **kwargs ):
"""Send report as an email
"""
try:
error_reporter = EmailErrorReporter( dataset.id, self.app )
error_reporter.send_report( user=job.get_user(), email=kwargs.get('email', None), message=kwargs.get('message', None) )
return ( "Your error report has been sent", "success" )
except Exception as e:
return ( "An error occurred sending the report by email: %s" % str( e ), "danger" )
__all__ = ( 'EmailPlugin', )
lib/galaxy/tools/error_reports/plugins/json.py
+57
View File
@@ -0,0 +1,57 @@
"""The module describes the ``json`` error plugin."""
from __future__ import absolute_import
import json
import logging
import os
import tempfile
from galaxy.util import string_as_bool
from ..plugins import ErrorPlugin
log = logging.getLogger( __name__ )
class JsonPlugin( ErrorPlugin ):
"""Write error report to a JSON file.
"""
plugin_type = "json"
def __init__( self, **kwargs ):
self.app = kwargs['app']
self.verbose = string_as_bool(kwargs.get('verbose', False))
self.user_submission = string_as_bool(kwargs.get('user_submission', False))
self.report_directory = kwargs.get( "directory", tempfile.gettempdir() )
if not os.path.exists(self.report_directory):
os.makedirs(self.report_directory)
def submit_report( self, dataset, job, tool, **kwargs ):
"""Write the report to a json file.
"""
path = os.path.join( self.report_directory, str(dataset.id))
with open( path, 'w' ) as handle:
data = {
'info' : job.info,
'id' : job.id,
'command_line' : job.command_line,
'stderr' : job.stderr,
'traceback': job.traceback,
'exit_code': job.exit_code,
'stdout': job.stdout,
'handler': job.handler,
'user': job.get_user().to_dict(),
'tool_version': job.tool_version,
'tool_xml': str(tool.config_file) if tool else None
}
if 'email' in kwargs:
data['email'] = kwargs['email']
if 'message' in kwargs:
data['message'] = kwargs['message']
json.dump(data, handle, indent=2)
return ( 'Wrote error report to %s' % path, 'success' )
__all__ = ( 'JsonPlugin', )
lib/galaxy/tools/error_reports/plugins/sentry.py
+52
View File
@@ -0,0 +1,52 @@
"""The module describes the ``sentry`` error plugin plugin."""
import logging
from galaxy.util import string_as_bool
from ..plugins import ErrorPlugin
log = logging.getLogger( __name__ )
class SentryPlugin( ErrorPlugin ):
"""Send error report to Sentry.
"""
plugin_type = "sentry"
def __init__( self, **kwargs ):
self.app = kwargs['app']
self.verbose = string_as_bool(kwargs.get('verbose', False))
self.user_submission = string_as_bool(kwargs.get('user_submission', False))
def submit_report( self, dataset, job, tool, **kwargs ):
"""Submit the error report to sentry
"""
if self.app.sentry_client:
extra = {
'info' : job.info,
'id' : job.id,
'command_line' : job.command_line,
'stderr' : job.stderr,
'traceback': job.traceback,
'exit_code': job.exit_code,
'stdout': job.stdout,
'handler': job.handler,
'user': job.get_user(),
'tool_version': job.tool_version,
'tool_xml': tool.config_file if tool else None
}
if 'email' in kwargs:
extra['email'] = kwargs['email']
if 'message' in kwargs:
extra['message'] = kwargs['message']
response = self.app.sentry_client.capture(
'raven.events.Message',
message="Galaxy Job Error: %s v.%s" % (job.tool_id, job.tool_version),
extra=extra,
)
return ( 'Submitted bug report to Sentry. Your guru meditation number is %s' % response, 'success' )
__all__ = ( 'SentryPlugin', )
lib/galaxy/tools/errors.py
+2 -2
View File
@@ -67,7 +67,7 @@ error_report_template_html = """
<h3>Error Localization</h3>
<table style="margin:1em">
<tbody>
<tr><td>Dataset</td><td>${dataset_id} (${dataset_id_encoded})</td></tr>
<tr><td>Dataset</td><td><a href="${hda_show_params_link}">${dataset_id} (${dataset_id_encoded})</a></td></tr>
<tr style="background-color: #f2f2f2"><td>History</td><td><a href="${history_view_link}">${history_id} (${history_id_encoded})</a></td></tr>
<tr><td>Failed Job</td><td>${hid}: ${history_item_name} (${hda_id_encoded})</td></tr>
</tbody>
@@ -84,7 +84,7 @@ ${message}
<h3>Detailed Job Information</h3>
Job environment and execution information is available at the job <a href="${hda_show_params_link}">Info Page</a>.
Job environment and execution information is available at the job <a href="${hda_show_params_link}">info page</a>.
<table style="margin:1em">
<tbody>
lib/galaxy/tools/parameters/grouping.py
+3
View File
@@ -452,6 +452,7 @@ class UploadDataset( Group ):
file_type = self.get_file_type( context )
d_type = self.get_datatype( trans, context )
dbkey = context.get( 'dbkey', None )
tag_using_filenames = context.get('tag_using_filenames', False)
writable_files = d_type.writable_files
writable_files_offset = 0
groups_incoming = [ None for _ in writable_files ]
@@ -470,6 +471,7 @@ class UploadDataset( Group ):
dataset.metadata = {}
dataset.composite_files = {}
dataset.uuid = None
dataset.tag_using_filenames = None
# load metadata
files_metadata = context.get( self.metadata_ref, {} )
metadata_name_substition_default_dict = dict( ( composite_file.substitute_name_with_metadata, d_type.metadata_spec[ composite_file.substitute_name_with_metadata ].default ) for composite_file in d_type.composite_files.values() if composite_file.substitute_name_with_metadata )
@@ -520,6 +522,7 @@ class UploadDataset( Group ):
dataset.datatype = d_type
dataset.ext = self.get_datatype_ext( trans, context )
dataset.dbkey = dbkey
dataset.tag_using_filenames = tag_using_filenames
rval.append( dataset )
return rval
lib/galaxy/tools/parser/interface.py
+1 -1
View File
@@ -8,8 +8,8 @@ import six
NOT_IMPLEMENTED_MESSAGE = "Galaxy tool format does not yet support this tool feature."
@six.add_metaclass(ABCMeta)
@six.python_2_unicode_compatible
@six.add_metaclass(ABCMeta)
class ToolSource(object):
""" This interface represents an abstract source to parse tool
information from.
lib/galaxy/tools/parser/output_objects.py
+12 -8
View File
@@ -118,11 +118,7 @@ class ToolOutputCollection( ToolOutputBase ):
if len( self.outputs ) > 1:
output_parts = [ToolOutputCollectionPart(self, k, v) for k, v in self.outputs.items()]
else:
# either must have specified structured_like or something worse
if self.structure.structured_like:
collection_prototype = inputs[ self.structure.structured_like ].collection
else:
collection_prototype = type_registry.prototype( self.structure.collection_type )
collection_prototype = self.structure.collection_prototype( inputs, type_registry )
def prototype_dataset_element_to_output( element, parent_ids=[] ):
name = element.element_identifier
@@ -178,9 +174,9 @@ class ToolOutputCollectionStructure( object ):
def __init__(
self,
collection_type,
collection_type_source,
structured_like,
dataset_collector_descriptions,
collection_type_source=None,
structured_like=None,
dataset_collector_descriptions=None,
):
self.collection_type = collection_type
self.collection_type_source = collection_type_source
@@ -194,6 +190,14 @@ class ToolOutputCollectionStructure( object ):
raise ValueError( "Cannot specify dynamic structure (discovered_datasets) and structured_like attribute." )
self.dynamic = dataset_collector_descriptions is not None
def collection_prototype( self, inputs, type_registry ):
# either must have specified structured_like or something worse
if self.structured_like:
collection_prototype = inputs[ self.structured_like ].collection
else:
collection_prototype = type_registry.prototype( self.collection_type )
return collection_prototype
class ToolOutputCollectionPart( object ):
lib/galaxy/tools/sort_collection_list.xml
+73
View File
@@ -0,0 +1,73 @@
<tool id="__SORTLIST__"
name="Sort Collection"
version="1.0.0"
tool_type="sort_collection">
<description>of list of datasets</description>
<type class="SortTool" module="galaxy.tools" />
<action module="galaxy.tools.actions.model_operations"
class="ModelOperationToolAction"/>
<inputs>
<param type="data_collection" collection_type="list" name="input" label="Input Collection" />
<param type="select" name="sort_type" label="Sort collection identifiers" help="">
<option value="alpha">alphabetically</option>
<option value="numeric">numerically (strips all characters except numbers)</option>
</param>
</inputs>
<outputs>
<collection name="output" format_source="input" type="list" label="${on_string} (sorted)" >
</collection>
</outputs>
<tests>
<test>
<param name="input">
<collection type="list">
<element name="def" value="simple_line_alternative.txt" />
<element name="abc" value="simple_line.txt" />
</collection>
</param>
<param name="sort_type" value="alpha" />
<output_collection name="output" type="list">
<element name="abc">
<assert_contents>
<has_text_matching expression="^This is a line of text.\n$" />
</assert_contents>
</element>
<element name="def">
<assert_contents>
<has_text_matching expression="^This is a different line of text.\n$" />
</assert_contents>
</element>
</output_collection>
</test>
<test>
<param name="input">
<collection type="list">
<element name="def0123400" value="simple_line_alternative.txt" />
<element name="abc5678" value="simple_line.txt" />
</collection>
</param>
<param name="sort_type" value="numeric" />
<output_collection name="output" type="list">
<element name="abc5678">
<assert_contents>
<has_text_matching expression="^This is a line of text.\n$" />
</assert_contents>
</element>
<element name="def0123400">
<assert_contents>
<has_text_matching expression="^This is a different line of text.\n$" />
</assert_contents>
</element>
</output_collection>
</test>
</tests>
<help><![CDATA[
This tool takes list-type collections - and produces a sorted ist from the inputs. The collection identifiers are sorted either alphabetically or numerically (where the characters other than 0-9 are stripped before sorting).
.. class:: infomark
This tool will create new history datasets from your collection but your quota usage will not increase.
]]></help>
</tool>
lib/galaxy/tools/toolbox/base.py
+4 -1
View File
@@ -454,7 +454,10 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ):
# We now likely have a Toolshed guid passed in, but no supporting database entries
# If the tool exists by exact id and is loaded then provide exact match within a list
if tool_id in self._tools_by_id:
return[ self._tools_by_id[ tool_id ] ]
if get_all_versions:
return [ self._tools_by_id[ tool_id ] ]
else:
return self._tools_by_id[ tool_id ]
return None
def has_tool( self, tool_id, tool_version=None, exact=False ):
lib/galaxy/util/biostar.py
+14 -1
View File
@@ -169,7 +169,20 @@ class BiostarErrorReporter( ErrorReporter ):
assert self._can_access_dataset( user ), Exception( "You are not allowed to access this dataset." )
tool_version_select_field, tools, tool = \
self.app.toolbox.get_tool_components( self.tool_id, tool_version=None, get_loaded_tools_by_lineage=False, set_selected=True )
payload = { 'title': 'Bug report on "%s" tool' % ( tool.name ), 'content': self.report.replace( '\n', '<br />' ).replace( '\r', '' ), 'tag_val': slugify( 'bug report' ) }
# Strip out unwanted HTML characters
html_remove = ['<html>', '<body>', '</body>', '</html>', '\n', '\r']
report = self.html_report
for tag in html_remove:
report = report.replace(tag, '')
# Must lstrip spaces or it isn't recognised as HTML
report = report.lstrip()
payload = {
'title': 'Bug report on "%s" tool' % ( tool.name ),
'content': report,
'tag_val': slugify( 'bug report' )
}
# Get footer for email from here
payload2 = populate_tool_payload( tool=tool )
if 'content' in payload2:
lib/galaxy/util/plugin_config.py
+1 -1
View File
@@ -72,7 +72,7 @@ def __load_plugins_from_dicts(plugins_dict, configs, extra_kwds):
def plugin_source_from_path(path):
if path.endswith(".yaml") or path.endswith(".yml"):
if path.endswith(".yaml") or path.endswith(".yml") or path.endswith(".yaml.sample") or path.endswith(".yml.sample"):
return ('dict', __read_yaml(path))
else:
return ('xml', ElementTree.parse( path ).getroot())
lib/galaxy/web/base/controller.py
+58 -36
View File
@@ -2,40 +2,63 @@
Contains functionality needed in every web interface
"""
import logging
import operator
import re
from six import string_types, text_type
from paste.httpexceptions import (
HTTPBadRequest,
HTTPInternalServerError,
HTTPNotImplemented,
HTTPRequestRangeNotSatisfiable
)
from six import (
string_types,
text_type
)
from sqlalchemy import true
from paste.httpexceptions import HTTPBadRequest, HTTPInternalServerError
from paste.httpexceptions import HTTPNotImplemented, HTTPRequestRangeNotSatisfiable
from galaxy import exceptions
from galaxy import web
from galaxy import model
from galaxy import security
from galaxy import util
from galaxy.web import error, url_for
from galaxy.web.form_builder import AddressField, CheckboxField, SelectField, TextArea, TextField
from galaxy.web.form_builder import build_select_field, HistoryField, PasswordField, WorkflowField, WorkflowMappingField
from galaxy.workflow.modules import WorkflowModuleInjector
from galaxy.security.validate_user_input import validate_publicname
from galaxy.util.sanitize_html import sanitize_html
from galaxy.model.item_attrs import UsesAnnotations
from galaxy.util.dictifiable import Dictifiable
from galaxy import (
exceptions,
model,
security,
util,
web
)
from galaxy.datatypes.interval import ChromatinInteractions
from galaxy.model import ExtendedMetadata, ExtendedMetadataIndex, LibraryDatasetDatasetAssociation, HistoryDatasetAssociation
from galaxy.managers import api_keys
from galaxy.managers import tags
from galaxy.managers import workflows
from galaxy.managers import base as managers_base
from galaxy.managers import users
from galaxy.managers import configuration
from galaxy.managers import (
api_keys,
base as managers_base,
configuration,
tags,
users,
workflows
)
from galaxy.model import (
ExtendedMetadata,
ExtendedMetadataIndex,
HistoryDatasetAssociation,
LibraryDatasetDatasetAssociation
)
from galaxy.model.item_attrs import UsesAnnotations
from galaxy.security.validate_user_input import validate_publicname
from galaxy.util.dictifiable import Dictifiable
from galaxy.util.sanitize_html import sanitize_html
from galaxy.web import (
error,
url_for
)
from galaxy.web.form_builder import (
AddressField,
build_select_field,
CheckboxField,
HistoryField,
PasswordField,
SelectField,
TextArea,
TextField,
WorkflowField,
WorkflowMappingField
)
from galaxy.workflow.modules import WorkflowModuleInjector
log = logging.getLogger( __name__ )
@@ -132,7 +155,7 @@ class BaseController( object ):
values = [ values ]
# TODO: it may be more helpful to the consumer if we error on incomplete 3-tuples
# (instead of relying on zip to shorten)
return zip( attrs, ops, values )
return list(zip( attrs, ops, values ))
def parse_limit_offset( self, qdict ):
"""
@@ -211,8 +234,8 @@ class BaseAPIController( BaseController ):
msg = "The following value(s) for associated users and/or groups could not be parsed: %s." % ', '.join( invalid )
msg += " Valid values are email addresses of users, names of groups, or IDs of both."
raise Exception( msg )
payload['in_users'] = map( str, new_in_users )
payload['in_groups'] = map( str, new_in_groups )
payload['in_users'] = list(map( str, new_in_users ))
payload['in_groups'] = list(map( str, new_in_groups ))
def not_implemented( self, trans, **kwd ):
raise HTTPNotImplemented()
@@ -903,7 +926,7 @@ class UsesVisualizationMixin( UsesLibraryMixinItems ):
if not job:
return None
tool = trans.app.toolbox.get_tool( job.tool_id )
tool = trans.app.toolbox.get_tool( job.tool_id, tool_version=job.tool_version )
if not tool:
return None
@@ -2176,7 +2199,6 @@ class UsesTagsMixin( SharableItemSecurityMixin ):
return self.get_tag_handler( trans )._get_item_tag_assoc( user, tagged_item, tag_name )
def set_tags_from_list( self, trans, item, new_tags_list, user=None ):
# Method deprecated - try to use TagsHandler instead.
tags_manager = tags.GalaxyTagManager( trans.app.model.context )
return tags_manager.set_tags_from_list( user, item, new_tags_list )
@@ -2322,6 +2344,6 @@ def sort_by_attr( seq, attr ):
# (seq[i].attr, i, seq[i]) and sort it. The second item of tuple is needed not
# only to provide stable sorting, but mainly to eliminate comparison of objects
# (which can be expensive or prohibited) in case of equal attribute values.
intermed = map( None, map( getattr, seq, ( attr, ) * len( seq ) ), xrange( len( seq ) ), seq )
intermed = [(getattr(v, attr), i, v) for i, v in enumerate(seq)]
intermed.sort()
return map( operator.getitem, intermed, ( -1, ) * len( intermed ) )
return [_[-1] for _ in intermed]
lib/galaxy/web/base/interactive_environments.py
+6 -7
View File
@@ -1,4 +1,3 @@
import ConfigParser
import json
import logging
import os
@@ -6,11 +5,11 @@ import random
import stat
import tempfile
import uuid
from subprocess import PIPE, Popen
from sys import platform as _platform
import yaml
from six.moves import configparser
from galaxy import model, web
from galaxy.containers import build_container_interfaces
@@ -121,7 +120,7 @@ class InteractiveEnvironmentRequest(object):
def load_deploy_config(self, default_dict={}):
# For backwards compat, any new variables added to the base .ini file
# will need to be recorded here. The ConfigParser doesn't provide a
# will need to be recorded here. The configparser doesn't provide a
# .get() that will ignore missing sections, so we must make use of
# their defaults dictionary instead.
default_dict = {
@@ -133,7 +132,7 @@ class InteractiveEnvironmentRequest(object):
'docker_galaxy_temp_dir': None,
'docker_connect_port': None,
}
viz_config = ConfigParser.SafeConfigParser(default_dict)
viz_config = configparser.SafeConfigParser(default_dict)
conf_path = os.path.join( self.attr.our_config_dir, self.attr.viz_id + ".ini" )
if not os.path.exists( conf_path ):
conf_path = "%s.sample" % conf_path
@@ -253,7 +252,7 @@ class InteractiveEnvironmentRequest(object):
env_override = {}
conf = self.get_conf_dict()
conf.update(env_override)
return dict([(key.upper(), item) for key, item in conf.items()])
return dict((key.upper(), item) for key, item in conf.items())
def _get_import_volume_for_run(self):
if self.use_volumes and self.attr.import_volume:
@@ -271,8 +270,8 @@ class InteractiveEnvironmentRequest(object):
volumes = []
env = self._get_env_for_run(env_override)
import_volume_def = self._get_import_volume_for_run()
env_str = ' '.join(['-e "%s=%s"' % (key, item) for key, item in env.items()])
volume_str = ' '.join(['-v "%s"' % volume for volume in volumes]) if self.use_volumes else ''
env_str = ' '.join('-e "%s=%s"' % (key, item) for key, item in env.items())
volume_str = ' '.join('-v "%s"' % volume for volume in volumes) if self.use_volumes else ''
import_volume_str = '-v "{import_volume}"'.format(import_volume=import_volume_def) if import_volume_def else ''
name = None
# This is the basic docker command such as "sudo -u docker docker {docker_args}"
lib/galaxy/web/base/pluginframework.py
+9 -7
View File
@@ -7,17 +7,19 @@ and base class for plugins that:
* serve templated html
* have some configuration at startup
"""
import imp
import logging
import os.path
import sys
import imp
from galaxy import util
from galaxy.util import odict
from galaxy.util import bunch
import mako.lookup
import logging
from galaxy import util
from galaxy.util import (
bunch,
odict
)
log = logging.getLogger( __name__ )
lib/galaxy/webapps/galaxy/api/jobs.py
+31
View File
@@ -352,3 +352,34 @@ class JobController( BaseAPIController, UsesLibraryMixinItems ):
if all( list( a.dataset.deleted is False for a in job.output_datasets ) ):
out.append( self.encode_all_ids( trans, job.to_dict( 'element' ), True ) )
return out
@expose_api
def error( self, trans, id, **kwd ):
"""
error( trans, id )
* POST /api/jobs/{id}/error
submits a bug report via the API.
:type id: string
:param id: Encoded job id
:rtype: dictionary
:returns: dictionary containing information regarding where the error report was sent.
"""
# Get dataset on which this error was triggered
try:
decoded_dataset_id = self.decode_id( kwd['dataset_id'] )
except Exception:
raise exceptions.MalformedId()
dataset = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( decoded_dataset_id )
# Get job
job = self.__get_job( trans, id )
tool = trans.app.toolbox.get_tool( job.tool_id, tool_version=job.tool_version ) or None
messages = trans.app.error_reports.default_error_plugin.submit_report(
dataset, job, tool, user_submission=True, user=trans.user,
email=kwd.get('email', trans.user.email),
message=kwd.get('message', None)
)
return { 'messages': messages }
lib/galaxy/webapps/galaxy/api/lda_datasets.py
+8 -2
View File
@@ -16,7 +16,7 @@ from galaxy import exceptions
from galaxy import util
from galaxy import web
from galaxy.exceptions import ObjectNotFound
from galaxy.managers import folders, roles
from galaxy.managers import folders, roles, tags
from galaxy.tools.actions import upload_common
from galaxy.tools.parameters import populate_state
from galaxy.util.streamball import StreamBall
@@ -56,6 +56,8 @@ class LibraryDatasetsController( BaseAPIController, UsesVisualizationMixin ):
current_user_roles = trans.get_current_user_roles()
tag_manager = tags.GalaxyTagManager( trans.sa_session )
# Build the full path for breadcrumb purposes.
full_path = self._build_path( trans, library_dataset.folder )
dataset_item = ( trans.security.encode_id( library_dataset.id ), library_dataset.name )
@@ -78,6 +80,7 @@ class LibraryDatasetsController( BaseAPIController, UsesVisualizationMixin ):
rval[ 'date_uploaded' ] = library_dataset.library_dataset_dataset_association.create_time.strftime( "%Y-%m-%d %I:%M %p" )
rval[ 'can_user_modify' ] = trans.app.security_agent.can_modify_library_item( current_user_roles, library_dataset) or trans.user_is_admin()
rval[ 'is_unrestricted' ] = trans.app.security_agent.dataset_is_public( library_dataset.library_dataset_dataset_association.dataset )
rval[ 'tags' ] = tag_manager.get_tags_str(library_dataset.library_dataset_dataset_association.tags)
# Manage dataset permission is always attached to the dataset itself, not the the ld or ldda to maintain consistency
rval[ 'can_user_manage' ] = trans.app.security_agent.can_manage_dataset( current_user_roles, library_dataset.library_dataset_dataset_association.dataset) or trans.user_is_admin()
@@ -396,6 +399,8 @@ class LibraryDatasetsController( BaseAPIController, UsesVisualizationMixin ):
:type file_type: str
:param dbkey: dbkey of the loaded genome, defaults to '?' (unknown)
:type dbkey: str
:param tag_using_filenames: flag whether to generate dataset tags from filenames
:type tag_using_filenames: bool
:type dictionary
:returns: dict containing information about the created upload job
:rtype: dictionary
@@ -408,7 +413,8 @@ class LibraryDatasetsController( BaseAPIController, UsesVisualizationMixin ):
kwd[ 'to_posix_lines' ] = True
kwd[ 'dbkey' ] = kwd.get( 'dbkey', '?' )
kwd[ 'file_type' ] = kwd.get( 'file_type', 'auto' )
kwd[ 'link_data_only' ] = 'link_to_files' if util.asbool( kwd.get( 'link_data', False ) ) else 'copy_files'
kwd['link_data_only'] = 'link_to_files' if util.string_as_bool( kwd.get( 'link_data', False ) ) else 'copy_files'
kwd[ 'tag_using_filenames' ] = util.string_as_bool( kwd.get( 'tag_using_filenames', None ) )
encoded_folder_id = kwd.get( 'encoded_folder_id', None )
if encoded_folder_id is not None:
folder_id = self.folder_manager.cut_and_decode( trans, encoded_folder_id )
lib/galaxy/webapps/galaxy/api/library_contents.py
+2
View File
@@ -182,6 +182,8 @@ class LibraryContentsController( BaseAPIController, UsesLibraryMixin, UsesLibrar
folder to create
* description: (optional, only if create_type is 'folder')
description of the folder to create
* tag_using_filename: (optional)
create tags on datasets using the file's original name
:returns: a dictionary describing the new item unless ``from_hdca_id`` is supplied,
in that case a list of such dictionaries is returned.
lib/galaxy/webapps/galaxy/api/users.py
+68
View File
@@ -11,6 +11,8 @@ from datetime import datetime
from markupsafe import escape
from sqlalchemy import false, true, and_, or_
import yaml
from galaxy import exceptions, util, web
from galaxy.exceptions import MessageException, ObjectInvalid
from galaxy.managers import users
@@ -250,6 +252,46 @@ class UserAPIController( BaseAPIController, UsesTagsMixin, CreatesUsersMixin, Cr
'nice_total_disk_usage': util.nice_size( usage ),
'quota_percent': percent}
def _get_extra_user_preferences(self, trans):
"""
Reads the file user_preferences_extra_conf.yml to display
admin defined user informations
"""
path = trans.app.config.user_preferences_extra_config_file
try:
with open(path, 'r') as stream:
config = yaml.load(stream)
except:
log.warn('Config file (%s) could not be found or is malformed.' % path)
return {}
return config['preferences'] if config else {}
def _build_extra_user_pref_inputs(self, preferences, user):
"""
Build extra user preferences inputs list.
Add values to the fields if present
"""
if not preferences:
return []
data = []
# Get data if present
data_key = "extra_user_preferences"
if data_key in user.preferences:
data = json.loads(user.preferences[data_key])
extra_pref_inputs = list()
# Build sections for different categories of inputs
for item, value in preferences.items():
if value is not None:
for input in value["inputs"]:
input['help'] = 'Required' if input['required'] else ''
field = item + '|' + input['name']
for data_item in data:
if field in data_item:
input['value'] = data[data_item]
extra_pref_inputs.append({'type': 'section', 'title': value['description'], 'name': item, 'expanded': True, 'inputs': value['inputs']})
return extra_pref_inputs
@expose_api
def get_information(self, trans, id, **kwd):
"""
@@ -302,6 +344,7 @@ class UserAPIController( BaseAPIController, UsesTagsMixin, CreatesUsersMixin, Cr
info_field['test_param']['data'].append({'label': info_form['name'], 'value': info_form['id']})
info_field['cases'].append({'value': info_form['id'], 'inputs': info_form['inputs']})
inputs.append(info_field)
address_inputs = [{'type': 'hidden', 'name': 'id', 'hidden': True}]
for field in AddressField.fields():
address_inputs.append({'type': 'text', 'name': field[0], 'label': field[1], 'help': field[2]})
@@ -315,6 +358,11 @@ class UserAPIController( BaseAPIController, UsesTagsMixin, CreatesUsersMixin, Cr
address_cache.append(input_copy)
address_repeat['cache'].append(address_cache)
inputs.append(address_repeat)
# Build input sections for extra user preferences
extra_user_pref = self._build_extra_user_pref_inputs( self._get_extra_user_preferences( trans ), user )
for item in extra_user_pref:
inputs.append(item)
else:
if user.active_repositories:
inputs.append(dict(id='name_input', name='username', label='Public name:', type='hidden', value=username, help='You cannot change your public name after you have created a repository in this tool shed.'))
@@ -382,6 +430,26 @@ class UserAPIController( BaseAPIController, UsesTagsMixin, CreatesUsersMixin, Cr
form_values = trans.model.FormValues(user_info_form, user_info_values)
trans.sa_session.add(form_values)
user.values = form_values
# Update values for extra user preference items
extra_user_pref_data = dict()
get_extra_pref_keys = self._get_extra_user_preferences( trans )
if get_extra_pref_keys is not None:
for key in get_extra_pref_keys:
key_prefix = key + '|'
for item in payload:
if item.startswith( key_prefix ):
# Show error message if the required field is empty
if payload[item] == "":
# Raise an exception when a required field is empty while saving the form
keys = item.split("|")
section = get_extra_pref_keys[keys[0]]
for input in section['inputs']:
if input['name'] == keys[1] and input['required']:
raise MessageException("Please fill the required field")
extra_user_pref_data[ item ] = payload[ item ]
user.preferences[ "extra_user_preferences" ] = json.dumps( extra_user_pref_data )
# Update user addresses
address_dicts = {}
address_count = 0
lib/galaxy/webapps/galaxy/buildapp.py
+2
View File
@@ -122,6 +122,7 @@ def paste_app_factory( global_conf, **kwargs ):
webapp.add_client_route( '/histories/list_shared' )
webapp.add_client_route( '/datasets/list' )
webapp.add_client_route( '/datasets/edit' )
webapp.add_client_route( '/datasets/error' )
webapp.add_client_route( '/workflow/run' )
webapp.add_client_route( '/workflow/import_workflow' )
webapp.add_client_route( '/custom_builds' )
@@ -761,6 +762,7 @@ def populate_api_routes( webapp, app ):
webapp.mapper.connect( 'job_inputs', '/api/jobs/{id}/inputs', controller='jobs', action='inputs', conditions=dict( method=['GET'] ) )
webapp.mapper.connect( 'job_outputs', '/api/jobs/{id}/outputs', controller='jobs', action='outputs', conditions=dict( method=['GET'] ) )
webapp.mapper.connect( 'build_for_rerun', '/api/jobs/{id}/build_for_rerun', controller='jobs', action='build_for_rerun', conditions=dict( method=['GET'] ) )
webapp.mapper.connect( 'job_error', '/api/jobs/{id}/error', controller='jobs', action='error', conditions=dict( method=['POST'] ) )
# Job files controllers. Only for consumption by remote job runners.
webapp.mapper.resource( 'file',
lib/galaxy/webapps/galaxy/controllers/admin.py
+7 -4
View File
@@ -1017,14 +1017,13 @@ class AdminGalaxy( controller.JSAppLauncher, AdminActions, UsesQuotaMixin, Quota
'inputs' : [{
'name' : 'name',
'label' : 'Name'
},{
}, {
'name' : 'description',
'label' : 'Description'
},
build_select_input( 'groups', 'Groups', all_groups, [] ),
build_select_input( 'users', 'Users', all_users, [] ),
{
'name' : 'auto_create',
build_select_input( 'users', 'Users', all_users, [] ), {
'name' : 'create_group_for_role',
'label' : 'Create a new role of the same name for this group:',
'type' : 'boolean'
} ] }
@@ -1080,7 +1079,11 @@ class AdminGalaxy( controller.JSAppLauncher, AdminActions, UsesQuotaMixin, Quota
'name' : 'name',
'label' : 'Name',
'value' : role.name
<<<<<<< HEAD
},{
=======
}, {
>>>>>>> admin_grid_roles
'name' : 'description',
'label' : 'Description',
'value' : role.description
lib/galaxy/webapps/galaxy/controllers/dataset.py
-13
View File
@@ -18,7 +18,6 @@ from galaxy.util.sanitize_html import sanitize_html
from galaxy.web import form_builder
from galaxy.web.base.controller import BaseUIController, ERROR, SUCCESS, url_for, UsesExtendedMetadataMixin
from galaxy.web.framework.helpers import grids, iff, time_ago, to_unicode
from galaxy.tools.errors import EmailErrorReporter
log = logging.getLogger( __name__ )
@@ -160,18 +159,6 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesItemRatings, Uses
exit_code = "Invalid dataset ID or you are not allowed to access this dataset"
return exit_code
@web.expose
def report_error( self, trans, id, email='', message="", **kwd ):
biostar_report = 'biostar' in str( kwd.get( 'submit_error_report') ).lower()
if biostar_report:
return trans.response.send_redirect( url_for( controller='biostar', action='biostar_tool_bug_report', hda=id, email=email, message=message ) )
try:
error_reporter = EmailErrorReporter( id, trans.app )
error_reporter.send_report( user=trans.user, email=email, message=message )
return trans.show_ok_message( "Your error report has been sent" )
except Exception as e:
return trans.show_error_message( "An error occurred sending the report by email: %s" % str( e ) )
@web.expose
def default(self, trans, dataset_id=None, **kwd):
return 'This link may not be followed from within Galaxy.'
lib/galaxy/webapps/galaxy/controllers/library_common.py
+1
View File
@@ -1151,6 +1151,7 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet
uploaded_dataset.dbkey = params.get( 'dbkey', None )
uploaded_dataset.to_posix_lines = params.get('to_posix_lines', None)
uploaded_dataset.space_to_tab = params.get( 'space_to_tab', None )
uploaded_dataset.tag_using_filenames = params.get( 'tag_using_filenames', True )
if in_folder:
uploaded_dataset.in_folder = in_folder
uploaded_dataset.data = upload_common.new_upload( trans, cntrller, uploaded_dataset, library_bunch )
lib/galaxy/workflow/extract.py
+1 -1
View File
@@ -305,7 +305,7 @@ class WorkflowSummary( object ):
def step_inputs( trans, job ):
tool = trans.app.toolbox.get_tool( job.tool_id )
tool = trans.app.toolbox.get_tool( job.tool_id, tool_version=job.tool_version )
param_values = job.get_param_values( trans.app, ignore_errors=True ) # If a tool was updated and e.g. had a text value changed to an integer, we don't want a traceback here
associations = __cleanup_param_values( tool.inputs, param_values )
tool_inputs = tool.params_to_strings( param_values, trans.app )
lib/tool_shed/metadata/metadata_generator.py
+1
View File
@@ -680,6 +680,7 @@ class MetadataGenerator( object ):
guid=guid,
name=tool.name,
version=tool.version,
profile=tool.profile,
description=tool.description,
version_string_cmd=tool.version_string_cmd,
tool_config=tool_config,
lib/tool_shed/util/workflow_util.py
+1 -1
View File
@@ -49,7 +49,7 @@ class RepoToolModule( ToolModule ):
break
else:
# We're in Galaxy.
self.tool = trans.app.toolbox.get_tool( self.tool_id )
self.tool = trans.app.toolbox.get_tool( self.tool_id, tool_version=self.tool_version )
if self.tool is None:
self.errors = 'unavailable'
self.post_job_actions = {}
lib/tool_shed/utility_containers/utility_container_manager.py
+5 -1
View File
@@ -136,7 +136,7 @@ class RepositoryDependency( object ):
class Tool( object ):
"""Tool object"""
def __init__( self, id=None, tool_config=None, tool_id=None, name=None, description=None, version=None, requirements=None,
def __init__( self, id=None, tool_config=None, tool_id=None, name=None, description=None, version=None, profile=None, requirements=None,
repository_id=None, changeset_revision=None, repository_installation_status=None ):
self.id = id
self.tool_config = tool_config
@@ -144,6 +144,7 @@ class Tool( object ):
self.name = name
self.description = description
self.version = version
self.profile = profile
self.requirements = requirements
self.repository_id = repository_id
self.changeset_revision = changeset_revision
@@ -438,6 +439,7 @@ class UtilityContainerManager( object ):
name='Name',
description='Description',
version='Version',
profile='Minimum Galaxy Version',
requirements='',
repository_id='',
changeset_revision='' )
@@ -479,6 +481,7 @@ class UtilityContainerManager( object ):
name = str( tool_dict.get( 'name', 'unknown' ) )
description = str( tool_dict.get( 'description', '' ) )
version = str( tool_dict.get( 'version', 'unknown' ) )
profile = str( tool_dict.get('profile', 'any'))
except Exception as e:
tool_config = str( e )
tool_id = 'unknown'
@@ -491,6 +494,7 @@ class UtilityContainerManager( object ):
name=name,
description=description,
version=version,
profile=profile,
requirements=requirements_str,
repository_id=repository_id,
changeset_revision=changeset_revision,
scripts/grt/export.py
+8 -2
View File
@@ -16,7 +16,7 @@ import logging
from collections import defaultdict
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')))
from galaxy.util.properties import load_app_properties
import galaxy
@@ -32,7 +32,7 @@ def _init(config, need_app=False):
if config.startswith('/'):
config_file = os.path.abspath(config)
else:
config_file = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, config))
config_file = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, config))
properties = load_app_properties(ini_file=config_file)
config = galaxy.config.Configuration(**properties)
@@ -292,6 +292,9 @@ def main(argv):
.filter(model.JobMetricNumeric.job_id > offset_start) \
.filter(model.JobMetricNumeric.job_id <= min(end_job_id, offset_start + args.batch_size)) \
.all():
# No associated job
if metric[0] not in job_tool_map:
continue
# If the tool is blacklisted, exclude everywhere
if job_tool_map[metric[0]] in blacklisted_tools:
continue
@@ -316,6 +319,9 @@ def main(argv):
.filter(model.JobParameter.job_id > offset_start) \
.filter(model.JobParameter.job_id <= min(end_job_id, offset_start + args.batch_size)) \
.all():
# No associated job
if param[0] not in job_tool_map:
continue
# If the tool is blacklisted, exclude everywhere
if job_tool_map[param[0]] in blacklisted_tools:
continue
scripts/grt/upload.py
+3 -5
View File
@@ -3,8 +3,6 @@
See doc/source/admin/grt.rst for more detailed usage information.
"""
from __future__ import print_function
import argparse
import os
import sys
@@ -37,7 +35,7 @@ def main(argv):
config = yaml.load(handle)
REPORT_DIR = args.report_directory
GRT_URL = config['grt']['url']
GRT_URL = config['grt']['url'].rstrip('/') + '/'
GRT_INSTANCE_ID = config['grt']['instance_id']
GRT_API_KEY = config['grt']['api_key']
@@ -55,8 +53,8 @@ def main(argv):
if report_id not in remote_reports:
logging.info("Uploading %s", report_id)
files = {
'meta': open(os.path.join(sys.argv[1], report_id + '.json'), 'rb'),
'data': open(os.path.join(sys.argv[1], report_id + '.tsv.gz'), 'rb')
'meta': open(os.path.join(REPORT_DIR, report_id + '.json'), 'rb'),
'data': open(os.path.join(REPORT_DIR, report_id + '.tar.gz'), 'rb')
}
data = {
'identifier': report_id
static/maps/mvc/dataset/dataset-error.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"dataset-error.js","sources":["../../../src/mvc/dataset/dataset-error.js"],"names":["define","Utils","Ui","Form","View","Backbone","extend","initialize","this","setElement","model","Model","dataset_id","Galaxy","params","render","data_url","root","get","self","url","success","dataset","job_url","creating_job","job","render_error_page","error","error_response","status","message","persistent","cls","display_message","$","$el","empty","append","_templateHeader","tool_id","stderr","_getBugFormTemplate","response","doNotClear","safe","UnescapedMessage","html","Message","inputs","help","options","type","name","label","value","user","area","form","title","buttons","save","Button","icon","floating","onclick","form_data","data","create","id","submit","ajax","messages","forEach"],"mappings":"AAAAA,QAAU,cAAe,iBAAkB,sBAAwB,SAAUC,EAAOC,EAAIC,GAGpF,GAAIC,GAAOC,SAASD,KAAKE,QACrBC,WAAY,WACRC,KAAKC,WAAY,UACjBD,KAAKE,MAAQ,GAAIL,UAASM,OAASC,WAAcC,OAAOC,OAAOF,aAC/DJ,KAAKO,UAITA,OAAQ,WACJ,GAAIC,GAAWH,OAAOI,KAAO,gBAAkBT,KAAKE,MAAMQ,IAAK,cAC3DC,EAAOX,IAEXP,GAAMiB,KACFE,IAAUJ,EACVK,QAAU,SAAUC,GAChB,GAAIC,GAAUV,OAAOI,KAAO,YAAcK,EAAQE,aAAe,YACjEvB,GAAMiB,KACFE,IAAUG,EACVF,QAAU,SAAUI,GACFZ,OAAOI,KAAO,YAAcK,EAAQE,aAAe,YACjEL,GAAKO,kBAAmBP,EAAMG,EAASG,IAE3CE,MAAU,WACN,GAAIC,IACAC,OAAU,QACVC,QAAW,uCACXC,YAAc,EACdC,IAAO,eAEXb,GAAKc,gBAAiBL,EAAgBT,EAAKe,EAAG,0BAI1DP,MAAU,WACN,GAAIC,IACAC,OAAU,QACVC,QAAW,2CACXC,YAAc,EACdC,IAAO,eAEXb,GAAKc,gBAAiBL,EAAgBT,EAAKe,EAAG,0BAM1DR,kBAAmB,SAAUP,EAAMG,EAASG,GACxCN,EAAKgB,IAAIC,QAAQC,OAAQlB,EAAKmB,mBAC9BnB,EAAKgB,IAAIE,OAAO,0BAChBlB,EAAKgB,IAAIE,OAAO,iDAAmDZ,EAAIc,QAAU,aACjFpB,EAAKgB,IAAIE,OAAO,2DAChBlB,EAAKgB,IAAIE,OAAO,qBAAuBZ,EAAIe,OAAS,UACpDrB,EAAKgB,IAAIE,OAAO,+BAChBlB,EAAKgB,IAAIE,OAAO,oUAChBlB,EAAKgB,IAAIE,OAAOlB,EAAKsB,oBAAoBnB,EAASG,KAItDQ,gBAAiB,SAAUS,EAAUP,EAAKQ,EAAYC,GAC9CA,EAOGD,EACCR,EAAIE,OAAQ,GAAInC,GAAG2C,iBAAkBH,GAAWP,KAEhDA,EAAIC,QAAQU,KAAM,GAAI5C,GAAG2C,iBAAkBH,GAAWP,KATvDQ,EACCR,EAAIE,OAAQ,GAAInC,GAAG6C,QAASL,GAAWP,KAEvCA,EAAIC,QAAQU,KAAM,GAAI5C,GAAG6C,QAASL,GAAWP,MAYzDG,gBAAiB,WACb,MAAO,oFAMXG,oBAAqB,SAASnB,EAASG,GACnC,GAAIN,GAAOX,KACPwC,IAEIC,KAAQ,qBACRC,WACAC,KAAQ,OACRC,KAAQ,QACRC,MAAS,aACTC,MAASzC,OAAO0C,KAAKrC,IAAI,WAGzB+B,KAAQ,gGACRC,WACAC,KAAQ,OACRK,MAAQ,EACRJ,KAAQ,UACRC,MAAS,YAgBbI,EAAO,GAAItD,IACXuD,MAAS,eACTV,OAASA,EACTW,SACIC,KAAU,GAAI1D,GAAG2D,QACbC,KAAW,SACXJ,MAAW,SACX1B,IAAW,4BACX+B,SAAW,QACXC,QAAW,WACP,GAAIC,GAAYR,EAAKS,KAAKC,SACtB/C,EAAMP,OAAOI,KAAO,YAAcQ,EAAI2C,GAAK,QAC/CH,GAAUrD,WAAaU,EAAQ8C,GAC/BjD,EAAKkD,OAAOJ,EAAW7C,QAKvC,OAAOqC,GAAKtB,KAIhBkC,OAAS,SAASJ,EAAW7C,GACzB,GAAID,GAAOX,IAEX0B,GAAEoC,MACEnB,KAAM,OACN/B,IAAKA,EACL8C,KAAMD,EACN5C,QAAS,SAAUqB,GAEfvB,EAAKgB,IAAIC,QAAQC,OAAQlB,EAAKmB,mBAE9BI,EAAS6B,SAASC,QAAQ,SAAS1C,GAC/BX,EAAKc,iBACDJ,OAAUC,EAAQ,GAClBA,QAAWA,EAAQ,GACnBC,YAAc,GACfZ,EAAKe,EAAG,sBAAuB,GAAM,MAGhDP,MAAU,WACN,GAAIC,IACAC,OAAU,QACVC,QAAW,iFACXC,YAAc,EACdC,IAAO,eAEXb,GAAKc,gBAAiBL,EAAgBT,EAAKe,EAAG,2BAM9D,QACI9B,KAAQA"}
static/maps/mvc/dataset/dataset-li-edit.js.map
+1 -1
View File
File diff suppressed because one or more lines are too long
static/maps/mvc/job/job-params.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"job-params.js","sources":["../../../src/mvc/job/job-params.js"],"names":["define","Utils","View","Backbone","extend","initialize","options","this","setElement","render","console","log","self","$el","empty","append"],"mappings":"AACAA,QAAU,eAAiB,SAAUC,GAkBjC,GAAIC,GAAOC,SAASD,KAAKE,QAErBC,WAAY,SAAUC,GAElBC,KAAKD,QAAUA,EACfC,KAAKC,WAAY,UACjBD,KAAKE,UAGTA,OAAQ,WACJC,QAAQC,IAAI,KACZ,IAAIC,GAAOL,IACXK,GAAKC,IAAIC,QAAQC,OAAQ,sBA4BjC,QACIb,KAAQA"}
static/maps/mvc/library/library-dataset-view.js.map
+1 -1
View File
File diff suppressed because one or more lines are too long
static/maps/mvc/library/library-foldertoolbar-view.js.map
+1 -1
View File
File diff suppressed because one or more lines are too long
static/scripts/bundled/admin.bundled.js
+3 -3
View File
File diff suppressed because one or more lines are too long
static/scripts/bundled/admin.bundled.js.map
+1 -1
View File
File diff suppressed because one or more lines are too long
static/scripts/bundled/analysis.bundled.js
+12 -12
View File
File diff suppressed because one or more lines are too long
static/scripts/bundled/analysis.bundled.js.map
+1 -1
View File
File diff suppressed because one or more lines are too long
static/scripts/bundled/libs.bundled.js
+4 -4
View File
File diff suppressed because one or more lines are too long
static/scripts/bundled/libs.bundled.js.map
+1 -1
View File
File diff suppressed because one or more lines are too long
static/scripts/mvc/dataset/dataset-error.js
+2
View File
@@ -0,0 +1,2 @@
define(["utils/utils","mvc/ui/ui-misc","mvc/form/form-view"],function(a,b,c){var d=Backbone.View.extend({initialize:function(){this.setElement("<div/>"),this.model=new Backbone.Model({dataset_id:Galaxy.params.dataset_id}),this.render()},render:function(){var b=Galaxy.root+"api/datasets/"+this.model.get("dataset_id"),c=this;a.get({url:b,success:function(b){var d=Galaxy.root+"api/jobs/"+b.creating_job+"?full=True";a.get({url:d,success:function(a){Galaxy.root+"api/jobs/"+b.creating_job+"?full=True";c.render_error_page(c,b,a)},error:function(){var a={status:"error",message:"Error occured while loading the job.",persistent:!0,cls:"errormessage"};c.display_message(a,c.$(".response-message"))}})},error:function(){var a={status:"error",message:"Error occured while loading the dataset.",persistent:!0,cls:"errormessage"};c.display_message(a,c.$(".response-message"))}})},render_error_page:function(a,b,c){a.$el.empty().append(a._templateHeader()),a.$el.append("<h2>Dataset Error</h2>"),a.$el.append("<p>An error occured while running the tool <b>"+c.tool_id+"</b>.</p>"),a.$el.append("<p>Tool execution generated the following messages:</p>"),a.$el.append('<pre class="code">'+c.stderr+"</pre>"),a.$el.append("<h2>Report This Error</pre>"),a.$el.append("<p>Usually the local Galaxy administrators regularly review errors that occur on the server. However, if you would like to provide additional information (such as what you were trying to do when the error occurred) and a contact e-mail address, we will be better able to investigate your problem and get back to you.</p>"),a.$el.append(a._getBugFormTemplate(b,c))},display_message:function(a,c,d,e){e?d?c.append(new b.UnescapedMessage(a).$el):c.empty().html(new b.UnescapedMessage(a).$el):d?c.append(new b.Message(a).$el):c.empty().html(new b.Message(a).$el)},_templateHeader:function(){return'<div class="page-container edit-attr"><div class="response-message"></div></div>'},_getBugFormTemplate:function(a,d){var e=this,f=[{help:"Your email address",options:[],type:"text",name:"email",label:"Your email",value:Galaxy.user.get("email")},{help:"Any additional comments you can provide regarding what you were doing at the time of the bug.",options:[],type:"text",area:!0,name:"message",label:"Message"}],g=new c({title:"Error Report",inputs:f,buttons:{save:new b.Button({icon:"fa-bug",title:"Report",cls:"ui-button btn btn-primary",floating:"clear",onclick:function(){var b=g.data.create(),c=Galaxy.root+"api/jobs/"+d.id+"/error";b.dataset_id=a.id,e.submit(b,c)}})}});return g.$el},submit:function(a,b){var c=this;$.ajax({type:"POST",url:b,data:a,success:function(a){c.$el.empty().append(c._templateHeader()),a.messages.forEach(function(a){c.display_message({status:a[1],message:a[0],persistent:!0},c.$(".response-message"),!0,!0)})},error:function(){var a={status:"error",message:"Error occured while saving. Please fill all the required fields and try again.",persistent:!0,cls:"errormessage"};c.display_message(a,c.$(".response-message"))}})}});return{View:d}});
//# sourceMappingURL=../../../maps/mvc/dataset/dataset-error.js.map
static/scripts/mvc/dataset/dataset-li-edit.js
+1 -1
View File
File diff suppressed because one or more lines are too long
static/scripts/mvc/job/job-params.js
+2
View File
@@ -0,0 +1,2 @@
define(["utils/utils"],function(a){var b=Backbone.View.extend({initialize:function(a){this.options=a,this.setElement("<div/>"),this.render()},render:function(){console.log("HI");var a=this;a.$el.empty().append("<h1>Testing</h1>")}});return{View:b}});
//# sourceMappingURL=../../../maps/mvc/job/job-params.js.map
static/scripts/mvc/library/library-dataset-view.js
+1 -1
View File
File diff suppressed because one or more lines are too long
static/scripts/mvc/library/library-foldertoolbar-view.js
+2 -2
View File
File diff suppressed because one or more lines are too long
static/style/blue/base.css
+1 -1
View File
File diff suppressed because one or more lines are too long
templates/show_params.mako
-6
View File
@@ -18,12 +18,6 @@
width: 25%;
}
.code {
white-space: pre-wrap;
background: #1d1f21;
color: white;
padding: 1em;
}
</style>
<%def name="inputs_recursive( input_params, param_values, depth=1, upgrade_messages=None )">
templates/webapps/galaxy/dataset/errors.mako
-120
View File
@@ -1,120 +0,0 @@
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<title>
Galaxy
%if app.config.brand:
| ${app.config.brand}
%endif
| Dataset generation errors
</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<link href="/static/style/base.css" rel="stylesheet" type="text/css" />
<style>
pre
{
background: white;
color: black;
border: dotted black 1px;
overflow: auto;
padding: 10px;
}
</style>
<script type="text/javascript">
function sendReport( button, form, target, doConfirm )
{
var doIt = true;
if ( doConfirm==true )
{
doIt = confirm( 'You are about to submit to a public forum, do you want to continue?' );
}
if ( doIt==true )
{
form.setAttribute( 'target', target );
for( i=0; i<form.elements.length; i++ )
{
if ( form.elements[i].type == 'submit' )
{
form.elements[i].disabled = true;
}
}
var hiddenInput = document.createElement('input');
hiddenInput.type = 'hidden';
hiddenInput.name = button.name;
hiddenInput.value = button.value;
form.appendChild( hiddenInput );
form.submit();
return false;
}
return false;
}
</script>
</head>
<body>
<h2>Dataset generation errors</h2>
<p><b>Dataset ${hda.hid}: ${hda.display_name() | h}</b></p>
<% job = hda.creating_job %>
%if job:
%if job.traceback:
The Galaxy framework encountered the following error while attempting to run the tool:
<pre>${ util.unicodify( job.traceback ) | h}</pre>
%endif
%if job.stderr or job.info:
Tool execution generated the following error message:
%if job.stderr:
<pre>${ util.unicodify( job.stderr ) | h}</pre>
%elif job.info:
<pre>${ util.unicodify( job.info ) | h}</pre>
%endif
%else:
Tool execution did not generate any error messages.
%endif
%if job.stdout:
The tool produced the following additional output:
<pre>${ util.unicodify( job.stdout ) | h}</pre>
%endif
%else:
The tool did not create any additional job / error info.
%endif
<%
if trans.user:
user_email = trans.user.email
else:
user_email = ''
%>
<h2>Report this error to the local Galaxy administrators</h2>
<p>
Usually the local Galaxy administrators regularly review errors that occur on the server.
However, if you would like to provide additional information (such as
what you were trying to do when the error occurred) and a contact e-mail
address, we will be better able to investigate your problem and get back
to you.
</p>
<div class="toolForm">
<div class="toolFormTitle">Error Report</div>
<div class="toolFormBody">
<form name="report_error" action="${h.url_for(controller='dataset', action='report_error')}" method="post" >
<input type="hidden" name="id" value="${trans.security.encode_id( hda.id)}" />
<div class="form-row">
<label>Your email</label>
<input type="text" name="email" size="40" value="${user_email|h}" />
</div>
<div class="form-row">
<label>Message</label>
<textarea name="message" rows="10" cols="60"></textarea>
</div>
<div class="form-row">
<input type="submit" name="submit_error_report" value="Report" onclick="return sendReport( this, this.form, '_self' );"/>
%if trans.app.config.biostar_url and trans.app.config.biostar_enable_bug_reports:
<input type="submit" name="submit_error_report" value="Post on Biostar" onclick="return sendReport( this, this.form, '_blank', true );"/>
%endif
</div>
</form>
</div>
</div>
</body>
</html>
templates/webapps/tool_shed/repository/common.mako
+2 -1
View File
@@ -280,7 +280,7 @@
elif folder.label == 'Invalid tool dependencies':
folder_label = "%s<i> - click the tool dependency to see why it is invalid</i>" % folder_label
elif folder.label == 'Valid tools':
col_span_str = 'colspan="3"'
col_span_str = 'colspan="4"'
if folder.description:
folder_label = "%s<i> - %s</i>" % ( folder_label, folder.description )
else:
@@ -870,6 +870,7 @@
%endif
<${cell_type}>${tool.description | h}</${cell_type}>
<${cell_type}>${tool.version | h}</${cell_type}>
<${cell_type}>${tool.profile | h}</${cell_type}>
##<${cell_type}>${tool.requirements | h}</${cell_type}>
</tr>
<%
test-data/test.vcf.gz
BIN
View File
Binary file not shown.
test/functional/tools/sample_datatypes_conf.xml
+3
View File
@@ -21,5 +21,8 @@
<datatype extension="bed" type="galaxy.datatypes.interval:Bed" display_in_upload="true" description="BED format provides a flexible way to define the data lines that are displayed in an annotation track. BED lines have three required columns and nine additional optional columns. The three required columns are chrom, chromStart and chromEnd." description_url="https://galaxyproject.org/learn/datatypes/#bed">
</datatype>
<datatype extension="xml" type="galaxy.datatypes.xml:GenericXml" mimetype="application/xml" display_in_upload="true"/>
<datatype extension="vcf" type="galaxy.datatypes.tabular:Vcf" display_in_upload="true"/>
<datatype extension="bgzip" type="galaxy.datatypes.binary:Binary" subclass="true" />
<datatype extension="vcf_bgzip" type="galaxy.datatypes.tabular:VcfGz" type_extension="bgzip" subclass="true" display_in_upload="true"/>
</registration>
</datatypes>
test/functional/tools/samples_tool_conf.xml
+2
View File
@@ -25,6 +25,7 @@
<tool file="composite_output.xml" />
<tool file="composite_output_tests.xml" />
<tool file="unicode_stream.xml" />
<tool file="vcf_bgzip.xml" />
<tool file="metadata.xml" />
<tool file="metadata_bam.xml" />
<tool file="metadata_bcf.xml" />
@@ -149,6 +150,7 @@
<tool file="${model_tools_path}/zip_collection.xml" />
<tool file="${model_tools_path}/filter_failed_collection.xml" />
<tool file="${model_tools_path}/flatten_collection.xml" />
<tool file="${model_tools_path}/sort_collection_list.xml" />
<tool file="${model_tools_path}/merge_collection.xml" />
<tool file="${model_tools_path}/relabel_from_file.xml" />
<tool file="${model_tools_path}/filter_from_file.xml" />
test/functional/tools/vcf_bgzip.xml
+18
View File
@@ -0,0 +1,18 @@
<tool id="vcf_bgzip_test" name="vcf_bgzip_test" version="0.1.0">
<command detect_errors="exit_code"><![CDATA[
stat '$input.metadata.tabix_index' &&
cp '$input' '$output'
]]></command>
<inputs>
<param name="input" format="vcf_bgzip" type="data" label="Source file"/>
</inputs>
<outputs>
<data format="vcf_bgzip" name="output" />
</outputs>
<tests>
<test>
<param name="input" ftype="vcf_bgzip" value="test.vcf.gz"/>
<output name="output" value="test.vcf.gz" md5="b08896c2d3ed4254e90b9372ef772821"/>
</test>
</tests>
</tool>
test/unit/tools/test_parsing.py
+4
View File
@@ -124,6 +124,10 @@ class XmlLoaderTestCase(BaseLoaderTestCase):
source_file_name = "bwa.xml"
source_contents = TOOL_XML_1
def test_tool_source_to_string(self):
# Previously this threw an Exception - test for regression.
str(self._tool_source)
def test_version(self):
assert self._tool_source.parse_version() == "1.0.1"