opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merging with dev

Browse Source
This commit is contained in:
anuprulez
2016-12-21 15:04:19 +01:00
parent 132e284d37 306c64d1bc
commit ef82eec610
156 changed files with 2587 additions and 1105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.ci/jenkins/integration/run_tests.sh
+1 -3
View File
@@ -1,5 +1,3 @@
#!/bin/bash
./scripts/common_startup.sh --dev-wheels
nosetests test/integration
./run_tests.sh --dockerize --db postgres --external_tmp --clean_pyc --integration "$@"
Makefile
+5
View File
@@ -13,6 +13,8 @@ IN_VENV=if [ -f $(VENV)/bin/activate ]; then . $(VENV)/bin/activate; fi;
PROJECT_URL?=https://github.com/galaxyproject/galaxy
GRUNT_DOCKER_NAME:=galaxy/client-builder:16.01
GRUNT_EXEC?=node_modules/grunt-cli/bin/grunt
WEBPACK_EXEC?=node_modules/webpack/bin/webpack.js
GXY_NODE_MODULES=client/node_modules
DOCS_DIR=doc
DOC_SOURCE_DIR=$(DOCS_DIR)/source
SLIDESHOW_DIR=$(DOC_SOURCE_DIR)/slideshow
@@ -92,6 +94,9 @@ client-install-libs: npm-deps ## Fetch updated client dependencies using bower.
client: grunt style ## Rebuild all client-side artifacts
charts: npm-deps ## Rebuild charts
NODE_PATH=$(GXY_NODE_MODULES) client/$(WEBPACK_EXEC) -p --config config/plugins/visualizations/charts/webpack.config.js
grunt-docker-image: ## Build docker image for running grunt
docker build -t ${GRUNT_DOCKER_NAME} client
client/galaxy/scripts/mvc/dataset/dataset-li-edit.js
+28 -19
View File
@@ -122,30 +122,38 @@ var DatasetListItemEdit = _super.extend(
return $details;
},
/**************************************************************************
/**************************************************************************
* Render help button to show tool help text without rerunning the tool.
* Issue #2100
*/
_renderToolHelpButton : function() {
var datasetID = this.model.attributes.dataset_id;
var jobID = this.model.attributes.creating_job;
var self = this;
var parseToolBuild = function(data) {
var toolName = data.name;
var toolHelp = (data.help) ? data.help : "No help is available for the tool.";
var helpString = '<div id="thdiv-' + datasetID + '" style="background:#eee; padding: 5px;"><hr><strong>Tool Help for ' + toolName + '</strong><br/><hr>';
helpString += toolHelp;
var helpString = '<div id="thdiv-' + datasetID + '" class="toolhelp">'
if (data.name && data.help){
helpString += '<strong>Tool help for ' + data.name + '</strong><hr/>';
helpString += data.help;
} else {
helpString += '<strong>Tool help is unavailable for this dataset.</strong><hr/>';
}
helpString += '</div>';
$('#dataset-' + datasetID).append($.parseHTML(helpString));
self.$el.find( '.details' ).append($.parseHTML(helpString));
};
var parseToolID = function(data) {
$.ajax({
url: '/api/tools/' + data.tool_id + '/build'
url: Galaxy.root + 'api/tools/' + data.tool_id + '/build'
}).done(function(data){
parseToolBuild(data);
}).fail(function(){console.log("Failed in api tools build call")});
}).fail(function(){
parseToolBuild({})
});
};
if (Galaxy.user.id === null){
return null
}
return faIconButton({
title: 'Tool Help',
classes: 'icon-btn',
@@ -153,15 +161,16 @@ var DatasetListItemEdit = _super.extend(
faIcon: 'fa-question',
onclick: function() {
var divString = 'thdiv-' + datasetID;
if ($("#" + divString).length > 0)
{
$("#" + divString).toggle();
if (self.$el.find(".toolhelp").length > 0){
self.$el.find(".toolhelp").toggle();
} else {
$.ajax({
url: '/api/jobs/' + jobID
}).done(function(data){
parseToolID(data);
}).fail(function(){console.log('Failed on recovering /api/jobs/' + jobID + ' call.')});
url: Galaxy.root + 'api/jobs/' + jobID
}).done(function(data){
parseToolID(data);
}).fail(function(){
console.log('Failed at recovering job information from the Galaxy API for job id "' + jobID + '".');
});
}
}
});
@@ -386,15 +395,15 @@ DatasetListItemEdit.prototype.templates = (function(){
var visualizationsTemplate = BASE_MVC.wrapTemplate([
'<% if( visualizations.length === 1 ){ %>',
'<a class="visualization-btn visualization-link icon-btn" href="<%- visualizations[0].href %>"',
'<a class="visualization-link icon-btn" href="<%- visualizations[0].href %>"',
' target="<%- visualizations[0].target %>" title="', _l( 'Visualize in' ),
' <%- visualizations[0].html %>">',
'<span class="fa fa-bar-chart-o"></span>',
'</a>',
'<% } else { %>',
'<div class="visualizations-dropdown dropdown">',
'<a class="visualization-btn icon-btn" data-toggle="dropdown" title="', _l( 'Visualize' ), '">',
'<div class="visualizations-dropdown dropdown icon-btn">',
'<a data-toggle="dropdown" title="', _l( 'Visualize' ), '">',
'<span class="fa fa-bar-chart-o"></span>',
'</a>',
'<ul class="dropdown-menu" role="menu">',
client/galaxy/scripts/mvc/tool/tool-form-composite.js
+22 -21
View File
@@ -107,24 +107,25 @@ define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view'
var wp_count = 0;
this.wp_inputs = {};
function _handleWorkflowParameter( value, callback ) {
var wp_name = self._isWorkflowParameter( value );
wp_name && callback( self.wp_inputs[ wp_name ] = self.wp_inputs[ wp_name ] || {
label : wp_name,
name : wp_name,
type : 'text',
color : 'hsl( ' + ( ++wp_count * 100 ) + ', 70%, 30% )',
style : 'ui-form-wp-source',
links : []
});
var re = /\$\{(.+?)\}/g;
while ( match = re.exec( String( value ) ) ) {
var wp_name = match[ 1 ];
callback( self.wp_inputs[ wp_name ] = self.wp_inputs[ wp_name ] || {
label : wp_name,
name : wp_name,
type : 'text',
color : 'hsl( ' + ( ++wp_count * 100 ) + ', 70%, 30% )',
style : 'ui-form-wp-source',
links : []
});
}
}
_.each( this.steps, function( step, i ) {
_.each( self.parms[ i ], function( input, name ) {
_handleWorkflowParameter( input.value, function( wp_input ) {
wp_input.links.push( step );
input.wp_linked = wp_input.name;
input.color = wp_input.color;
input.wp_linked = true;
input.type = 'text';
input.value = null;
input.backdrop = true;
input.style = 'ui-form-wp-target';
});
@@ -310,8 +311,15 @@ define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view'
new_value = { values: [ new_value.values[ 0 ] ] };
}
} else if ( input.wp_linked ) {
var wp_field = self.wp_form.field_list[ self.wp_form.data.match( input.wp_linked ) ];
wp_field && ( new_value = wp_field.value() );
new_value = input.value;
var re = /\$\{(.+?)\}/g;
while ( match = re.exec( input.value ) ) {
var wp_field = self.wp_form.field_list[ self.wp_form.data.match( match[ 1 ] ) ];
var wp_value = wp_field && wp_field.value();
if ( wp_value ) {
new_value = new_value.replace( new RegExp( '\\' + match[ 0 ], 'g' ), wp_value );
}
}
}
if ( new_value !== undefined ) {
field.value( new_value );
@@ -469,13 +477,6 @@ define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view'
_.each( this.forms, function( form ) { form && form.portlet[ enabled ? 'enable' : 'disable' ]() } );
},
/** Handle workflow parameter */
_isWorkflowParameter: function( value ) {
if ( String( value ).substring( 0, 1 ) === '$' ) {
return Utils.sanitize( value.substring( 2, value.length - 1 ) )
}
},
/** Is data input module/step */
_isDataStep: function( steps ) {
lst = $.isArray( steps ) ? steps : [ steps ] ;
client/galaxy/style/less/dataset.less
+8 -6
View File
@@ -234,12 +234,6 @@
.icon-btn {
margin-right: @icon-btn-margin;
}
// have to reapply here - dropdown wrapped in div
.icon-btn.visualization-btn {
margin-left: 0px;
border-radius: 0px 3px 3px 0px;
border-right: 1px solid @icon-btn-border;
}
}
.right {
.icon-btn-group();
@@ -297,6 +291,14 @@
}
}
.toolhelp {
width: 100%;
background:#eee;
border-radius: 3px;
margin-top: 5px;
padding: 3px;
}
.dataset-peek {
.list-item .vertical-spacing;
width: 100%;
config/datatypes_conf.xml.sample
+8 -1
View File
@@ -62,12 +62,15 @@
<datatype extension="data" type="galaxy.datatypes.data:Data" mimetype="application/octet-stream" max_optional_metadata_filesize="1048576" />
<datatype extension="data_manager_json" type="galaxy.datatypes.text:Json" mimetype="application/json" subclass="True" display_in_upload="False"/>
<datatype extension="dbn" type="galaxy.datatypes.sequence:DotBracket" display_in_upload="true" description="Dot-Bracket format is a text-based format for storing both an RNA sequence and its corresponding 2D structure." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Dbn"/>
<datatype extension="fai" type="galaxy.datatypes.tabular:Tabular" display_in_upload="true" subclass="True" description="A Fasta Index File is a text file consisting of lines each with five TAB-delimited columns : Name, Length, offset, linebases, Linewidth" description_url="http://www.htslib.org/doc/faidx.html" />
<datatype extension="fasta" type="galaxy.datatypes.sequence:Fasta" display_in_upload="true" description="A sequence in FASTA format consists of a single-line description, followed by lines of sequence data. The first character of the description line is a greater-than ('>') symbol in the first column. All lines should be shorter than 80 characters." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Fasta">
<converter file="fasta_to_tabular_converter.xml" target_datatype="tabular"/>
<converter file="fasta_to_bowtie_base_index_converter.xml" target_datatype="bowtie_base_index"/>
<converter file="fasta_to_bowtie_color_index_converter.xml" target_datatype="bowtie_color_index"/>
<converter file="fasta_to_2bit.xml" target_datatype="twobit"/>
<converter file="fasta_to_len.xml" target_datatype="len"/>
<converter file="fasta_to_fai.xml" target_datatype="fai"/>
<display file="igv/genome_fasta.xml" inherit="True"/>
</datatype>
<datatype extension="fastq" type="galaxy.datatypes.sequence:Fastq" display_in_upload="true" description="FASTQ format is a text-based format for storing both a biological sequence (usually nucleotide sequence) and its corresponding quality scores." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Fastq">
<converter file="fastq_to_fqtoc.xml" target_datatype="fqtoc"/>
@@ -399,7 +402,7 @@
<datatype extension="embl" type="galaxy.datatypes.data:Text" subclass="True"/>
<datatype extension="fitch" type="galaxy.datatypes.data:Text" subclass="True"/>
<datatype extension="gcg" type="galaxy.datatypes.data:Text" subclass="True"/>
<datatype extension="genbank" type="galaxy.datatypes.data:Text" subclass="True" edam_format="format_1936"/>
<datatype extension="genbank" type="galaxy.datatypes.sequence:Genbank" edam_format="format_1936"/>
<datatype extension="hennig86" type="galaxy.datatypes.data:Text" subclass="True"/>
<datatype extension="ig" type="galaxy.datatypes.data:Text" subclass="True"/>
<datatype extension="jackknifer" type="galaxy.datatypes.data:Text" subclass="True"/>
@@ -547,6 +550,8 @@
<datatype extension="maskinfo-asn1" type="galaxy.datatypes.data:GenericAsn1" mimetype="text/plain" subclass="True" display_in_upload="true" />
<datatype extension="maskinfo-asn1-binary" type="galaxy.datatypes.binary:GenericAsn1Binary" mimetype="application/octet-stream" subclass="True" display_in_upload="true" />
<datatype extension="pssm-asn1" type="galaxy.datatypes.data:GenericAsn1" mimetype="text/plain" subclass="True" display_in_upload="true" />
<!-- PlantTribes datatypes -->
<datatype extension="smat" type="galaxy.datatypes.estscan:Smat" display_in_upload="true" />
</registration>
<sniffers>
<!--
@@ -556,6 +561,7 @@
defined format first, followed by next-most rigidly defined,
and so on.
-->
<sniffer type="galaxy.datatypes.estscan:Smat"/>
<sniffer type="galaxy.datatypes.mothur:Sabund"/>
<sniffer type="galaxy.datatypes.mothur:Otu"/>
<sniffer type="galaxy.datatypes.mothur:GroupAbund"/>
@@ -626,6 +632,7 @@
<sniffer type="galaxy.datatypes.text:Html"/>
<sniffer type="galaxy.datatypes.images:Pdf"/>
<sniffer type="galaxy.datatypes.sequence:Axt"/>
<sniffer type="galaxy.datatypes.sequence:Genbank"/>
<sniffer type="galaxy.datatypes.interval:Bed"/>
<sniffer type="galaxy.datatypes.interval:CustomTrack"/>
<sniffer type="galaxy.datatypes.interval:Gtf"/>
config/galaxy.ini.sample
+10 -9
View File
@@ -203,7 +203,8 @@ paste.app_factory = galaxy.web.buildapp:app_factory
# Pass debug flag to conda commands.
#conda_debug = False
# conda channels to enable by default (http://conda.pydata.org/docs/custom-channels.html)
#conda_ensure_channels = conda-forge,r,bioconda,iuc
# the recommended channel order is the one from BioConda (https://github.com/bioconda/bioconda-recipes/blob/master/config.yml#L8)
#conda_ensure_channels = iuc,bioconda,r,defaults,conda-forge
# Set to True to instruct Galaxy to look for and install missing tool
# dependencies before each job runs.
#conda_auto_install = False
@@ -327,7 +328,7 @@ paste.app_factory = galaxy.web.buildapp:app_factory
# Interactive environment plugins root directory: where to look for interactive
# environment plugins. By default none will be loaded. Set to
# config/plugins/interactive_environments to load Galaxy's stock plugins
# (currently just IPython). These will require Docker to be configured and
# (currently just Jupyter). These will require Docker to be configured and
# have security considerations, so proceed with caution. The path is relative to the
# Galaxy root dir. To use an absolute path begin the path with '/'. This is a comma
# separated list.
@@ -552,7 +553,7 @@ nglims_config_file = tool-data/nglims.yaml
# URL (with schema http/https) of the Galaxy instance as accessible within your
# local network - if specified used as a default by pulsar file staging and
# IPython Docker container for communicating back with Galaxy via the API.
# Jupyter Docker container for communicating back with Galaxy via the API.
#galaxy_infrastructure_url = http://localhost:8080
# If the above URL cannot be determined ahead of time in dynamic environments
@@ -670,8 +671,8 @@ nglims_config_file = tool-data/nglims.yaml
# services based on Galaxy's session cookie. It will attempt to do this by
# default though you do need to install node+npm and do an npm install from
# `lib/galaxy/web/proxy/js`. It is generally more robust to configure this
# externally managing it however Galaxy is managed. If True Galaxy will only
# launch the proxy if it is actually going to be used (e.g. for IPython).
# externally, managing it however Galaxy is managed. If True, Galaxy will only
# launch the proxy if it is actually going to be used (e.g. for Jupyter).
#dynamic_proxy_manage=True
# As of 16.04 Galaxy supports multiple proxy types. The original NodeJS
@@ -780,12 +781,12 @@ nglims_config_file = tool-data/nglims.yaml
# See: https://developer.mozilla.org/en-US/docs/Web/HTTP/Access_control_CORS
#allowed_origin_hostnames = None
# Set the following to True to use IPython nbconvert to build HTML from IPython
# Set the following to True to use Jupyter nbconvert to build HTML from Jupyter
# notebooks in Galaxy histories. This process may allow users to execute
# arbitrary code or serve arbitrary HTML. If enabled, IPython must be
# arbitrary code or serve arbitrary HTML. If enabled, Jupyter must be
# available and on Galaxy's PATH, to do this run
# `pip install jinja2 pygments ipython` in Galaxy's virtualenv.
#trust_ipython_notebook_conversion = False
# `pip install jinja2 pygments jupyter` in Galaxy's virtualenv.
#trust_jupyter_notebook_conversion = False
# Debug enables access to various config options useful for development and
# debugging: use_lint, use_profile, use_printdebug and use_interactive. It
config/plugins/interactive_environments/jupyter/config/jupyter.ini.sample
+4
View File
@@ -40,3 +40,7 @@ command_inject = --sig-proxy=true -e DEBUG=false -e DEFAULT_CONTAINER_RUNTIME=12
# share data between the IE and Galaxy.
#docker_galaxy_temp_dir = None
# If your Docker container exposes more then one port, Galaxy needs to know to
# which ports it needs to connect. With this option you can specify the port number
# inside your container to which Galaxy should connect the UI.
# docker_connect_port = None
config/plugins/visualizations/charts/static/repository/build/registry.js
+1 -1
View File
File diff suppressed because one or more lines are too long
config/plugins/visualizations/charts/static/repository/build/registry.js.map
+1 -1
View File
File diff suppressed because one or more lines are too long
config/plugins/visualizations/charts/static/repository/visualizations/utilities/tabular-form.js
+1 -2
View File
@@ -53,13 +53,12 @@ define( [], function() {
{ value : 's' } ]
}
}
return {
title : '',
library : '',
tag : '',
keywords : '',
datatypes : [ 'tabular', 'csv' ],
datatypes : [ 'bed', 'bedgraph', 'bedstrict', 'bed6', 'bed12', 'chrint', 'customtrack', 'gff', 'gff3', 'gtf', 'interval', 'encodepeak', 'wig', 'scidx', 'fli', 'csv', 'tsv', 'eland', 'elandmulti', 'picard_interval_list', 'gatk_dbsnp', 'gatk_tranche', 'gatk_recal', 'ct', 'pileup', 'sam', 'taxonomy', 'tabular', 'vcf', 'xls' ],
use_panels : 'both',
settings : {
x_axis_label : axisLabel( 'x_axis_label' ),
config/plugins/visualizations/charts/webpack.config.js
+1 -2
View File
@@ -1,4 +1,3 @@
var webpack = require( 'webpack' );
var path = require( 'path' );
var root = path.join( __dirname, 'static/repository' );
var grunt = require( 'grunt' );
@@ -45,4 +44,4 @@ module.exports = {
resolve : {
root : root
}
};
};
display_applications/igv/genome_fasta.xml
+92
View File
@@ -0,0 +1,92 @@
<?xml version="1.0"?>
<display id="igv_fasta" version="1.0.0" name="display with IGV">
<!-- Load links from file: one line to one link -->
<dynamic_links site_type="igv" skip_startswith="#" id="0" name="1">
<!-- Define parameters by column from file, allow splitting on builds -->
<dynamic_param name="site_id" value="0"/>
<dynamic_param name="site_name" value="1"/>
<dynamic_param name="site_link" value="2"/>
<dynamic_param name="site_dbkeys" value="3" split="True" separator="," />
<dynamic_param name="site_organisms" value="4" split="True" separator="," />
<!-- Filter out some of the links based upon matching site_dbkeys to dataset dbkey -->
<filter>${$site_id.startswith( 'local_' ) or $dataset.dbkey in $site_dbkeys}</filter>
<!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
<url>${redirect_url}</url>
<param type="data" name="fasta_file" url="galaxy_${DATASET_HASH}.${dataset.ext}" />
<param type="template" name="site_organism" strip="True" >
#if ($dataset.dbkey in $site_dbkeys)
$site_organisms[ $site_dbkeys.index( $fasta_file.dbkey ) ]
#else:
$fasta_file.dbkey
#end if
</param>
<param type="template" name="jnlp" url="galaxy_${DATASET_HASH}.jnlp" viewable="True" mimetype="application/x-java-jnlp-file"><![CDATA[
<?xml version="1.0" encoding="utf-8"?>
<jnlp
spec="1.0+"
codebase="${site_link}">
<information>
<title>IGV 1.5</title>
<vendor>The Broad Institute</vendor>
<homepage href="http://www.broadinstitute.org/igv"/>
<description>IGV Software</description>
<description kind="short">IGV</description>
</information>
<security>
<all-permissions/>
</security>
<resources>
<j2se version="1.5+" initial-heap-size="256m" max-heap-size="1100m"/>
<jar href="igv.jar" download="eager" main="true"/>
<jar href="batik-codec.jar" download="eager"/>
<property name="apple.laf.useScreenMenuBar" value="true"/>
<property name="com.apple.mrj.application.growbox.intrudes" value="false"/>
<property name="com.apple.mrj.application.live-resize" value="true"/>
<property name="com.apple.macos.smallTabs" value="true"/>
</resources>
<resources os="Mac" arch="i386">
<property name="apple.awt.graphics.UseQuartz" value="false"/>
<nativelib href="hdfnative-macintel.jar"/>
</resources>
<resources os="Mac" arch="ppc">
<property name="apple.awt.graphics.UseQuartz" value="false"/>
<nativelib href="hdfnative-macppc.jar"/>
</resources>
<resources os="Mac" arch="PowerPC">
<property name="apple.awt.graphics.UseQuartz" value="false"/>
<nativelib href="hdfnative-macppc.jar"/>
</resources>
<resources os="Windows">
<property name="sun.java2d.noddraw" value="true"/>
<nativelib href="hdfnative-win.jar"/>
</resources>
<resources os="Linux">
<nativelib href="hdfnative-linux64.jar"/>
</resources>
<application-desc main-class="org.broad.igv.ui.IGVMainFrame">
<argument>-g</argument>
<argument>${fasta_file.url}</argument>
</application-desc>
</jnlp>
]]>
</param>
<param type="data" name="fai_file" dataset="fasta_file" url="galaxy_${DATASET_HASH}.fasta.fai" format="fai" />
<param type="template" name="redirect_url" strip="True" >
#if $site_id.startswith( 'local_' )
${site_link}?genome=${fasta_file.qp}&amp;merge=true&amp;name=name=${qp( ( $fasta_file.name or $DATASET_HASH ).replace( ',', ';' ) )}
#elif $site_id.startswith( 'web_link_' ):
${site_link}?sessionURL=${fasta_file.qp}&amp;genome=${fasta_file.qp}&amp;merge=true&amp;name=${qp( ( $fasta_file.name or $DATASET_HASH ).replace( ',', ';' ) )}
#else:
${jnlp.url}
#end if
</param>
</dynamic_links>
<dynamic_links from_data_table="igv_broad_genomes" skip_startswith="#" id="value" name="name">
<!-- Our input data table is one line per dbkey -->
<filter>${ $dataset.dbkey == $value }</filter>
<!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
<url>http://www.broadinstitute.org/igv/projects/current/igv.php?sessionURL=${fasta_file.qp}&amp;genome=${fasta_file.dbkey}&amp;merge=true&amp;name=${qp( ( $fasta_file.name or $DATASET_HASH ).replace( ',', ';' ) )}</url>
<param type="data" name="fasta_file" url="galaxy_${DATASET_HASH}.${dataset.ext}" />
</dynamic_links>
</display>
<!-- Delphine Lariviere - Dan Blankenberg -->
doc/source/admin/dependency_resolvers.rst
+1 -1
View File
@@ -182,7 +182,7 @@ debug
ensure_channels
conda channels to enable by default. See
http://conda.pydata.org/docs/custom-channels.html for more
information about channels. (default: conda-forge,r,bioconda,iuc).
information about channels. (default: iuc,bioconda,r,defaults,conda-forge).
auto_install
Set to True to instruct Galaxy to look for and install missing tool
doc/source/admin/interactive_environments.rst
+3 -8
View File
@@ -1,11 +1,6 @@
Galaxy Interactive Environments (GIEs)
======================================
GIEs were a new feature back in Galaxy 15.05, leading with the release of the
IPython IE. They were presented at GCC2015, and the RStudio IE was released as
part of 15.07. The IPython IE was superceded by a Project Jupyter IE in
Galaxy XXXXX.
A GIE is a Docker container, launched by Galaxy, proxied by Galaxy, with some
extra sugar inside the container to allow users to interact easily with their
Galaxy histories.
@@ -181,7 +176,7 @@ Please note you will need to be using apache2.4 with ``mod_proxy_wstunnel``.
proxy_redirect off;
}
# Project Jupyter / IPython specific. Other IEs may require their own routes.
# Project Jupyter specific. Other IEs may require their own routes.
location ~ ^/galaxy/gie_proxy/jupyter/(?<nbtype>[^/]+)/api/kernels(?<rest>.*?)$ {
proxy_pass http://localhost:8800/galaxy/gie_proxy/jupyter/$nbtype/api/kernels$rest;
proxy_redirect off;
@@ -229,8 +224,8 @@ Docker host with
$ docker -H tcp://gx-docker:4243 run -it busybox sh
So far so good! Now we need to configure Galaxy to use our new Docker host
to start the Interactive Environments. For that we need to edit the IPython GIE
configuration, ``ipython.ini`` to use our custom docker host
to start the Interactive Environments. For that we need to edit the Jupyter GIE
configuration, ``jupyter.ini`` to use our custom docker host
.. code-block:: ini
doc/source/admin/interactive_environments.svg
+6 -6
View File
@@ -17,7 +17,7 @@
inkscape:version="0.48.5 r10040"
sodipodi:docname="diagram.svg">
<title
id="title5754">Galaxy IPython Graphic</title>
id="title5754">Galaxy Jupyter Graphic</title>
<defs
id="defs4">
<marker
@@ -481,7 +481,7 @@
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title>Galaxy IPython Graphic</dc:title>
<dc:title>Galaxy Jupyter Graphic</dc:title>
<dc:creator>
<cc:Agent>
<dc:title>Björn Grüning, Torsten Houwaart, Eric Rasche</dc:title>
@@ -523,7 +523,7 @@
sodipodi:role="line"
id="tspan3805"
x="142.18896"
y="291.41959">Launch IPython</tspan><tspan
y="291.41959">Launch Jupyter</tspan><tspan
sodipodi:role="line"
x="142.18896"
y="303.91959"
@@ -701,7 +701,7 @@
sodipodi:role="line"
id="tspan5090-9-4"
x="214.56726"
y="364.25571">ipython_galaxy_notebook.ipynb</tspan></text>
y="364.25571">jupyter_galaxy_notebook.ipynb</tspan></text>
<rect
style="fill:#ffffff;stroke:#000000;stroke-width:0.625;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;stroke-dashoffset:0.25"
id="rect5068-6"
@@ -777,7 +777,7 @@
y="306.04703"
x="211.61868"
id="tspan5302"
sodipodi:role="line">IPython + scipy stack installed</tspan></text>
sodipodi:role="line">Jupyter + scipy stack installed</tspan></text>
<rect
ry="0"
y="281.74686"
@@ -817,7 +817,7 @@
y="378.99741"
x="215.05746"
sodipodi:role="line"
style="font-size:6px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:start;line-height:125%;writing-mode:lr-tb;text-anchor:start;fill:#000000;stroke:none;font-family:Sans;-inkscape-font-specification:Sans">IPython Webservice </tspan></text>
style="font-size:6px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:start;line-height:125%;writing-mode:lr-tb;text-anchor:start;fill:#000000;stroke:none;font-family:Sans;-inkscape-font-specification:Sans">Jupyter Webservice </tspan></text>
<text
xml:space="preserve"
style="font-size:6px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:start;line-height:125%;letter-spacing:0px;word-spacing:0px;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;font-family:Courier 10 Pitch;-inkscape-font-specification:Courier 10 Pitch"
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 175 KiB

After

Width:  |  Height:  |  Size: 175 KiB

doc/source/dev/faq.rst
+1 -1
View File
@@ -7,7 +7,7 @@ to provide relatively concrete answers for specific Galaxy development scenarios
... interact with the Galaxy database interactively?
----------------------------------------------------
This can be done with either IPython/Jupyter or a plain python console, depending on your preferences::
This can be done with either IPython or a plain Python console, depending on your preferences::
python -i scripts/db_shell.py
doc/source/dev/interactive_environments.rst
+18 -18
View File
@@ -14,21 +14,21 @@ plugins, for those familiar with developing those
.. code-block:: console
$ tree $GALAXY_ROOT/config/plugins/interactive_environments/ipython/
config/plugins/interactive_environments/ipython/
$ tree $GALAXY_ROOT/config/plugins/interactive_environments/jupyter
config/plugins/interactive_environments/jupyter
├── config
│ ├── ipython.ini
│ ├── ipython.ini.sample
│ └── ipython.xml
│ ├── jupyter.ini
│ ├── jupyter.ini.sample
│ └── jupyter.xml
├── static
│ └── js
│ └── ipython.js
│ └── jupyter.js
└── templates
├── ipython.mako
├── jupyter.mako
└── notebook.ipynb
We'll use the variable ``{gie}`` to stand in for the name of your GIE. It
should match ``[a-z]+``, like ``ipython`` or ``rstudio``. Here you can see the
should match ``[a-z]+``, like ``jupyter`` or ``rstudio``. Here you can see the
``config/`` directory with a ``{gie}.ini.sample`` providing docker and image
configuration, and then ``{gie}.xml`` which outlines that it is a GIE.
@@ -37,7 +37,7 @@ you are actively developing a GIE, you'll need to restart Galaxy after adding
any resources to that file, before they can be accessed in the browser.
Lastly, and most importantly, there's the templates folder. This normally just
contains ``{gie}.mako``, however the IPython file needs an extra template file.
contains ``{gie}.mako``, however the Jupyter file needs an extra template file.
First Steps, Configuration
--------------------------
@@ -186,9 +186,9 @@ We'll continue appending to our ``helloworld.mako`` the HTML code that's actuall
</body>
</html>
We've glossed over some of the features of this file, but most IEs do a significant amount of "magic" in the top half of the mako template. For instance, the IPython notebook:
We've glossed over some of the features of this file, but most IEs do a significant amount of "magic" in the top half of the mako template. For instance, the Jupyter notebook:
- If the user is trying to run the IPython GIE Visualization on an existing notebook in their history, then that gets loaded into the docker container via the temp directory and set as the default notebook
- If the user is trying to run the Jupyter GIE Visualization on an existing notebook in their history, then that gets loaded into the docker container via the temp directory and set as the default notebook
- Otherwise a default notebook is built for the user.
The RStudio notebook:
@@ -252,7 +252,7 @@ This function is very short. Historically, the GIE process involved a complex da
Since the NodeJS proxy takes care of authentication/authorization, we can
reduce the helloworld ``_handle_notebook_loading`` function to a simple
``append_notebook`` call. You may wish to look at the IPython and RStudio GIEs
``append_notebook`` call. You may wish to look at the Jupyter and RStudio GIEs
for examples of the complex things that can be done at every step.
The GIE Container
@@ -301,7 +301,7 @@ Here's an example Dockerfile for our helloworld container
ADD ./startup.sh /startup.sh
ADD ./monitor_traffic.sh /monitor_traffic.sh
# /import will be the universal mount-point for IPython
# /import will be the universal mount-point for Jupyter
# The Galaxy instance can copy in data that needs to be present to the
# container
RUN mkdir /import
@@ -426,11 +426,11 @@ As you can see, a LOT is going on! We'll break it down further:
- The user's API key is provided, allowing you to access datasets and
submit jobs on their behalf. If you have an environment like
IPython/RStudio, it is **highly recommended** that you provide some magic
Jupyter/RStudio, it is **highly recommended** that you provide some magic
by which the user can use their API key without embedding it in the
notebook. If you do embed it somehow in a document that gets saved to
their history, anyone can impersonate that user if they get a hold of it.
In the IPython GIE we have a variable that just runs
In the Jupyter GIE we have a variable that just runs
``os.environ.get('API_KEY')`` to avoid embedding it in their notebook.
- A CORS Origin is provided for very strict servers, but it may be easier
to simply void CORS requirements within the nginx proxy in your
@@ -449,7 +449,7 @@ As you can see, a LOT is going on! We'll break it down further:
which is a small utility script to provide API access to Galaxy to get
and fetch data, based on those environment variables.
- The ``HISTORY_ID`` of the current history the user is on is provided. In
the IPython/RStudio containers, we provide a dead simple method for users
the Jupyter/RStudio containers, we provide a dead simple method for users
to download datasets from their current history which will be visible to
them on the right hand side of their screen.
- A ``PROXY_PREFIX`` is provided which should be used in the nginx conf.
@@ -466,9 +466,9 @@ As you can see, a LOT is going on! We'll break it down further:
Most of this information is usually required to build friendly, easy-to-use
GIEs. One of the strong points of GIEs is their magic interaction with Galaxy.
Here we've mounted a volume read-only, but in real life you may wish to provide
connectivity like IPython and RStudio provide, allowing the user to load
connectivity like Jupyter and RStudio provide, allowing the user to load
datasets on demand for interactive analysis, and then to store analysis
artefacts (and a log of what was done inside the container, à la IPython's
artefacts (and a log of what was done inside the container, à la Jupyter's
"notebooks") back to their current history.
If everything went well, at this point you should see a directory listing show up:
doc/source/project/organization.rst
+1 -1
View File
@@ -56,7 +56,7 @@ Members
- Eric Rasche (@erasche)
- Nicola Soranzo (@nsoranzo)
- James Taylor (@jxtx)
- Nitesh Turaga (@nitesh1989)
- Nitesh Turaga (@nturaga)
- Marius van den Beek (@mvdbeek)
Membership
doc/source/releases/16.10.rst
+12
View File
@@ -41,6 +41,16 @@ Highlighted Enhancements
`Pull Request 2943`_
* Various enhancements for API driven installation of tool shed repositories.
Security
-------------------------------
.. security
* Fix an ACE vulnerability with gff filter tools (thanks to David Wyde).
`Commit c1e3087`_
* Escape filename in upload dialog, default and composite (thanks to David Wyde).
`Pull Request 3278`_
Enhancements
-------------------------------
@@ -679,3 +689,5 @@ Fixes
.. _Pull Request 3262: https://github.com/galaxyproject/galaxy/pull/3262
.. _Pull Request 3279: https://github.com/galaxyproject/galaxy/pull/3279
.. _Commit c1e3087: https://github.com/galaxyproject/galaxy/commit/c1e3087ca35dbca1b0328954fe4769d666d3f934
.. _Pull Request 3278: https://github.com/galaxyproject/galaxy/pull/3278
doc/source/releases/16.10_announce.rst
+26
View File
@@ -49,6 +49,32 @@ Deprecation Notices
* API-like endpoints (returning JSON) that start with ``<galaxy>/root/`` in the URL are now deprecated and will be removed in future releases. Please use proper API (e.g. ``/api/histories`` or ``/api/tools``) to replace your implementations.
* The ``jsonp`` parameter for search API endpoints in Tool Shed is deprecated and will be removed in the future. Instead the presence of a ``callback`` argument will trigger the JSONP format to be returned.
Security
========
An arbitrary code execution vulnerability in two tools and an XSS vulnerability
with the upload tool were identified this release cycle and have been fixed
concurrently with the release. In addition, the fixes have been backported to
older releases.
The Galaxy Committers would like to thank David Wyde for disclosing these
vulnerabilities. Details follow:
1. The vulnerable tools are "Filter GFF data by attribute" and "Filter GFF data
by feature count", both of which are provided with and enabled by default in
the Galaxy server. These two tools share code with each other and the more
general "Filter data on any column using simple expressions" tool. The
latter was fixed in a previous security disclosure but these GFF variants of
the tool were missed when updating the Filter tool. These tools use the
Python ``eval`` and ``exec`` functions and do not properly sanitize input to
these functions. The fix for this issue has been applied to Galaxy releases
back to v14.10 and can be found in `Commit c1e3087`_
2. An uploaded file's name was not properly sanitized, and so a specially
crafted filename uploaded to the Galaxy server could be used as an XSS
attack vector. The fix for this issue has been applied to Galaxy releases
back to v16.07 and can be found in `Pull Request 3278`_.
Release Notes
=============
lib/galaxy/config.py
+6 -2
View File
@@ -276,7 +276,11 @@ class Configuration( object ):
self.sanitize_whitelist_file = resolve_path( kwargs.get( 'sanitize_whitelist_file', "config/sanitize_whitelist.txt" ), self.root )
self.serve_xss_vulnerable_mimetypes = string_as_bool( kwargs.get( 'serve_xss_vulnerable_mimetypes', False ) )
self.allowed_origin_hostnames = self._parse_allowed_origin_hostnames( kwargs )
self.trust_ipython_notebook_conversion = string_as_bool( kwargs.get( 'trust_ipython_notebook_conversion', False ) )
if "trust_jupyter_notebook_conversion" in kwargs:
trust_jupyter_notebook_conversion = string_as_bool( kwargs.get( 'trust_jupyter_notebook_conversion', False ) )
else:
trust_jupyter_notebook_conversion = string_as_bool( kwargs.get( 'trust_ipython_notebook_conversion', False ) )
self.trust_jupyter_notebook_conversion = trust_jupyter_notebook_conversion
self.enable_old_display_applications = string_as_bool( kwargs.get( "enable_old_display_applications", "True" ) )
self.brand = kwargs.get( 'brand', None )
self.welcome_url = kwargs.get( 'welcome_url', '/static/welcome.html' )
@@ -706,7 +710,7 @@ class Configuration( object ):
return resolve_path( path, self.root )
def guess_galaxy_port(self):
# Code derived from IPython work ie.mako
# Code derived from Jupyter work ie.mako
config = configparser.SafeConfigParser({'port': '8080'})
if self.config_file:
config.read( self.config_file )
lib/galaxy/datatypes/converters/bam_to_bai.xml
+13 -15
View File
@@ -1,16 +1,14 @@
<tool id="CONVERTER_Bam_Bai_0" name="Bam to Bai" hidden="true">
<requirements>
<requirement type="package">samtools</requirement>
</requirements>
<command>samtools index $input1 $output1</command>
<inputs>
<page>
<param format="bam" name="input1" type="data" label="Choose BAM"/>
</page>
</inputs>
<outputs>
<data format="bai" name="output1"/>
</outputs>
<help>
</help>
<tool id="CONVERTER_Bam_Bai_0" name="Bam to Bai" version="1.0.0" hidden="true">
<requirements>
<requirement type="package">samtools</requirement>
</requirements>
<command>samtools index '$input1' '$output1'</command>
<inputs>
<param format="bam" name="input1" type="data" label="Choose BAM"/>
</inputs>
<outputs>
<data format="bai" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/bcf_bgzip_to_bcf_converter.xml
+9 -11
View File
@@ -1,14 +1,12 @@
<tool id="CONVERTER_bcf_bgzip_to_bcf" name="Convert BCF_BGZIP to BCF" version="0.0.1" hidden="false">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">bcf_bgzip_to_bcf_converter.py '$input1' '$output1'</command>
<inputs>
<page>
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>python '$__tool_directory__/bcf_bgzip_to_bcf_converter.py' '$input1' '$output1'</command>
<inputs>
<param format="bcf_bgzip" name="input1" type="data" label="Choose bcf_bgzip file"/>
</page>
</inputs>
<outputs>
<data format="bcf" name="output1"/>
</outputs>
<help>
</help>
</inputs>
<outputs>
<data format="bcf" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/bcf_to_bcf_bgzip_converter.xml
+9 -11
View File
@@ -1,14 +1,12 @@
<tool id="CONVERTER_bcf_to_bcf_bgzip" name="Convert BCF to BCF_BGZIP" version="0.0.1" hidden="false">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">bcf_to_bcf_bgzip_converter.py '$input1' '$output1'</command>
<inputs>
<page>
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>python '$__tool_directory__/bcf_to_bcf_bgzip_converter.py' '$input1' '$output1'</command>
<inputs>
<param format="bcf" name="input1" type="data" label="Choose bcf file"/>
</page>
</inputs>
<outputs>
<data format="bcf_bgzip" name="output1"/>
</outputs>
<help>
</help>
</inputs>
<outputs>
<data format="bcf_bgzip" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/bed_gff_or_vcf_to_bigwig_converter.xml
+6 -4
View File
@@ -5,11 +5,12 @@
<requirement type="package">bedtools</requirement>
</requirements>
<command>
<![CDATA[
## Remove comments and sort by chromosome.
grep -v '^#' $input | sort -k1,1 |
grep -v '^#' '$input' | sort -k1,1 |
## Generate coverage bedgraph.
bedtools genomecov -bg -i stdin -g $chromInfo
bedtools genomecov -bg -i stdin -g '$chromInfo'
## Only use split option for bed and gff/gff3/gtf.
#if $input.ext in [ 'bed', 'gff', 'gff3', 'gtf' ]:
@@ -18,10 +19,11 @@
## Streaming the bedgraph file to wigToBigWig is fast but very memory intensive; hence, this
## should only be used on systems with large RAM.
## | wigToBigWig stdin $chromInfo $output
## | wigToBigWig stdin '$chromInfo' '$output'
## This can be used anywhere.
> temp.bg ; bedGraphToBigWig temp.bg $chromInfo $output
> temp.bg && bedGraphToBigWig temp.bg '$chromInfo' '$output'
]]>
</command>
<inputs>
<param format="bed,gff,vcf" name="input" type="data" label="Choose input file"/>
lib/galaxy/datatypes/converters/bed_to_bgzip_converter.xml
+12 -14
View File
@@ -1,14 +1,12 @@
<tool id="CONVERTER_bed_to_bgzip_0" name="Convert BED to BGZIP" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">bgzip.py -P bed $input1 $output1</command>
<inputs>
<page>
<param format="bed" name="input1" type="data" label="Choose BED file"/>
</page>
</inputs>
<outputs>
<data format="bgzip" name="output1"/>
</outputs>
<help>
</help>
</tool>
<tool id="CONVERTER_bed_to_bgzip_0" name="Convert BED to BGZIP" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>python '$__tool_directory__/bgzip.py' -P bed '$input1' '$output1'</command>
<inputs>
<param format="bed" name="input1" type="data" label="Choose BED file"/>
</inputs>
<outputs>
<data format="bgzip" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/bed_to_fli_converter.xml
+13 -13
View File
@@ -1,13 +1,13 @@
<tool id="CONVERTER_bed_to_fli_0" name="Convert BED to Feature Location Index">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command interpreter="python">interval_to_fli.py -F bed '$input1' '$output1'</command>
<inputs>
<param format="bed" name="input1" type="data" label="Choose BED file"/>
</inputs>
<outputs>
<data format="fli" name="output1"/>
</outputs>
<help>
</help>
</tool>
<tool id="CONVERTER_bed_to_fli_0" name="Convert BED to Feature Location Index" version="1.0.0">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command>python '$__tool_directory__/interval_to_fli.py' -F bed '$input1' '$output1'</command>
<inputs>
<param format="bed" name="input1" type="data" label="Choose BED file"/>
</inputs>
<outputs>
<data format="fli" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/bed_to_gff_converter.xml
+13 -13
View File
@@ -1,13 +1,13 @@
<tool id="CONVERTER_bed_to_gff_0" name="Convert BED to GFF" version="2.0.0">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command interpreter="python">bed_to_gff_converter.py '$input1' '$output1'</command>
<inputs>
<param format="bed" name="input1" type="data" label="Choose BED file"/>
</inputs>
<outputs>
<data format="gff" name="output1"/>
</outputs>
<help>
</help>
</tool>
<tool id="CONVERTER_bed_to_gff_0" name="Convert BED to GFF" version="2.0.0">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command>python '$__tool_directory__/bed_to_gff_converter.py' '$input1' '$output1'</command>
<inputs>
<param format="bed" name="input1" type="data" label="Choose BED file"/>
</inputs>
<outputs>
<data format="gff" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/bed_to_interval_index_converter.xml
+12 -14
View File
@@ -1,14 +1,12 @@
<tool id="CONVERTER_bed_to_interval_index_0" name="Convert BED to Interval Index" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">interval_to_interval_index_converter.py $input1 $output1</command>
<inputs>
<page>
<param format="bed" name="input1" type="data" label="Choose BED file"/>
</page>
</inputs>
<outputs>
<data format="interval_index" name="output1"/>
</outputs>
<help>
</help>
</tool>
<tool id="CONVERTER_bed_to_interval_index_0" name="Convert BED to Interval Index" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>python '$__tool_directory__/interval_to_interval_index_converter.py' '$input1' '$output1'</command>
<inputs>
<param format="bed" name="input1" type="data" label="Choose BED file"/>
</inputs>
<outputs>
<data format="interval_index" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/bed_to_tabix_converter.xml
+9 -11
View File
@@ -1,15 +1,13 @@
<tool id="CONVERTER_bed_to_tabix_0" name="Convert BED to tabix" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">interval_to_tabix_converter.py -P bed '$input1' '$bgzip' '$output1'</command>
<inputs>
<page>
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>python '$__tool_directory__/interval_to_tabix_converter.py' -P bed '$input1' '$bgzip' '$output1'</command>
<inputs>
<param format="bed" name="input1" type="data" label="Choose BED file"/>
<param format="bgzip" name="bgzip" type="data" label="BGZIP file"/>
</page>
</inputs>
<outputs>
<data format="tabix" name="output1"/>
</outputs>
<help>
</help>
</inputs>
<outputs>
<data format="tabix" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.xml
+12 -14
View File
@@ -1,14 +1,12 @@
<tool id="CONVERTER_BedGraph_0" name="Index BedGraph for Track Viewer" hidden="true">
<!-- Used internally to generate track indexes -->
<command interpreter="python">bedgraph_to_array_tree_converter.py '$input' '$output'</command>
<inputs>
<page>
<param format="bedgraph" name="input" type="data" label="Choose BedGraph"/>
</page>
</inputs>
<outputs>
<data format="array_tree" name="output"/>
</outputs>
<help>
</help>
</tool>
<tool id="CONVERTER_BedGraph_0" name="Index BedGraph for Track Viewer" version="1.0.0" hidden="true">
<!-- Used internally to generate track indexes -->
<command>python '$__tool_directory__/bedgraph_to_array_tree_converter.py' '$input' '$output'</command>
<inputs>
<param format="bedgraph" name="input" type="data" label="Choose BedGraph"/>
</inputs>
<outputs>
<data format="array_tree" name="output"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/bedgraph_to_bigwig_converter.xml
+14 -14
View File
@@ -1,15 +1,15 @@
<tool id="CONVERTER_bedgraph_to_bigwig" name="Convert BedGraph to BigWig" hidden="true">
<!-- Used internally to generate track indexes -->
<requirements>
<requirement type="package">ucsc_tools</requirement>
</requirements>
<command>grep -v "^track" '$input' | wigToBigWig -clip stdin $chromInfo '$output'</command>
<inputs>
<param format="bedgraph" name="input" type="data" label="Choose wiggle"/>
</inputs>
<outputs>
<data format="bigwig" name="output"/>
</outputs>
<help>
</help>
<tool id="CONVERTER_bedgraph_to_bigwig" name="Convert BedGraph to BigWig" version="1.0.0" hidden="true">
<!-- Used internally to generate track indexes -->
<requirements>
<requirement type="package">ucsc_tools</requirement>
</requirements>
<command>grep -v "^track" '$input' | wigToBigWig -clip stdin '$chromInfo' '$output'</command>
<inputs>
<param format="bedgraph" name="input" type="data" label="Choose wiggle"/>
</inputs>
<outputs>
<data format="bigwig" name="output"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/encodepeak_to_bgzip_converter.xml
+18 -19
View File
@@ -1,19 +1,18 @@
<tool id="CONVERTER_encodepeak_to_bgzip_0" name="Convert ENCODEPeak to BGZIP" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">bgzip.py
-c ${input1.metadata.chromCol}
-s ${input1.metadata.startCol}
-e ${input1.metadata.endCol}
'$input1' '$output1'
</command>
<inputs>
<page>
<param format="ENCODEPeak" name="input1" type="data" label="Choose ENCODEPeak file"/>
</page>
</inputs>
<outputs>
<data format="bgzip" name="output1"/>
</outputs>
<help>
</help>
</tool>
<tool id="CONVERTER_encodepeak_to_bgzip_0" name="Convert ENCODEPeak to BGZIP" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>
python '$__tool_directory__/bgzip.py'
-c ${input1.metadata.chromCol}
-s ${input1.metadata.startCol}
-e ${input1.metadata.endCol}
'$input1' '$output1'
</command>
<inputs>
<param format="ENCODEPeak" name="input1" type="data" label="Choose ENCODEPeak file"/>
</inputs>
<outputs>
<data format="bgzip" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/encodepeak_to_tabix_converter.xml
+15 -16
View File
@@ -1,20 +1,19 @@
<tool id="CONVERTER_encodepeak_to_tabix_0" name="Convert ENCODEPeak to tabix" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">interval_to_tabix_converter.py
-c ${input1.metadata.chromCol}
-s ${input1.metadata.startCol}
-e ${input1.metadata.endCol}
'$input1' '$bgzip' '$output1'
</command>
<inputs>
<page>
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>
python '$__tool_directory__/interval_to_tabix_converter.py'
-c ${input1.metadata.chromCol}
-s ${input1.metadata.startCol}
-e ${input1.metadata.endCol}
'$input1' '$bgzip' '$output1'
</command>
<inputs>
<param format="encodepeak" name="input1" type="data" label="Choose ENCODEPeak file"/>
<param format="bgzip" name="bgzip" type="data" label="BGZIP file"/>
</page>
</inputs>
<outputs>
<data format="tabix" name="output1"/>
</outputs>
<help>
</help>
</inputs>
<outputs>
<data format="tabix" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/fasta_to_bowtie_base_index_converter.xml
+21 -20
View File
@@ -1,22 +1,23 @@
<tool id="CONVERTER_fasta_to_bowtie_base_index" name="Convert FASTA to Bowtie base space Index" version="1.0.0">
<requirements><requirement type='package'>bowtie</requirement></requirements>
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<requirements>
<requirement type="package">bowtie</requirement>
</requirements>
<command>
mkdir '${output.files_path}'
&amp;&amp; bowtie-build --quiet
-f
'$input' '${output.files_path}/${output.metadata.base_name}'
</command>
<inputs>
<param name="input" type="data" format="fasta" label="Fasta file"/>
</inputs>
<outputs>
<data name="output" format="bowtie_base_index"/>
</outputs>
<help>
</help>
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<requirements>
<requirement type="package">bowtie</requirement>
</requirements>
<command>
<![CDATA[
mkdir '${output.files_path}'
&& bowtie-build --quiet
-f
'$input' '${output.files_path}/${output.metadata.base_name}'
]]>
</command>
<inputs>
<param name="input" type="data" format="fasta" label="Fasta file"/>
</inputs>
<outputs>
<data name="output" format="bowtie_base_index"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/fasta_to_bowtie_color_index_converter.xml
+22 -21
View File
@@ -1,23 +1,24 @@
<tool id="CONVERTER_fasta_to_bowtie_color_index" name="Convert FASTA to Bowtie color space Index" version="1.0.0">
<requirements><requirement type='package'>bowtie</requirement></requirements>
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<requirements>
<requirement type="package">bowtie</requirement>
</requirements>
<command>
mkdir '${output.files_path}'
&amp;&amp; bowtie-build --quiet
--color
-f
'$input' '${output.files_path}/${output.metadata.base_name}'
</command>
<inputs>
<param name="input" type="data" format="fasta" label="Fasta file"/>
</inputs>
<outputs>
<data name="output" format="bowtie_color_index"/>
</outputs>
<help>
</help>
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<requirements>
<requirement type="package">bowtie</requirement>
</requirements>
<command>
<![CDATA[
mkdir '${output.files_path}'
&& bowtie-build --quiet
--color
-f
'$input' '${output.files_path}/${output.metadata.base_name}'
]]>
</command>
<inputs>
<param name="input" type="data" format="fasta" label="Fasta file"/>
</inputs>
<outputs>
<data name="output" format="bowtie_color_index"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/fasta_to_fai.xml
+14
View File
@@ -0,0 +1,14 @@
<tool id="CONVERTER_fasta_to_fai" name="Convert FASTA to fai file" version="1.0.0">
<requirements>
<requirement type="package" version="0.1.19">samtools</requirement>
</requirements>
<command><![CDATA[ln -s '$input' temp.fasta && samtools faidx temp.fasta && mv temp.fasta.fai '$output' && rm temp.fasta ]]></command>
<inputs>
<param name="input" type="data" format="fasta" label="Fasta file"/>
</inputs>
<outputs>
<data name="output" format="fai"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/fasta_to_len.xml
+1 -1
View File
@@ -1,7 +1,7 @@
<tool id="CONVERTER_fasta_to_len" name="Convert FASTA to len file" version="1.0.0">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command interpreter="python">fasta_to_len.py '$input' '$output' 0</command>
<command>python '$__tool_directory__/fasta_to_len.py' '$input' '$output' 0</command>
<inputs>
<param name="input" type="data" format="fasta" label="Fasta file"/>
</inputs>
lib/galaxy/datatypes/converters/fasta_to_tabular_converter.xml
+11 -11
View File
@@ -1,13 +1,13 @@
<tool id="CONVERTER_fasta_to_tabular" name="Convert FASTA to Tabular" version="1.0.1">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command interpreter="python">fasta_to_tabular_converter.py '$input' '$output'</command>
<inputs>
<param name="input" type="data" format="fasta" label="Fasta file"/>
</inputs>
<outputs>
<data name="output" format="tabular"/>
</outputs>
<help>
</help>
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command>python '$__tool_directory__/fasta_to_tabular_converter.py' '$input' '$output'</command>
<inputs>
<param name="input" type="data" format="fasta" label="Fasta file"/>
</inputs>
<outputs>
<data name="output" format="tabular"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/fastq_to_fqtoc.xml
+8 -10
View File
@@ -1,13 +1,11 @@
<tool id="CONVERTER_fastq_to_fqtoc0" name="Convert FASTQ files to seek locations" version="1.0.0" hidden="true">
<command interpreter="python">fastq_to_fqtoc.py '$input1' '$output1'</command>
<inputs>
<page>
<command>python '$__tool_directory__/fastq_to_fqtoc.py' '$input1' '$output1'</command>
<inputs>
<param format="fastq" name="input1" type="data" label="Choose FASTQ file"/>
</page>
</inputs>
<outputs>
<data format="fqtoc" name="output1"/>
</outputs>
<help>
</help>
</inputs>
<outputs>
<data format="fqtoc" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.xml
+10 -10
View File
@@ -1,12 +1,12 @@
<tool id="CONVERTER_fastqsolexa_to_fasta_0" name="Convert Fastqsolexa to Fasta" version="1.0.0">
<description>converts Fastqsolexa file to Fasta format</description>
<command interpreter="python">fastqsolexa_to_fasta_converter.py '$input' '$output'</command>
<inputs>
<param name="input" type="data" format="fastqsolexa" label="Choose Fastqsolexa file"/>
</inputs>
<outputs>
<data name="output" format="fasta"/>
</outputs>
<help>
</help>
<description>converts Fastqsolexa file to Fasta format</description>
<command>python '$__tool_directory__/fastqsolexa_to_fasta_converter.py' '$input' '$output'</command>
<inputs>
<param name="input" type="data" format="fastqsolexa" label="Choose Fastqsolexa file"/>
</inputs>
<outputs>
<data name="output" format="fasta"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.xml
+10 -10
View File
@@ -1,11 +1,11 @@
<tool id="CONVERTER_fastqsolexa_to_qual_0" name="Convert Fastqsolexa to Qual">
<command interpreter="python">fastqsolexa_to_qual_converter.py '$input1' '$output1' ${input1.extension}</command>
<inputs>
<param format="fastqsolexa" name="input1" type="data" label="Choose Fastqsolexa file"/>
</inputs>
<outputs>
<data format="qualsolexa" name="output1" />
</outputs>
<help>
</help>
<tool id="CONVERTER_fastqsolexa_to_qual_0" name="Convert Fastqsolexa to Qual" version="1.0.0">
<command>python '$__tool_directory__/fastqsolexa_to_qual_converter.py' '$input1' '$output1' ${input1.extension}</command>
<inputs>
<param format="fastqsolexa" name="input1" type="data" label="Choose Fastqsolexa file"/>
</inputs>
<outputs>
<data format="qualsolexa" name="output1" />
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/gff_to_bed_converter.xml
+13 -13
View File
@@ -1,13 +1,13 @@
<tool id="CONVERTER_gff_to_bed_0" name="Convert GFF to BED">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command interpreter="python">gff_to_bed_converter.py '$input1' '$output1'</command>
<inputs>
<param format="gff" name="input1" type="data" label="Choose GFF file"/>
</inputs>
<outputs>
<data format="bed" name="output1"/>
</outputs>
<help>
</help>
</tool>
<tool id="CONVERTER_gff_to_bed_0" name="Convert GFF to BED" version="1.0.0">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command>python '$__tool_directory__/gff_to_bed_converter.py' '$input1' '$output1'</command>
<inputs>
<param format="gff" name="input1" type="data" label="Choose GFF file"/>
</inputs>
<outputs>
<data format="bed" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/gff_to_bgzip_converter.xml
+12 -14
View File
@@ -1,14 +1,12 @@
<tool id="CONVERTER_gff_to_bgzip_0" name="Convert GFF to BGZIP" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">bgzip.py -P gff '$input1' '$output1'</command>
<inputs>
<page>
<param format="gff" name="input1" type="data" label="Choose GFF file"/>
</page>
</inputs>
<outputs>
<data format="bgzip" name="output1"/>
</outputs>
<help>
</help>
</tool>
<tool id="CONVERTER_gff_to_bgzip_0" name="Convert GFF to BGZIP" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>python '$__tool_directory__/bgzip.py' -P gff '$input1' '$output1'</command>
<inputs>
<param format="gff" name="input1" type="data" label="Choose GFF file"/>
</inputs>
<outputs>
<data format="bgzip" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/gff_to_fli_converter.xml
+13 -13
View File
@@ -1,13 +1,13 @@
<tool id="CONVERTER_gff_to_fli_0" name="Convert GFF to Feature Location Index">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command interpreter="python">interval_to_fli.py -F $input1.extension '$input1' '$output1'</command>
<inputs>
<param format="gff" name="input1" type="data" label="Choose GFF file"/>
</inputs>
<outputs>
<data format="fli" name="output1"/>
</outputs>
<help>
</help>
</tool>
<tool id="CONVERTER_gff_to_fli_0" name="Convert GFF to Feature Location Index" version="1.0.0">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command>python '$__tool_directory__/interval_to_fli.py' -F $input1.extension '$input1' '$output1'</command>
<inputs>
<param format="gff" name="input1" type="data" label="Choose GFF file"/>
</inputs>
<outputs>
<data format="fli" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/gff_to_interval_index_converter.xml
+12 -14
View File
@@ -1,14 +1,12 @@
<tool id="CONVERTER_gff_to_interval_index_0" name="Convert GFF to Interval Index" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">gff_to_interval_index_converter.py '$input1' '$output1'</command>
<inputs>
<page>
<param format="gff" name="input1" type="data" label="Choose GFF file"/>
</page>
</inputs>
<outputs>
<data format="interval_index" name="output1"/>
</outputs>
<help>
</help>
</tool>
<tool id="CONVERTER_gff_to_interval_index_0" name="Convert GFF to Interval Index" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>python '$__tool_directory__/gff_to_interval_index_converter.py' '$input1' '$output1'</command>
<inputs>
<param format="gff" name="input1" type="data" label="Choose GFF file"/>
</inputs>
<outputs>
<data format="interval_index" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/gff_to_tabix_converter.xml
+9 -11
View File
@@ -1,15 +1,13 @@
<tool id="CONVERTER_gff_to_tabix_0" name="Convert GFF to tabix" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">interval_to_tabix_converter.py -P gff '$input1' '$bgzip' '$output1'</command>
<inputs>
<page>
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>python '$__tool_directory__/interval_to_tabix_converter.py' -P gff '$input1' '$bgzip' '$output1'</command>
<inputs>
<param format="gff" name="input1" type="data" label="Choose GFF file"/>
<param format="bgzip" name="bgzip" type="data" label="BGZIP file"/>
</page>
</inputs>
<outputs>
<data format="tabix" name="output1"/>
</outputs>
<help>
</help>
</inputs>
<outputs>
<data format="tabix" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/interval_to_bed12_converter.xml
+17 -14
View File
@@ -1,15 +1,18 @@
<tool id="CONVERTER_interval_to_bed12_0" name="Convert Genomic Intervals To Strict BED12">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command interpreter="python">interval_to_bedstrict_converter.py '$output1' '$input1' ${input1.metadata.chromCol} ${input1.metadata.startCol} ${input1.metadata.endCol} ${input1.metadata.strandCol} ${input1.metadata.nameCol} ${input1.extension} 12</command>
<inputs>
<page>
<param format="interval" name="input1" type="data" label="Choose intervals"/>
</page>
</inputs>
<outputs>
<data format="bed12" name="output1"/>
</outputs>
<help>
</help>
<tool id="CONVERTER_interval_to_bed12_0" name="Convert Genomic Intervals To Strict BED12" version="1.0.0">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>
python '$__tool_directory__/interval_to_bedstrict_converter.py'
'$output1' '$input1' ${input1.metadata.chromCol}
${input1.metadata.startCol} ${input1.metadata.endCol}
${input1.metadata.strandCol} ${input1.metadata.nameCol}
${input1.extension} 12
</command>
<inputs>
<param format="interval" name="input1" type="data" label="Choose intervals"/>
</inputs>
<outputs>
<data format="bed12" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/interval_to_bed6_converter.xml
+11 -14
View File
@@ -1,15 +1,12 @@
<tool id="CONVERTER_interval_to_bed6_0" name="Convert Genomic Intervals To Strict BED6">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command interpreter="python">interval_to_bedstrict_converter.py '$output1' '$input1' ${input1.metadata.chromCol} ${input1.metadata.startCol} ${input1.metadata.endCol} ${input1.metadata.strandCol} ${input1.metadata.nameCol} ${input1.extension} 6</command>
<inputs>
<page>
<param format="interval" name="input1" type="data" label="Choose intervals"/>
</page>
</inputs>
<outputs>
<data format="bed6" name="output1"/>
</outputs>
<help>
</help>
<tool id="CONVERTER_interval_to_bed6_0" name="Convert Genomic Intervals To Strict BED6" version="1.0.0">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>python '$__tool_directory__/interval_to_bedstrict_converter.py' '$output1' '$input1' ${input1.metadata.chromCol} ${input1.metadata.startCol} ${input1.metadata.endCol} ${input1.metadata.strandCol} ${input1.metadata.nameCol} ${input1.extension} 6</command>
<inputs>
<param format="interval" name="input1" type="data" label="Choose intervals"/>
</inputs>
<outputs>
<data format="bed6" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/interval_to_bed_converter.xml
+12 -15
View File
@@ -1,15 +1,12 @@
<tool id="CONVERTER_interval_to_bed_0" name="Convert Genomic Intervals To BED">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command interpreter="python">interval_to_bed_converter.py '$output1' '$input1' ${input1.metadata.chromCol} ${input1.metadata.startCol} ${input1.metadata.endCol} ${input1.metadata.strandCol} ${input1.metadata.nameCol}</command>
<inputs>
<page>
<param format="interval" name="input1" type="data" label="Choose intervals"/>
</page>
</inputs>
<outputs>
<data format="bed" name="output1"/>
</outputs>
<help>
</help>
</tool>
<tool id="CONVERTER_interval_to_bed_0" name="Convert Genomic Intervals To BED" version="1.0.0">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>python '$__tool_directory__/interval_to_bed_converter.py' '$output1' '$input1' ${input1.metadata.chromCol} ${input1.metadata.startCol} ${input1.metadata.endCol} ${input1.metadata.strandCol} ${input1.metadata.nameCol}</command>
<inputs>
<param format="interval" name="input1" type="data" label="Choose intervals"/>
</inputs>
<outputs>
<data format="bed" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.xml
+11 -14
View File
@@ -1,15 +1,12 @@
<tool id="CONVERTER_interval_to_bedstrict_0" name="Convert Genomic Intervals To Strict BED">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command interpreter="python">interval_to_bedstrict_converter.py '$output1' '$input1' ${input1.metadata.chromCol} ${input1.metadata.startCol} ${input1.metadata.endCol} ${input1.metadata.strandCol} ${input1.metadata.nameCol} ${input1.extension}</command>
<inputs>
<page>
<param format="interval" name="input1" type="data" label="Choose intervals"/>
</page>
</inputs>
<outputs>
<data format="bedstrict" name="output1"/>
</outputs>
<help>
</help>
<tool id="CONVERTER_interval_to_bedstrict_0" name="Convert Genomic Intervals To Strict BED" version="1.0.0">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>python '$__tool_directory__/interval_to_bedstrict_converter.py' '$output1' '$input1' ${input1.metadata.chromCol} ${input1.metadata.startCol} ${input1.metadata.endCol} ${input1.metadata.strandCol} ${input1.metadata.nameCol} ${input1.extension}</command>
<inputs>
<param format="interval" name="input1" type="data" label="Choose intervals"/>
</inputs>
<outputs>
<data format="bedstrict" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/interval_to_bgzip_converter.xml
+18 -19
View File
@@ -1,19 +1,18 @@
<tool id="CONVERTER_interval_to_bgzip_0" name="Convert Interval to BGZIP" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">bgzip.py
-c ${input1.metadata.chromCol}
-s ${input1.metadata.startCol}
-e ${input1.metadata.endCol}
'$input1' '$output1'
</command>
<inputs>
<page>
<param format="interval" name="input1" type="data" label="Choose Interval file"/>
</page>
</inputs>
<outputs>
<data format="bgzip" name="output1"/>
</outputs>
<help>
</help>
</tool>
<tool id="CONVERTER_interval_to_bgzip_0" name="Convert Interval to BGZIP" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>
python '$__tool_directory__/bgzip.py'
-c ${input1.metadata.chromCol}
-s ${input1.metadata.startCol}
-e ${input1.metadata.endCol}
'$input1' '$output1'
</command>
<inputs>
<param format="interval" name="input1" type="data" label="Choose Interval file"/>
</inputs>
<outputs>
<data format="bgzip" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/interval_to_bigwig_converter.xml
+34 -37
View File
@@ -1,37 +1,34 @@
<tool id="CONVERTER_interval_to_bigwig_0" name="Convert Genomic Intervals To Coverage">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<requirements>
<requirement type="package">ucsc_tools</requirement>
<requirement type="package">bedtools</requirement>
</requirements>
<command>
## Remove comments and sort by chromosome.
grep -v '^#' '$input1' | sort -k${input1.metadata.chromCol},${input1.metadata.chromCol} |
## Create simple BED by cutting chrom, start, and end columns.
awk -v OFS=' ' '{print $${input1.metadata.chromCol},$${input1.metadata.startCol},$${input1.metadata.endCol} }' |
## Generate coverage bedgraph.
bedtools genomecov -bg -split -i stdin -g $chromInfo
## Streaming the bedgraph file to wigToBigWig is fast but very memory intensive; hence, this
## should only be used on systems with large RAM.
## | wigToBigWig stdin $chromInfo '$output'
## This can be used anywhere.
> temp.bg ; bedGraphToBigWig temp.bg $chromInfo '$output'
</command>
<inputs>
<page>
<param format="interval" name="input1" type="data" label="Choose intervals"/>
</page>
</inputs>
<outputs>
<data format="bigwig" name="output"/>
</outputs>
<help>
</help>
</tool>
<tool id="CONVERTER_interval_to_bigwig_0" name="Convert Genomic Intervals To Coverage" version="1.0.0">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<requirements>
<requirement type="package">ucsc_tools</requirement>
<requirement type="package">bedtools</requirement>
</requirements>
<command>
<![CDATA[
## Remove comments and sort by chromosome.
grep -v '^#' '$input1' | sort -k${input1.metadata.chromCol},${input1.metadata.chromCol} |
## Create simple BED by cutting chrom, start, and end columns.
awk -v OFS=' ' '{print $${input1.metadata.chromCol},$${input1.metadata.startCol},$${input1.metadata.endCol} }' |
## Generate coverage bedgraph.
bedtools genomecov -bg -split -i stdin -g '$chromInfo'
## Streaming the bedgraph file to wigToBigWig is fast but very memory intensive; hence, this
## should only be used on systems with large RAM.
## | wigToBigWig stdin '$chromInfo' '$output'
## This can be used anywhere.
> temp.bg && bedGraphToBigWig temp.bg '$chromInfo' '$output'
]]>
</command>
<inputs>
<param format="interval" name="input1" type="data" label="Choose intervals"/>
</inputs>
<outputs>
<data format="bigwig" name="output"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/interval_to_coverage.xml
+15 -17
View File
@@ -1,18 +1,16 @@
<tool id="CONVERTER_interval_to_coverage_0" name="Convert Genomic Intervals To COVERAGE">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command interpreter="python">interval_to_coverage.py '$input1' '$output1'
-1 ${input1.metadata.chromCol},${input1.metadata.startCol},${input1.metadata.endCol},${input1.metadata.strandCol}
-2 ${output1.metadata.chromCol},${output1.metadata.positionCol},${output1.metadata.forwardCol},${output1.metadata.reverseCol}
</command>
<inputs>
<page>
<param format="interval" name="input1" type="data" label="Choose intervals"/>
</page>
</inputs>
<outputs>
<data format="coverage" name="output1"/>
</outputs>
<help>
</help>
<tool id="CONVERTER_interval_to_coverage_0" name="Convert Genomic Intervals To COVERAGE" version="1.0.0">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>
python '$__tool_directory__/interval_to_coverage.py' '$input1' '$output1'
-1 ${input1.metadata.chromCol},${input1.metadata.startCol},${input1.metadata.endCol},${input1.metadata.strandCol}
-2 ${output1.metadata.chromCol},${output1.metadata.positionCol},${output1.metadata.forwardCol},${output1.metadata.reverseCol}
</command>
<inputs>
<param format="interval" name="input1" type="data" label="Choose intervals"/>
</inputs>
<outputs>
<data format="coverage" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/interval_to_interval_index_converter.xml
+18 -19
View File
@@ -1,19 +1,18 @@
<tool id="CONVERTER_interval_to_interval_index_0" name="Convert Interval to Interval Index" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">interval_to_interval_index_converter.py
-c ${input1.metadata.chromCol}
-s ${input1.metadata.startCol}
-e ${input1.metadata.endCol}
'$input1' '$output1'
</command>
<inputs>
<page>
<param format="interval" name="input1" type="data" label="Choose Interval file"/>
</page>
</inputs>
<outputs>
<data format="interval_index" name="output1"/>
</outputs>
<help>
</help>
</tool>
<tool id="CONVERTER_interval_to_interval_index_0" name="Convert Interval to Interval Index" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>
python '$__tool_directory__/interval_to_interval_index_converter.py'
-c ${input1.metadata.chromCol}
-s ${input1.metadata.startCol}
-e ${input1.metadata.endCol}
'$input1' '$output1'
</command>
<inputs>
<param format="interval" name="input1" type="data" label="Choose Interval file"/>
</inputs>
<outputs>
<data format="interval_index" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/interval_to_tabix_converter.xml
+15 -16
View File
@@ -1,20 +1,19 @@
<tool id="CONVERTER_interval_to_tabix_0" name="Convert Interval to tabix" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">interval_to_tabix_converter.py
-c ${input1.metadata.chromCol}
-s ${input1.metadata.startCol}
-e ${input1.metadata.endCol}
'$input1' '$bgzip' '$output1'
</command>
<inputs>
<page>
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>
python '$__tool_directory__/interval_to_tabix_converter.py'
-c ${input1.metadata.chromCol}
-s ${input1.metadata.startCol}
-e ${input1.metadata.endCol}
'$input1' '$bgzip' '$output1'
</command>
<inputs>
<param format="interval" name="input1" type="data" label="Choose Interval file"/>
<param format="bgzip" name="bgzip" type="data" label="BGZIP file"/>
</page>
</inputs>
<outputs>
<data format="tabix" name="output1"/>
</outputs>
<help>
</help>
</inputs>
<outputs>
<data format="tabix" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/len_to_linecount.xml
+5 -1
View File
@@ -1,7 +1,11 @@
<tool id="CONVERTER_len_to_linecount" name="Convert Len file to Linecount" version="1.0.0">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command>wc -l '$input' | awk '{print $1}' &gt; '$output' </command>
<command>
<![CDATA[
wc -l '$input' | awk '{print $1}' > '$output'
]]>
</command>
<inputs>
<param name="input" type="data" format="len" label="Fasta file"/>
</inputs>
lib/galaxy/datatypes/converters/lped_to_fped_converter.py
+1 -1
View File
@@ -81,7 +81,7 @@ def main():
"""call fbater
need to work with rgenetics composite datatypes
so in and out are html files with data in extrafiles path
<command interpreter="python">rg_convert_lped_fped.py '$input1/$input1.metadata.base_name'
<command>python '$__tool_directory__/rg_convert_lped_fped.py' '$input1/$input1.metadata.base_name'
'$output1' '$output1.extra_files_path'
</command>
"""
lib/galaxy/datatypes/converters/lped_to_fped_converter.xml
+15 -15
View File
@@ -1,15 +1,15 @@
<tool id="lped2fpedconvert" name="Convert lped to fped" version="0.01">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command interpreter="python">
lped_to_fped_converter.py '$input1.extra_files_path/$input1.metadata.base_name' '$output1' '$output1.files_path'
</command>
<inputs>
<param format="lped" name="input1" type="data" label="Choose linkage pedigree file"/>
</inputs>
<outputs>
<data format="fped" name="output1" metadata_source="input1"/>
</outputs>
<help>
</help>
</tool>
<tool id="lped2fpedconvert" name="Convert lped to fped" version="0.01">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command>
python '$__tool_directory__/lped_to_fped_converter.py' '$input1.extra_files_path/$input1.metadata.base_name' '$output1' '$output1.files_path'
</command>
<inputs>
<param format="lped" name="input1" type="data" label="Choose linkage pedigree file"/>
</inputs>
<outputs>
<data format="fped" name="output1" metadata_source="input1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/lped_to_pbed_converter.py
+1 -1
View File
@@ -81,7 +81,7 @@ def main():
"""
need to work with rgenetics composite datatypes
so in and out are html files with data in extrafiles path
<command interpreter="python">lped_to_pbed_converter.py '$input1/$input1.metadata.base_name'
<command>python '$__tool_directory__/lped_to_pbed_converter.py' '$input1/$input1.metadata.base_name'
'$output1' '$output1.extra_files_path' '${GALAXY_DATA_INDEX_DIR}/rg/bin/plink'
</command>
"""
lib/galaxy/datatypes/converters/lped_to_pbed_converter.xml
+16 -16
View File
@@ -1,16 +1,16 @@
<tool id="lped2pbedconvert" name="Convert lped to plink pbed" version="0.01">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command interpreter="python">
lped_to_pbed_converter.py '$input1.extra_files_path/$input1.metadata.base_name'
'$output1' '$output1.files_path' 'plink'
</command>
<inputs>
<param format="lped" name="input1" type="data" label="Choose linkage pedigree file"/>
</inputs>
<outputs>
<data format="pbed" name="output1" metadata_source="input1"/>
</outputs>
<help>
</help>
</tool>
<tool id="lped2pbedconvert" name="Convert lped to plink pbed" version="0.01">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command>
python '$__tool_directory__/lped_to_pbed_converter.py' '$input1.extra_files_path/$input1.metadata.base_name'
'$output1' '$output1.files_path' 'plink'
</command>
<inputs>
<param format="lped" name="input1" type="data" label="Choose linkage pedigree file"/>
</inputs>
<outputs>
<data format="pbed" name="output1" metadata_source="input1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/maf_to_fasta_converter.xml
+12 -15
View File
@@ -1,15 +1,12 @@
<tool id="CONVERTER_maf_to_fasta_0" name="Convert MAF to Fasta" version="1.0.1">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">maf_to_fasta_converter.py '$output1' '$input1'</command>
<inputs>
<page>
<param format="maf" name="input1" type="data" label="Choose MAF file"/>
</page>
</inputs>
<outputs>
<data format="fasta" name="output1"/>
</outputs>
<help>
</help>
<!-- <code file="maf_to_fasta_converter_code.py"/>-->
</tool>
<tool id="CONVERTER_maf_to_fasta_0" name="Convert MAF to Fasta" version="1.0.1">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>python '$__tool_directory__/maf_to_fasta_converter.py' '$output1' '$input1'</command>
<inputs>
<param format="maf" name="input1" type="data" label="Choose MAF file"/>
</inputs>
<outputs>
<data format="fasta" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/maf_to_interval_converter.xml
+12 -15
View File
@@ -1,15 +1,12 @@
<tool id="CONVERTER_maf_to_interval_0" name="Convert MAF to Genomic Intervals" version="1.0.2">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">maf_to_interval_converter.py '$output1' '$input1' '${input1.metadata.dbkey}'</command>
<inputs>
<page>
<param format="maf" name="input1" type="data" label="Choose MAF file"/>
</page>
</inputs>
<outputs>
<data format="interval" name="output1"/>
</outputs>
<help>
</help>
<!-- <code file="maf_to_interval_converter_code.py"/> -->
</tool>
<tool id="CONVERTER_maf_to_interval_0" name="Convert MAF to Genomic Intervals" version="1.0.2">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>python '$__tool_directory__/maf_to_interval_converter.py' '$output1' '$input1' '${input1.metadata.dbkey}'</command>
<inputs>
<param format="maf" name="input1" type="data" label="Choose MAF file"/>
</inputs>
<outputs>
<data format="interval" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py
+2 -3
View File
@@ -75,11 +75,10 @@ def main():
.. raw:: xml
<command interpreter="python">
pbed_ldreduced_converter.py '$input1.extra_files_path/$input1.metadata.base_name' '$winsize' '$winmove' '$r2thresh'
<command>
python '$__tool_directory__/pbed_ldreduced_converter.py' '$input1.extra_files_path/$input1.metadata.base_name' '$winsize' '$winmove' '$r2thresh'
'$output1' '$output1.files_path' 'plink'
</command>
"""
nparm = 7
if len(sys.argv) < nparm:
lib/galaxy/datatypes/converters/pbed_ldreduced_converter.xml
+12 -16
View File
@@ -1,18 +1,14 @@
<tool id="pbed2ldindepconvert" name="Convert plink pbed to ld reduced format" version="0.01">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command interpreter="python">
pbed_ldreduced_converter.py '$input1.extra_files_path/$input1.metadata.base_name' '60' '55' '0.1' '$output1' '$output1.files_path' 'plink'
</command>
<inputs>
<page>
<param format="pbed" name="input1" type="data" label="Choose a compressed Plink binary format genotype file"/>
</page>
</inputs>
<outputs>
<data format="ldindep" name="output1" metadata_source="input1"/>
</outputs>
<help>
</help>
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>
python '$__tool_directory__/pbed_ldreduced_converter.py' '$input1.extra_files_path/$input1.metadata.base_name' '60' '55' '0.1' '$output1' '$output1.files_path' 'plink'
</command>
<inputs>
<param format="pbed" name="input1" type="data" label="Choose a compressed Plink binary format genotype file"/>
</inputs>
<outputs>
<data format="ldindep" name="output1" metadata_source="input1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/pbed_to_lped_converter.py
+1 -1
View File
@@ -49,7 +49,7 @@ def main():
"""
need to work with rgenetics composite datatypes
so in and out are html files with data in extrafiles path
<command interpreter="python">pbed_to_lped_converter.py '$input1/$input1.metadata.base_name'
<command>python '$__tool_directory__/pbed_to_lped_converter.py' '$input1/$input1.metadata.base_name'
'$output1' '$output1.extra_files_path' '${GALAXY_DATA_INDEX_DIR}/rg/bin/plink'
</command>
"""
lib/galaxy/datatypes/converters/pbed_to_lped_converter.xml
+16 -16
View File
@@ -1,16 +1,16 @@
<tool id="pbed2lpedconvert" name="Convert plink pbed to linkage lped" version="0.01">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command interpreter="python">
pbed_to_lped_converter.py '$input1.extra_files_path/$input1.metadata.base_name'
'$output1' '$output1.files_path' 'plink'
</command>
<inputs>
<param format="pbed" name="input1" type="data" label="Choose compressed Plink binary format genotype file"/>
</inputs>
<outputs>
<data format="lped" name="output1" metadata_source="input1"/>
</outputs>
<help>
</help>
</tool>
<tool id="pbed2lpedconvert" name="Convert plink pbed to linkage lped" version="0.01">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command>
python '$__tool_directory__/pbed_to_lped_converter.py' '$input1.extra_files_path/$input1.metadata.base_name'
'$output1' '$output1.files_path' plink
</command>
<inputs>
<param format="pbed" name="input1" type="data" label="Choose compressed Plink binary format genotype file"/>
</inputs>
<outputs>
<data format="lped" name="output1" metadata_source="input1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.xml
+1 -1
View File
@@ -1,6 +1,6 @@
<tool id="CONVERTER_picard_interval_list_to_bed6" name="Convert Picard Interval List to BED6" version="1.0.0">
<description>converter</description>
<command interpreter="python">picard_interval_list_to_bed6_converter.py '$input' '$output'</command>
<command>python '$__tool_directory__/picard_interval_list_to_bed6_converter.py' '$input' '$output'</command>
<inputs>
<param name="input" type="data" format="picard_interval_list" label="Picard Interval List file"/>
</inputs>
lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.xml
+15 -15
View File
@@ -1,15 +1,15 @@
<tool id="CONVERTER_pileup_to_interval_index_0" name="Convert Pileup to Interval Index" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">pileup_to_interval_index_converter.py '$input' '$output'
</command>
<inputs>
<page>
<param format="pileup" name="input" type="data" label="Choose Pileup file"/>
</page>
</inputs>
<outputs>
<data format="interval_index" name="output"/>
</outputs>
<help>
</help>
</tool>
<tool id="CONVERTER_pileup_to_interval_index_0" name="Convert Pileup to Interval Index" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>
python '$__tool_directory__/pileup_to_interval_index_converter.py'
'$input' '$output'
</command>
<inputs>
<param format="pileup" name="input" type="data" label="Choose Pileup file"/>
</inputs>
<outputs>
<data format="interval_index" name="output"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/sam_to_bam.xml
+1 -1
View File
@@ -8,7 +8,7 @@
<requirements>
<requirement type="package">samtools</requirement>
</requirements>
<command interpreter="python">sam_to_bam.py '$input1' '$output'</command>
<command>python '$__tool_directory__/sam_to_bam.py' '$input1' '$output'</command>
<inputs>
<param name="input1" type="data" format="sam" label="SAM file"/>
</inputs>
lib/galaxy/datatypes/converters/sam_to_bigwig_converter.xml
+5 -3
View File
@@ -5,14 +5,16 @@
<requirement type="package">bedtools</requirement>
</requirements>
<command>
samtools view -bh '$input' | bedtools genomecov -bg -split -ibam stdin -g $chromInfo
<![CDATA[
samtools view -bh '$input' | bedtools genomecov -bg -split -ibam stdin -g '$chromInfo'
## Streaming the bedgraph file to wigToBigWig is fast but very memory intensive; hence, this
## should only be used on systems with large RAM.
## | wigToBigWig stdin $chromInfo '$output'
## | wigToBigWig stdin '$chromInfo' '$output'
## This can be used anywhere.
> temp.bg ; bedGraphToBigWig temp.bg $chromInfo '$output'
> temp.bg && bedGraphToBigWig temp.bg '$chromInfo' '$output'
]]>
</command>
<inputs>
<param format="bam" name="input" type="data" label="Choose BAM file"/>
lib/galaxy/datatypes/converters/tabular_to_dbnsfp.xml
+10 -10
View File
@@ -1,12 +1,12 @@
<tool id="tabular_to_dbnsfp" name="Convert tabular to dbnsfp" version="1.0.0">
<description></description>
<command interpreter="python">tabular_to_dbnsfp.py '$input' '$dbnsfp.extra_files_path/dbNSFP.gz'</command>
<inputs>
<param format="tabular" name="input" type="data" label="Choose a dbnsfp tabular file"/>
</inputs>
<outputs>
<data format="snpsiftdbnsfp" name="dbnsfp"/>
</outputs>
<help>
</help>
<description></description>
<command>python '$__tool_directory__/tabular_to_dbnsfp.py' '$input' '$dbnsfp.extra_files_path/dbNSFP.gz'</command>
<inputs>
<param format="tabular" name="input" type="data" label="Choose a dbnsfp tabular file"/>
</inputs>
<outputs>
<data format="snpsiftdbnsfp" name="dbnsfp"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/vcf_bgzip_to_tabix_converter.xml
+9 -11
View File
@@ -1,14 +1,12 @@
<tool id="CONVERTER_vcf_bgzip_to_tabix_0" name="Convert BGZ VCF to tabix" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">interval_to_tabix_converter.py -P 'vcf' '' '$input1' '$output1'</command>
<inputs>
<page>
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>python '$__tool_directory__/interval_to_tabix_converter.py' -P 'vcf' '' '$input1' '$output1'</command>
<inputs>
<param format="vcf_bgzip" name="input1" type="data" label="Choose BGZIP'd VCF file"/>
</page>
</inputs>
<outputs>
<data format="tabix" name="output1"/>
</outputs>
<help>
</help>
</inputs>
<outputs>
<data format="tabix" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/vcf_to_bgzip_converter.xml
+12 -14
View File
@@ -1,14 +1,12 @@
<tool id="CONVERTER_vcf_to_bgzip_0" name="Convert VCF to BGZIP" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">bgzip.py -P vcf '$input1' '$output1'</command>
<inputs>
<page>
<param format="vcf" name="input1" type="data" label="Choose Vcf file"/>
</page>
</inputs>
<outputs>
<data format="bgzip" name="output1"/>
</outputs>
<help>
</help>
</tool>
<tool id="CONVERTER_vcf_to_bgzip_0" name="Convert VCF to BGZIP" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>python '$__tool_directory__/bgzip.py' -P vcf '$input1' '$output1'</command>
<inputs>
<param format="vcf" name="input1" type="data" label="Choose Vcf file"/>
</inputs>
<outputs>
<data format="bgzip" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.xml
+12 -14
View File
@@ -1,14 +1,12 @@
<tool id="CONVERTER_vcf_to_interval_index_0" name="Convert VCF to Interval Index" version="1.0.0" hidden="true">
<description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description>
<command interpreter="python">vcf_to_interval_index_converter.py '$input1' '$output1'</command>
<inputs>
<page>
<param format="vcf" name="input1" type="data" label="Choose VCF file"/>
</page>
</inputs>
<outputs>
<data format="interval_index" name="output1"/>
</outputs>
<help>
</help>
</tool>
<tool id="CONVERTER_vcf_to_interval_index_0" name="Convert VCF to Interval Index" version="1.0.0" hidden="true">
<description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description>
<command>python '$__tool_directory__/vcf_to_interval_index_converter.py' '$input1' '$output1'</command>
<inputs>
<param format="vcf" name="input1" type="data" label="Choose VCF file"/>
</inputs>
<outputs>
<data format="interval_index" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/vcf_to_tabix_converter.xml
+9 -11
View File
@@ -1,15 +1,13 @@
<tool id="CONVERTER_vcf_to_tabix_0" name="Convert Vcf to tabix" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">interval_to_tabix_converter.py -P vcf '$input1' '$bgzip' '$output1'</command>
<inputs>
<page>
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>python '$__tool_directory__/interval_to_tabix_converter.py' -P vcf '$input1' '$bgzip' '$output1'</command>
<inputs>
<param format="vcf" name="input1" type="data" label="Choose Vcf file"/>
<param format="bgzip" name="bgzip" type="data" label="BGZIP file"/>
</page>
</inputs>
<outputs>
<data format="tabix" name="output1"/>
</outputs>
<help>
</help>
</inputs>
<outputs>
<data format="tabix" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip_converter.xml
+12 -14
View File
@@ -1,14 +1,12 @@
<tool id="CONVERTER_vcf_to_vcf_bgzip_0" name="Convert VCF to VCF_BGZIP" version="1.0.1" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command interpreter="python">vcf_to_vcf_bgzip.py '$input1' '$output1'</command>
<inputs>
<page>
<param format="vcf" name="input1" type="data" label="Choose Vcf file"/>
</page>
</inputs>
<outputs>
<data format="vcf_bgzip" name="output1"/>
</outputs>
<help>
</help>
</tool>
<tool id="CONVERTER_vcf_to_vcf_bgzip_0" name="Convert VCF to VCF_BGZIP" version="1.0.1" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<command>python '$__tool_directory__/vcf_to_vcf_bgzip.py' '$input1' '$output1'</command>
<inputs>
<param format="vcf" name="input1" type="data" label="Choose Vcf file"/>
</inputs>
<outputs>
<data format="vcf_bgzip" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/wig_to_bigwig_converter.xml
+19 -19
View File
@@ -1,20 +1,20 @@
<tool id="CONVERTER_wig_to_bigwig" name="Convert Wiggle to BigWig" hidden="true">
<!-- Used internally to generate track indexes -->
<requirements>
<requirement type="package">ucsc_tools</requirement>
</requirements>
<command>
grep -v "^track" '$input' | wigToBigWig -clip stdin $chromInfo '$output'
2&gt;&amp;1 || echo "Error running wiggle to bigwig converter." >&amp;2
</command>
<inputs>
<page>
<param format="wig" name="input" type="data" label="Choose wiggle"/>
</page>
</inputs>
<outputs>
<data format="bigwig" name="output"/>
</outputs>
<help>
</help>
<tool id="CONVERTER_wig_to_bigwig" name="Convert Wiggle to BigWig" version="1.0.0" hidden="true">
<!-- Used internally to generate track indexes -->
<requirements>
<requirement type="package">ucsc_tools</requirement>
</requirements>
<command>
<![CDATA[
grep -v "^track" '$input' | wigToBigWig -clip stdin '$chromInfo' '$output'
2>&1 || echo "Error running Wiggle to BigWig converter." >&2
]]>
</command>
<inputs>
<param format="wig" name="input" type="data" label="Choose wiggle"/>
</inputs>
<outputs>
<data format="bigwig" name="output"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.xml
+12 -14
View File
@@ -1,14 +1,12 @@
<tool id="CONVERTER_Wiggle_0" name="Index Wiggle for Track Viewer" hidden="true">
<!-- Used internally to generate track indexes -->
<command interpreter="python">wiggle_to_array_tree_converter.py '$input' '$output'</command>
<inputs>
<page>
<param format="wiggle" name="input" type="data" label="Choose wiggle"/>
</page>
</inputs>
<outputs>
<data format="array_tree" name="output"/>
</outputs>
<help>
</help>
</tool>
<tool id="CONVERTER_Wiggle_0" name="Index Wiggle for Track Viewer" version="1.0.0" hidden="true">
<!-- Used internally to generate track indexes -->
<command>python '$__tool_directory__/wiggle_to_array_tree_converter.py' '$input' '$output'</command>
<inputs>
<param format="wiggle" name="input" type="data" label="Choose wiggle"/>
</inputs>
<outputs>
<data format="array_tree" name="output"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/wiggle_to_simple_converter.xml
+11 -11
View File
@@ -1,11 +1,11 @@
<tool id="CONVERTER_wiggle_to_interval_0" name="Wiggle to Interval">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command interpreter="python">wiggle_to_simple_converter.py '$input' '$out_file1' </command>
<inputs>
<param format="wig" name="input" type="data" label="Convert"/>
</inputs>
<outputs>
<data format="interval" name="out_file1" />
</outputs>
</tool>
<tool id="CONVERTER_wiggle_to_interval_0" name="Wiggle to Interval" version="1.0.0">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command>python '$__tool_directory__/wiggle_to_simple_converter.py' '$input' '$out_file1'</command>
<inputs>
<param format="wig" name="input" type="data" label="Convert"/>
</inputs>
<outputs>
<data format="interval" name="out_file1" />
</outputs>
</tool>
lib/galaxy/datatypes/genetics.py
+20 -7
View File
@@ -14,6 +14,7 @@ august 20 2007
import logging
import os
import re
import sys
import urllib
from cgi import escape
@@ -28,6 +29,9 @@ from galaxy.web import url_for
gal_Log = logging.getLogger(__name__)
verbose = False
# https://genome.ucsc.edu/goldenpath/help/hgGenomeHelp.html
VALID_GENOME_GRAPH_MARKERS = re.compile('^(chr.*|RH.*|rs.*|SNP_.*|CN.*|A_.*)')
class GenomeGraphs( Tabular ):
"""
@@ -172,16 +176,25 @@ class GenomeGraphs( Tabular ):
>>> GenomeGraphs().sniff( fname )
True
"""
f = open(filename, 'r')
f.readline() # header
rows = [f.readline().split()[1:] for x in range(3)] # small sample, trimming first column
with open(filename, 'r') as f:
buf = f.read(1024)
rows = [l.split() for l in buf.splitlines()[1:4]] # break on lines and drop header, small sample
if len(rows) < 1:
return False
for row in rows:
if len(row) < 1:
# Must actually have at least one value
if len(row) < 2:
# Must actually have a marker and at least one numeric value
return False
first_val = row[0]
if not VALID_GENOME_GRAPH_MARKERS.match(first_val):
return False
rest_row = row[1:]
try:
[float(x) for x in row] # first col has been removed
except:
[float(x) for x in rest_row] # first col has been removed
except ValueError:
return False
return True
lib/galaxy/datatypes/sequence.py
+17
View File
@@ -1066,3 +1066,20 @@ class DotBracket ( Sequence ):
# Number of lines is less than 3
return False
class Genbank(data.Text):
"""Class representing a Genbank sequence"""
edam_format = "format_1936"
edam_data = "data_0849"
file_ext = "genbank"
def sniff(self, filename):
try:
with open(filename, 'r') as handle:
line = handle.readline().strip()
return line.startswith('LOCUS ')
except:
pass
return False
lib/galaxy/datatypes/sniff.py
+3
View File
@@ -353,6 +353,9 @@ def guess_ext( fname, sniff_order, is_multi_byte=False ):
>>> fname = get_test_fname('mothur_datatypetest_true.mothur.otu')
>>> guess_ext(fname, sniff_order)
'mothur.otu'
>>> fname = get_test_fname('1.gg')
>>> guess_ext(fname, sniff_order)
'gg'
"""
file_ext = None
for datatype in sniff_order:
lib/galaxy/datatypes/test/1.smat
+11
View File
@@ -0,0 +1,11 @@
FORMAT: hse.conf CODING REGION 6 3 1 s C+G: 0 43
-1 0 3 -2
2 1 -8 0
1 0 2 -4
-1 -1 4 -3
0 -1 3 -3
3 0 -8 0
0 0 2 -1
-3 0 4 -2
2 -1 1 -3
3 1 -9 -1
lib/galaxy/datatypes/text.py
+67 -4
View File
@@ -108,7 +108,7 @@ class Ipynb( Json ):
def set_peek( self, dataset, is_multi_byte=False ):
if not dataset.dataset.purged:
dataset.peek = get_file_peek( dataset.file_name, is_multi_byte=is_multi_byte )
dataset.blurb = "IPython Notebook"
dataset.blurb = "Jupyter Notebook"
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disc'
@@ -129,7 +129,7 @@ class Ipynb( Json ):
def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, **kwd):
config = trans.app.config
trust = getattr( config, 'trust_ipython_notebook_conversion', False )
trust = getattr( config, 'trust_jupyter_notebook_conversion', False )
if trust:
return self._display_data_trusted(trans, dataset, preview=preview, filename=filename, to_ext=to_ext, **kwd)
else:
@@ -144,13 +144,13 @@ class Ipynb( Json ):
ofilename = ofile_handle.name
ofile_handle.close()
try:
cmd = 'ipython nbconvert --to html --template full %s --output %s' % (dataset.file_name, ofilename)
cmd = 'jupyter nbconvert --to html --template full %s --output %s' % (dataset.file_name, ofilename)
log.info("Calling command %s" % cmd)
subprocess.call(cmd, shell=True)
ofilename = '%s.html' % ofilename
except:
ofilename = dataset.file_name
log.exception( 'Command "%s" failed. Could not convert the IPython Notebook to HTML, defaulting to plain text.' % cmd )
log.exception( 'Command "%s" failed. Could not convert the Jupyter Notebook to HTML, defaulting to plain text.' % cmd )
return open( ofilename )
def set_meta( self, dataset, **kwd ):
@@ -542,3 +542,66 @@ class SnpSiftDbNSFP( Text ):
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disc'
class Smat(Text):
file_ext = "smat"
def display_peek(self, dataset):
try:
return dataset.peek
except:
return "ESTScan scores matrices (%s)" % (nice_size(dataset.get_size()))
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte)
dataset.blurb = "ESTScan scores matrices"
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disc'
def sniff(self, filename):
"""
The use of ESTScan implies the creation of scores matrices which
reflect the codons preferences in the studied organisms. The
ESTScan package includes scripts for generating these files. The
output of these scripts consists of the matrices, one for each
isochor, and which look like this:
FORMAT: hse_4is.conf CODING REGION 6 3 1 s C+G: 0 44
-1 0 2 -2
2 1 -8 0
>>> from galaxy.datatypes.sniff import get_test_fname
>>> fname = get_test_fname('test_space.txt')
>>> Smat().sniff(fname)
False
>>> fname = get_test_fname('test_tab.bed')
>>> Smat().sniff(fname)
False
>>> fname = get_test_fname('1.smat')
>>> Smat().sniff(fname)
True
"""
line_no = 0
with open(filename, "r") as fh:
line_no += 1
if line_no > 10000:
return True
line = fh.readline(500)
if line_no == 1 and not line.startswith('FORMAT'):
# The first line is always the start of a format section.
return False
if not line.startswith('FORMAT'):
if line.find('\t') >= 0:
# Smat files are not tabular.
return False
items = line.split()
if len(items) != 4:
return False
for item in items:
# Make sure each item is an integer.
if re.match(r"[-+]?\d+$", item) is None:
return False
return True
lib/galaxy/jobs/__init__.py
+38 -2
View File
@@ -536,7 +536,8 @@ class JobConfiguration( object ):
rval.append( dict(
condition=resubmit.get('condition'),
destination=resubmit.get('destination'),
handler=resubmit.get('handler')
handler=resubmit.get('handler'),
delay=resubmit.get('delay'),
) )
return rval
@@ -764,7 +765,30 @@ class JobConfiguration( object ):
log.warning("Legacy destination with id '%s' could not be converted: Unknown runner plugin: %s" % (id, destination.runner))
class JobWrapper( object ):
class HasResourceParameters:
def get_resource_parameters( self, job=None ):
# Find the dymically inserted resource parameters and give them
# to rule.
if job is None:
job = self.get_job()
app = self.app
param_values = job.get_param_values( app, ignore_errors=True )
resource_params = {}
try:
resource_params_raw = param_values[ "__job_resource" ]
if resource_params_raw[ "__job_resource__select" ].lower() in [ "1", "yes", "true" ]:
for key, value in resource_params_raw.items():
resource_params[ key ] = value
except KeyError:
pass
return resource_params
class JobWrapper( object, HasResourceParameters ):
"""
Wraps a 'model.Job' with convenience methods for running processes and
state management.
@@ -1093,6 +1117,18 @@ class JobWrapper( object ):
job.set_state( job.states.PAUSED )
self.sa_session.add( job )
def is_ready_for_resubmission( self, job=None ):
if job is None:
job = self.get_job()
destination_params = job.destination_params
if "__resubmit_delay_seconds" in destination_params:
delay = float(destination_params["__resubmit_delay_seconds"])
if job.seconds_since_update < delay:
return False
return True
def mark_as_resubmitted( self, info=None ):
job = self.get_job()
self.sa_session.refresh( job )
lib/galaxy/jobs/actions/post.py
+8 -5
View File
@@ -125,13 +125,16 @@ class RenameDatasetAction(DefaultJobAction):
if len(tokens) > 1:
input_file_var = tokens[0].strip()
# Treat . as special symbol (breaks parameter names anyway)
# to allow access to repeat elements, for instance first
# repeat in cat1 would be something like queries_0.input2.
input_file_var = input_file_var.replace(".", "|")
for i in range(1, len(tokens)):
operations.append(tokens[i].strip())
# Treat . as special symbol (breaks parameter names anyway)
# to allow access to repeat elements, for instance first
# repeat in cat1 would be something like queries_0.input2.
# TODO: update the help text (input_terminals) on the action to
# show correct valid inputs.
input_file_var = input_file_var.replace(".", "|")
replacement = ""
# Lookp through inputs find one with "to_be_replaced" input
# variable name, and get the replacement name
lib/galaxy/jobs/command_factory.py
+4 -1
View File
@@ -86,7 +86,10 @@ def build_command(
if create_tool_working_directory:
# usually working will already exist, but it will not for task
# split jobs.
commands_builder.prepend_command("mkdir -p working; cd working")
# Remove the working directory incase this is for instance a SLURM re-submission.
# xref https://github.com/galaxyproject/galaxy/issues/3289
commands_builder.prepend_command("rm -rf working; mkdir -p working; cd working")
if include_work_dir_outputs:
__handle_work_dir_outputs(commands_builder, job_wrapper, runner, remote_command_params)
lib/galaxy/jobs/handler.py
+21 -16
View File
@@ -149,22 +149,27 @@ class JobHandlerQueue( object ):
self.queue.put( ( job.id, job.tool_id ) )
else:
# Already dispatched and running
job_wrapper = self.job_wrapper( job )
# Use the persisted destination as its params may differ from
# what's in the job_conf xml
job_destination = JobDestination(id=job.destination_id, runner=job.job_runner_name, params=job.destination_params)
# resubmits are not persisted (it's a good thing) so they
# should be added back to the in-memory destination on startup
try:
config_job_destination = self.app.job_config.get_destination( job.destination_id )
job_destination.resubmit = config_job_destination.resubmit
except KeyError:
log.warning( '(%s) Recovered destination id (%s) does not exist in job config (but this may be normal in the case of a dynamically generated destination)', job.id, job.destination_id )
job_wrapper.job_runner_mapper.cached_job_destination = job_destination
job_wrapper = self.__recover_job_wrapper( job )
self.dispatcher.recover( job, job_wrapper )
if self.sa_session.dirty:
self.sa_session.flush()
def __recover_job_wrapper(self, job):
# Already dispatched and running
job_wrapper = self.job_wrapper( job )
# Use the persisted destination as its params may differ from
# what's in the job_conf xml
job_destination = JobDestination(id=job.destination_id, runner=job.job_runner_name, params=job.destination_params)
# resubmits are not persisted (it's a good thing) so they
# should be added back to the in-memory destination on startup
try:
config_job_destination = self.app.job_config.get_destination( job.destination_id )
job_destination.resubmit = config_job_destination.resubmit
except KeyError:
log.debug( '(%s) Recovered destination id (%s) does not exist in job config (but this may be normal in the case of a dynamically generated destination)', job.id, job.destination_id )
job_wrapper.job_runner_mapper.cached_job_destination = job_destination
return job_wrapper
def __monitor( self ):
"""
Continually iterate the waiting jobs, checking is each is ready to
@@ -260,10 +265,10 @@ class JobHandlerQueue( object ):
for job in resubmit_jobs:
log.debug( '(%s) Job was resubmitted and is being dispatched immediately', job.id )
# Reassemble resubmit job destination from persisted value
jw = self.job_wrapper( job )
jw.job_runner_mapper.cached_job_destination = JobDestination( id=job.destination_id, runner=job.job_runner_name, params=job.destination_params )
self.increase_running_job_count(job.user_id, jw.job_destination.id)
self.dispatcher.put( jw )
jw = self.__recover_job_wrapper( job )
if jw.is_ready_for_resubmission(job):
self.increase_running_job_count(job.user_id, jw.job_destination.id)
self.dispatcher.put( jw )
# Iterate over new and waiting jobs and look for any that are
# ready to run
new_waiting_jobs = []
lib/galaxy/jobs/mapper.py
+1 -12
View File
@@ -130,18 +130,7 @@ class JobRunnerMapper( object ):
actual_args[ "user_email" ] = user_email
if "resource_params" in function_arg_names:
# Find the dymically inserted resource parameters and give them
# to rule.
param_values = self.__job_params( job )
resource_params = {}
try:
resource_params_raw = param_values[ "__job_resource" ]
if resource_params_raw[ "__job_resource__select" ].lower() in [ "1", "yes", "true" ]:
for key, value in resource_params_raw.iteritems():
resource_params[ key ] = value
except KeyError:
pass
actual_args[ "resource_params" ] = resource_params
actual_args[ "resource_params" ] = self.job_wrapper.get_resource_parameters( job )
if "workflow_invocation_uuid" in function_arg_names:
param_values = job.raw_param_dict( )
lib/galaxy/jobs/metrics/instrumenters/env.py
+1 -3
View File
@@ -9,9 +9,7 @@ log = logging.getLogger( __name__ )
class EnvFormatter( formatting.JobMetricFormatter ):
def format( self, key, value ):
return ( "%s (runtime environment variable)" % key, value )
pass
class EnvPlugin( InstrumentPlugin ):
lib/galaxy/jobs/runners/__init__.py
+16 -15
View File
@@ -368,6 +368,17 @@ class BaseJobRunner( object ):
except:
log.exception('Caught exception in runner state handler:')
def fail_job( self, job_state, exception=False ):
if getattr( job_state, 'stop_job', True ):
self.stop_job( self.sa_session.query( self.app.model.Job ).get( job_state.job_wrapper.job_id ) )
self._handle_runner_state( 'failure', job_state )
# Not convinced this is the best way to indicate this state, but
# something necessary
if not job_state.runner_state_handled:
job_state.job_wrapper.fail( getattr( job_state, 'fail_message', 'Job failed' ), exception=exception )
if job_state.job_wrapper.cleanup_job == "always":
job_state.cleanup()
def mark_as_resubmitted( self, job_state, info=None ):
job_state.job_wrapper.mark_as_resubmitted( info=info )
if not self.app.config.track_jobs_in_database:
@@ -382,12 +393,15 @@ class JobState( object ):
runner_states = Bunch(
WALLTIME_REACHED='walltime_reached',
MEMORY_LIMIT_REACHED='memory_limit_reached',
UNKNOWN_ERROR='unknown_error',
GLOBAL_WALLTIME_REACHED='global_walltime_reached',
OUTPUT_SIZE_LIMIT='output_size_limit'
)
def __init__( self ):
def __init__( self, job_wrapper, job_destination ):
self.runner_state_handled = False
self.job_wrapper = job_wrapper
self.job_destination = job_destination
def set_defaults( self, files_dir ):
if self.job_wrapper is not None:
@@ -421,16 +435,14 @@ class AsynchronousJobState( JobState ):
"""
def __init__( self, files_dir=None, job_wrapper=None, job_id=None, job_file=None, output_file=None, error_file=None, exit_code_file=None, job_name=None, job_destination=None ):
super( AsynchronousJobState, self ).__init__()
super( AsynchronousJobState, self ).__init__( job_wrapper, job_destination )
self.old_state = None
self._running = False
self.check_count = 0
self.start_time = None
self.job_wrapper = job_wrapper
# job_id is the DRM's job id, not the Galaxy job id
self.job_id = job_id
self.job_destination = job_destination
self.job_file = job_file
self.output_file = output_file
@@ -613,17 +625,6 @@ class AsynchronousJobRunner( BaseJobRunner ):
log.exception( "(%s/%s) Job wrapper finish method failed" % ( galaxy_id_tag, external_job_id ) )
job_state.job_wrapper.fail( "Unable to finish job", exception=True )
def fail_job( self, job_state ):
if getattr( job_state, 'stop_job', True ):
self.stop_job( self.sa_session.query( self.app.model.Job ).get( job_state.job_wrapper.job_id ) )
self._handle_runner_state( 'failure', job_state )
# Not convinced this is the best way to indicate this state, but
# something necessary
if not job_state.runner_state_handled:
job_state.job_wrapper.fail( getattr( job_state, 'fail_message', 'Job failed' ) )
if job_state.job_wrapper.cleanup_job == "always":
job_state.cleanup()
def mark_as_finished(self, job_state):
self.work_queue.put( ( self.finish_job, job_state ) )
lib/galaxy/jobs/runners/local.py
+24 -8
View File
@@ -77,9 +77,7 @@ class LocalJobRunner( BaseJobRunner ):
return job_file, exit_code_path
def queue_job( self, job_wrapper ):
# prepare the job
include_metadata = asbool( job_wrapper.job_destination.params.get( "embed_metadata_in_job", DEFAULT_EMBED_METADATA_IN_JOB ) )
if not self.prepare_job( job_wrapper, include_metadata=include_metadata ):
if not self._prepare_job_local( job_wrapper ):
return
stderr = stdout = ''
@@ -123,17 +121,16 @@ class LocalJobRunner( BaseJobRunner ):
log.debug('execution finished: %s' % command_line)
except Exception:
log.exception("failure running job %d" % job_wrapper.job_id)
job_wrapper.fail( "failure running job", exception=True )
self._fail_job_local(job_wrapper, "failure running job")
return
external_metadata = not asbool( job_wrapper.job_destination.params.get( "embed_metadata_in_job", DEFAULT_EMBED_METADATA_IN_JOB ) )
if external_metadata:
self._handle_metadata_externally( job_wrapper, resolve_requirements=True )
self._handle_metadata_if_needed(job_wrapper)
# Finish the job!
try:
job_wrapper.finish( stdout, stderr, exit_code )
except:
log.exception("Job wrapper finish method failed")
job_wrapper.fail("Unable to finish job", exception=True)
self._fail_job_local(job_wrapper, "Unable to finish job")
def stop_job( self, job ):
# if our local job has JobExternalOutputMetadata associated, then our primary job has to have already finished
@@ -168,6 +165,25 @@ class LocalJobRunner( BaseJobRunner ):
# local jobs can't be recovered
job_wrapper.change_state( model.Job.states.ERROR, info="This job was killed when Galaxy was restarted. Please retry the job." )
def _fail_job_local( self, job_wrapper, message ):
job_destination = job_wrapper.job_destination
job_state = JobState(job_wrapper, job_destination)
job_state.fail_message = message
job_state.stop_job = False
self.fail_job(job_state, exception=True)
def _handle_metadata_if_needed(self, job_wrapper):
if not self._embed_metadata(job_wrapper):
self._handle_metadata_externally(job_wrapper, resolve_requirements=True)
def _embed_metadata(self, job_wrapper):
job_destination = job_wrapper.job_destination
embed_metadata = asbool(job_destination.params.get("embed_metadata_in_job", DEFAULT_EMBED_METADATA_IN_JOB))
return embed_metadata
def _prepare_job_local(self, job_wrapper):
return self.prepare_job(job_wrapper, include_metadata=self._embed_metadata(job_wrapper))
def _check_pid( self, pid ):
try:
os.kill( pid, 0 )
lib/galaxy/jobs/runners/state_handlers/_safe_eval.py
+141
View File
@@ -0,0 +1,141 @@
import re
from ast import (
Module,
parse,
walk,
)
AST_NODE_TYPE_WHITELIST = [
'Expr', 'Load', 'Str', 'Num', 'BoolOp', 'Compare', 'And', 'Eq', 'NotEq',
'Or', 'GtE', 'LtE', 'Lt', 'Gt', 'BinOp', 'Add', 'Div', 'Sub', 'Mult', 'Mod',
'Pow', 'LShift', 'GShift', 'BitAnd', 'BitOr', 'BitXor', 'UnaryOp', 'Invert',
'Not', 'NotIn', 'In', 'Is', 'IsNot', 'List', 'Index', 'Subscript',
# Further checks
'Name', 'Call', 'Attribute',
]
BUILTIN_AND_MATH_FUNCTIONS = 'abs|all|any|bin|chr|cmp|complex|divmod|float|hex|int|len|long|max|min|oct|ord|pow|range|reversed|round|sorted|str|sum|type|unichr|unicode|log|exp|sqrt|ceil|floor'.split('|')
STRING_AND_LIST_METHODS = [ name for name in dir('') + dir([]) if not name.startswith('_') ]
VALID_FUNCTIONS = BUILTIN_AND_MATH_FUNCTIONS + STRING_AND_LIST_METHODS
def _check_name(ast_node, allowed_variables=[]):
name = ast_node.id
return name in (VALID_FUNCTIONS + allowed_variables)
if re.match(r'^c\d+$', name):
return True
return name in VALID_FUNCTIONS
def _check_attribute( ast_node ):
attribute_name = ast_node.attr
if attribute_name not in STRING_AND_LIST_METHODS:
return False
return True
def _check_call( ast_node ):
# If we are calling a function or method, it better be a math,
# string or list function.
ast_func = ast_node.func
ast_func_class = ast_func.__class__.__name__
if ast_func_class == 'Name':
if ast_func.id not in BUILTIN_AND_MATH_FUNCTIONS:
return False
elif ast_func_class == 'Attribute':
if not _check_attribute( ast_func ):
return False
else:
return False
return True
def _check_expression(text, allowed_variables=[]):
"""
>>> allowed_variables = ["c1", "c2", "c3", "c4", "c5"]
>>> _check_expression("c1", allowed_variables)
True
>>> _check_expression("eval('1+1')", allowed_variables)
False
>>> _check_expression("import sys", allowed_variables)
False
>>> _check_expression("[].__str__", allowed_variables)
False
>>> _check_expression("__builtins__", allowed_variables)
False
>>> _check_expression("'x' in globals", allowed_variables)
False
>>> _check_expression("'x' in [1,2,3]", allowed_variables)
True
>>> _check_expression("c3=='chr1' and c5>5", allowed_variables)
True
>>> _check_expression("c3=='chr1' and d5>5", allowed_variables) # Invalid d5 reference
False
>>> _check_expression("c3=='chr1' and c5>5 or exec", allowed_variables)
False
>>> _check_expression("type(c1) != type(1)", allowed_variables)
True
>>> _check_expression("c1.split(',')[1] == '1'", allowed_variables)
True
>>> _check_expression("exec 1", allowed_variables)
False
>>> _check_expression("str(c2) in [\\\"a\\\",\\\"b\\\"]", allowed_variables)
True
"""
try:
module = parse(text)
except SyntaxError:
return False
if not isinstance(module, Module):
return False
statements = module.body
if not len(statements) == 1:
return False
expression = statements[0]
if expression.__class__.__name__ != 'Expr':
return False
for ast_node in walk(expression):
ast_node_class = ast_node.__class__.__name__
# Toss out everything that is not a "simple" expression,
# imports, error handling, etc...
if ast_node_class not in AST_NODE_TYPE_WHITELIST:
return False
# White-list more potentially dangerous types AST elements.
if ast_node_class == 'Name':
# In order to prevent loading 'exec', 'eval', etc...
# put string restriction on names allowed.
if not _check_name(ast_node, allowed_variables):
return False
# Check only valid, white-listed functions are called.
elif ast_node_class == 'Call':
if not _check_call(ast_node):
return False
# Check only valid, white-listed attributes are accessed
elif ast_node_class == 'Attribute':
if not _check_attribute(ast_node):
return False
return True
def safe_eval(expression, variables):
"""
>>> safe_eval("moo", {"moo": 5})
5
>>> exception_thrown = False
>>> try: safe_eval("moo", {"cow": 5})
... except Exception as e: exception_thrown = True
>>> exception_thrown
True
"""
if not _check_expression(expression, allowed_variables=variables.keys()):
raise Exception("Invalid expression [%s], only a very simple subset of Python is allowed." % expression)
return eval(expression, globals(), variables)
lib/galaxy/jobs/runners/state_handlers/resubmit.py
+150 -17
View File
@@ -1,40 +1,114 @@
import logging
from datetime import datetime
__all__ = ('failure', )
log = logging.getLogger(__name__)
from galaxy import model
from galaxy.jobs.runners import JobState
from ._safe_eval import safe_eval
MESSAGES = dict(
walltime_reached='it reached the walltime',
memory_limit_reached='it exceeded the amount of allocated memory'
memory_limit_reached='it exceeded the amount of allocated memory',
unknown_error='it encountered an unknown error'
)
def eval_condition(condition, job_state):
runner_state = getattr(job_state, 'runner_state', None) or JobState.runner_states.UNKNOWN_ERROR
attempt = 1
now = datetime.utcnow()
last_running_state = None
last_queued_state = None
for state in job_state.job_wrapper.get_job().state_history:
if state.state == model.Job.states.RUNNING:
last_running_state = state
elif state.state == model.Job.states.QUEUED:
last_queued_state = state
elif state.state == model.Job.states.RESUBMITTED:
attempt = attempt + 1
seconds_running = 0
seconds_since_queued = 0
if last_running_state:
seconds_running = (now - last_running_state.create_time).total_seconds()
if last_queued_state:
seconds_since_queued = (now - last_queued_state.create_time).total_seconds()
condition_locals = {
"walltime_reached": runner_state == JobState.runner_states.WALLTIME_REACHED,
"memory_limit_reached": runner_state == JobState.runner_states.MEMORY_LIMIT_REACHED,
"unknown_error": JobState.runner_states.UNKNOWN_ERROR,
"any_failure": True,
"any_potential_job_failure": True, # Add a hook here - later on allow tools to describe things that are definitely input problems.
"attempt": attempt,
"seconds_running": seconds_running,
"seconds_since_queued": seconds_since_queued,
}
# Small optimization to eliminate the need to parse AST and eval for simple variables.
if condition in condition_locals:
return condition_locals[condition]
else:
return safe_eval(condition, condition_locals)
def failure(app, job_runner, job_state):
runner_state = getattr(job_state, 'runner_state', None)
if (not runner_state or
runner_state not in (job_state.runner_states.WALLTIME_REACHED,
job_state.runner_states.MEMORY_LIMIT_REACHED)):
# Leave handler quickly if no resubmit conditions specified or if the runner state doesn't allow resubmission.
resubmit_definitions = job_state.job_destination.get('resubmit')
if not resubmit_definitions:
return
runner_state = getattr(job_state, 'runner_state', None) or JobState.runner_states.UNKNOWN_ERROR
if (runner_state not in (JobState.runner_states.WALLTIME_REACHED,
JobState.runner_states.MEMORY_LIMIT_REACHED,
JobState.runner_states.UNKNOWN_ERROR)):
# not set or not a handleable runner state
return
_handle_resubmit_definitions(resubmit_definitions, app, job_runner, job_state)
def _handle_resubmit_definitions(resubmit_definitions, app, job_runner, job_state):
runner_state = getattr(job_state, 'runner_state', None) or JobState.runner_states.UNKNOWN_ERROR
# Setup environment for evaluating resubmission conditions and related expression.
expression_context = _ExpressionContext(job_state)
# Intercept jobs that hit the walltime and have a walltime or
# nonspecific resubmit destination configured
for resubmit in job_state.job_destination.get('resubmit'):
for resubmit in resubmit_definitions:
condition = resubmit.get('condition', None)
if condition and condition != runner_state:
if condition and not expression_context.safe_eval(condition):
# There is a resubmit defined for the destination but
# its condition is not for the encountered state
continue
log.info("(%s/%s) Job will be resubmitted to '%s' because %s at "
external_id = getattr(job_state, "job_id", None)
if external_id:
job_log_prefix = "(%s/%s)" % (job_state.job_wrapper.job_id, job_state.job_id)
else:
job_log_prefix = "(%s)" % (job_state.job_wrapper.job_id)
destination = resubmit['destination']
log.info("%s Job will be resubmitted to '%s' because %s at "
"the '%s' destination",
job_state.job_wrapper.job_id,
job_state.job_id,
resubmit['destination'],
MESSAGES[job_state.runner_state],
job_log_prefix,
destination,
MESSAGES[runner_state],
job_state.job_wrapper.job_destination.id )
# fetch JobDestination for the id or tag
new_destination = app.job_config.get_destination(
resubmit['destination'])
if destination:
new_destination = app.job_config.get_destination(destination)
else:
new_destination = job_state.job_destination
# Resolve dynamic if necessary
new_destination = (job_state.job_wrapper.job_runner_mapper
.cache_job_destination(new_destination))
@@ -43,8 +117,8 @@ def failure(app, job_runner, job_state):
job_state.job_wrapper.invalidate_external_metadata()
job = job_state.job_wrapper.get_job()
if resubmit.get('handler', None):
log.debug('(%s/%s) Job reassigned to handler %s',
job_state.job_wrapper.job_id, job_state.job_id,
log.debug('%s Job reassigned to handler %s',
job_log_prefix,
resubmit['handler'])
job.set_handler(resubmit['handler'])
job_runner.sa_session.add( job )
@@ -54,6 +128,16 @@ def failure(app, job_runner, job_state):
# resubmit
job_state.job_wrapper.job_runner_mapper \
.cached_job_destination = new_destination
# Handle delaying before resubmission if needed.
raw_delay = resubmit.get('delay')
if raw_delay:
delay = str(expression_context.safe_eval(str(raw_delay)))
try:
# ensure result acts like a number when persisted.
float(delay)
new_destination.params['__resubmit_delay_seconds'] = str(delay)
except ValueError:
log.warning("Cannot delay job with delay [%s], does not appear to be a number." % delay)
job_state.job_wrapper.set_job_destination(new_destination)
# Clear external ID (state change below flushes the change)
job.job_runner_external_id = None
@@ -62,6 +146,55 @@ def failure(app, job_runner, job_state):
job.params = {}
job_state.runner_state_handled = True
info = "This job was resubmitted to the queue because %s on its " \
"compute resource." % MESSAGES[job_state.runner_state]
"compute resource." % MESSAGES[runner_state]
job_runner.mark_as_resubmitted(job_state, info=info)
return
class _ExpressionContext(object):
def __init__(self, job_state):
self._job_state = job_state
self._lazy_context = None
def safe_eval(self, condition):
if condition.isdigit():
return int(condition)
if self._lazy_context is None:
runner_state = getattr(self._job_state, 'runner_state', None) or JobState.runner_states.UNKNOWN_ERROR
attempt = 1
now = datetime.utcnow()
last_running_state = None
last_queued_state = None
for state in self._job_state.job_wrapper.get_job().state_history:
if state.state == model.Job.states.RUNNING:
last_running_state = state
elif state.state == model.Job.states.QUEUED:
last_queued_state = state
elif state.state == model.Job.states.RESUBMITTED:
attempt = attempt + 1
seconds_running = 0
seconds_since_queued = 0
if last_running_state:
seconds_running = (now - last_running_state.create_time).total_seconds()
if last_queued_state:
seconds_since_queued = (now - last_queued_state.create_time).total_seconds()
self._lazy_context = {
"walltime_reached": runner_state == JobState.runner_states.WALLTIME_REACHED,
"memory_limit_reached": runner_state == JobState.runner_states.MEMORY_LIMIT_REACHED,
"unknown_error": JobState.runner_states.UNKNOWN_ERROR,
"any_failure": True,
"any_potential_job_failure": True, # Add a hook here - later on allow tools to describe things that are definitely input problems.
"attempt": attempt,
"seconds_running": seconds_running,
"seconds_since_queued": seconds_since_queued,
}
# Small optimization to eliminate the need to parse AST and eval for simple variables.
if condition in self._lazy_context:
return self._lazy_context[condition]
else:
return safe_eval(condition, self._lazy_context)
lib/galaxy/managers/collections.py
+7 -1
View File
@@ -126,7 +126,13 @@ class DatasetCollectionManager( object ):
if collection_type_description.has_subcollections( ):
# Nested collection - recursively create collections and update identifiers.
self.__recursively_create_collections( trans, element_identifiers )
elements = self.__load_elements( trans, element_identifiers )
new_collection = False
for element_identifier in element_identifiers:
if element_identifier.get("src") == "new_collection" and element_identifier.get('collection_type') == '':
new_collection = True
elements = self.__load_elements(trans, element_identifier['element_identifiers'])
if not new_collection:
elements = self.__load_elements( trans, element_identifiers )
# else if elements is set, it better be an ordered dict!
if elements is not self.ELEMENTS_UNINITIALIZED:
lib/galaxy/model/__init__.py
+4
View File
@@ -732,6 +732,10 @@ class Job( object, JobLike, Dictifiable ):
config_value = default
return config_value
@property
def seconds_since_update( self ):
return (galaxy.model.orm.now.now() - self.update_time).total_seconds()
class Task( object, JobLike ):
"""
lib/galaxy/tools/actions/__init__.py
+9 -1
View File
@@ -770,7 +770,15 @@ def determine_output_format(output, parameter_context, input_datasets, input_dat
if collection_name in input_dataset_collections:
try:
input_collection = input_dataset_collections[collection_name][0][0]
input_dataset = input_collection.collection[element_index].element_object
input_collection_collection = input_collection.collection
try:
input_element = input_collection_collection[element_index]
except KeyError:
for element in input_collection_collection.dataset_elements:
if element.element_identifier == element_index:
input_element = element
break
input_dataset = input_element.element_object
input_extension = input_dataset.ext
ext = input_extension
except Exception as e:
lib/galaxy/tools/deps/__init__.py
+29 -21
View File
@@ -174,15 +174,19 @@ class CachedDependencyManager(DependencyManager):
def build_cache(self, requirements, **kwds):
resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds)
cacheable_dependencies = [dep for req, dep in resolved_dependencies.items() if dep.cacheable]
hashed_requirements_dir = self.get_hashed_requirements_path(cacheable_dependencies)
if kwds.get('force_rebuild', False) and os.path.exists(hashed_requirements_dir):
try:
shutil.rmtree(hashed_requirements_dir)
except Exception:
log.warning("Could not delete cached requirements directory '%s'" % hashed_requirements_dir)
pass
[dep.build_cache(hashed_requirements_dir) for dep in cacheable_dependencies]
cacheable_dependencies = [dep for dep in resolved_dependencies.values() if dep.cacheable]
hashed_dependencies_dir = self.get_hashed_dependencies_path(cacheable_dependencies)
if os.path.exists(hashed_dependencies_dir):
if kwds.get('force_rebuild', False):
try:
shutil.rmtree(hashed_dependencies_dir)
except Exception:
log.warning("Could not delete cached dependencies directory '%s'" % hashed_dependencies_dir)
raise
else:
log.debug("Cached dependencies directory '%s' already exists, skipping build", hashed_dependencies_dir)
return
[dep.build_cache(hashed_dependencies_dir) for dep in cacheable_dependencies]
def dependency_shell_commands( self, requirements, **kwds ):
"""
@@ -192,24 +196,28 @@ class CachedDependencyManager(DependencyManager):
cached environments.
"""
resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds)
cacheable_dependencies = [dep for req, dep in resolved_dependencies.items() if dep.cacheable]
hashed_requirements_dir = self.get_hashed_requirements_path(cacheable_dependencies)
if os.path.exists(hashed_requirements_dir):
[dep.set_cache_path(hashed_requirements_dir) for dep in cacheable_dependencies]
cacheable_dependencies = [dep for dep in resolved_dependencies.values() if dep.cacheable]
hashed_dependencies_dir = self.get_hashed_dependencies_path(cacheable_dependencies)
if os.path.exists(hashed_dependencies_dir):
[dep.set_cache_path(hashed_dependencies_dir) for dep in cacheable_dependencies]
commands = [dep.shell_commands(req) for req, dep in resolved_dependencies.items()]
return commands
def hash_requirements(self, resolved_dependencies):
"""Return hash for requirements"""
def hash_dependencies(self, resolved_dependencies):
"""Return hash for dependencies"""
resolved_dependencies = [(dep.name, dep.version, dep.exact, dep.dependency_type) for dep in resolved_dependencies]
hash_str = json.dumps(sorted(resolved_dependencies))
return hash_util.new_secure_hash(hash_str)[:8] # short hash
def get_hashed_requirements_path(self, resolved_dependencies):
def get_hashed_dependencies_path(self, resolved_dependencies):
"""
Returns the path to the hashed requirements directory (but does not evaluate whether the path exists)
:param requirements:
:return:
Returns the path to the hashed dependencies directory (but does not evaluate whether the path exists).
:param resolved_dependencies: list of resolved dependencies
:type resolved_dependencies: list
:return: path
:rtype: str
"""
req_hashes = self.hash_requirements(resolved_dependencies)
return os.path.join(self.extra_config['tool_dependency_cache_dir'], req_hashes)
req_hashes = self.hash_dependencies(resolved_dependencies)
return os.path.abspath(os.path.join(self.extra_config['tool_dependency_cache_dir'], req_hashes))

Some files were not shown because too many files have changed in this diff Show More